Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Bug 290526 Parallelize part of the NSS build r=rrelyea
This still serializes many targets, but at least these targets
themself run their build in parallel. The main serialization
happens in nss/Makefile and nss/coreconf/rules.mk's all target.

We can't add these as real dependencies, as all Makefile snippets
use the same variable names. I tried to always run sub-makes to
hack in the depndencies, but these don't know of each other, so
targets very often run twice, and this breaks the build.

Having a tests:: target and a tests directory leads to misery (and
doesn't work), so it's renamed to check.

This just works with NSS_DISABLE_GTESTS=1 specified and is fixed
by a follow up patch, which removes the double-colon usage and
adds the directory dependencies!

Differential Revision: https://phabricator.services.mozilla.com/D69019

--HG--
extra : moz-landing-system : lando
  • Loading branch information
jmglogow committed May 5, 2020
1 parent 203b36d commit 0a164dd
Show file tree
Hide file tree
Showing 9 changed files with 100 additions and 85 deletions.
11 changes: 8 additions & 3 deletions Makefile
Expand Up @@ -47,9 +47,14 @@ include $(CORE_DEPTH)/coreconf/rules.mk
# (7) Execute "local" rules. (OPTIONAL). #
#######################################################################

nss_build_all: build_nspr all latest

nss_clean_all: clobber_nspr clobber
nss_build_all::
$(MAKE) build_nspr
$(MAKE) all
$(MAKE) latest

nss_clean_all::
$(MAKE) clobber_nspr
$(MAKE) clobber

NSPR_CONFIG_STATUS = $(CORE_DEPTH)/../nspr/$(OBJDIR_NAME)/config.status
NSPR_CONFIGURE = $(CORE_DEPTH)/../nspr/configure
Expand Down
5 changes: 3 additions & 2 deletions cmd/shlibsign/Makefile
Expand Up @@ -95,5 +95,6 @@ else
endif
endif

libs install :: $(CHECKLOC)

libs::
$(MAKE) install
$(MAKE) $(CHECKLOC)
2 changes: 0 additions & 2 deletions coreconf/Makefile
Expand Up @@ -11,5 +11,3 @@ DIRS = nsinstall

include $(DEPTH)/coreconf/config.mk
include $(DEPTH)/coreconf/rules.mk

export:: libs
2 changes: 1 addition & 1 deletion coreconf/README
Expand Up @@ -390,7 +390,7 @@ OVERVIEW of "rules.mk":
"Tinderbox" to automatically generate
binary releases on various platforms

tests:: automation tool used to run the
check:: automation tool used to run the
"regress" and "reporter" tools
on various regression test suites

Expand Down
1 change: 1 addition & 0 deletions coreconf/nsinstall/Makefile
Expand Up @@ -20,6 +20,7 @@ include $(DEPTH)/coreconf/config.mk

ifeq (,$(filter-out OS2 WIN%,$(OS_TARGET)))
PROGRAM =
TARGETS =
else
TARGETS = $(PROGRAM)
INSTALL = true
Expand Down
118 changes: 69 additions & 49 deletions coreconf/rules.mk
Expand Up @@ -13,7 +13,38 @@
# Double-Colon rules for utilizing the binary release model. #
#######################################################################

all:: export libs
all::
$(MAKE) -C coreconf/nsinstall program
$(MAKE) export
# pre-build child dir -> parent dir dependencies
# ckfw/builtins -> ckfw
IGNORE_DIRS=1 $(MAKE) -C lib/ckfw libs
# ckfw/builtins/testlib -> ckfw/builtins + base
$(MAKE) -C lib/base libs
IGNORE_DIRS=1 $(MAKE) -C lib/ckfw/builtins libs
$(MAKE) -C lib/util libs
# softoken + legacydb -> util + sqlite + freebl + dbm
$(MAKE) -C lib/dbm libs
$(MAKE) -C lib/sqlite libs
$(MAKE) -C lib/freebl libs
# nss -> util + certhi + cryptohi + pk11wrap + certdb + pki + dev (+ base) + pkix
$(MAKE) -C lib/certhigh libs
$(MAKE) -C lib/cryptohi libs
$(MAKE) -C lib/pk11wrap libs
$(MAKE) -C lib/certdb libs
$(MAKE) -C lib/pki libs
$(MAKE) -C lib/dev libs
$(MAKE) -C lib/libpkix libs
$(MAKE) -C lib/nss libs
# smime -> nss + pkcs12 + pkcs7
$(MAKE) -C lib/pkcs7 libs
$(MAKE) -C lib/pkcs12 libs
# all dependencies done -> finish libraries
$(MAKE) -C lib libs
# cmd library sectool
$(MAKE) -C cmd/lib libs
# everything still missing
$(MAKE) libs

ifeq ($(AUTOCLEAN),1)
autobuild:: clean export private_export libs program install
Expand All @@ -28,6 +59,14 @@ ifeq (,$(filter-out _WIN%,$(NS_USE_GCC)_$(OS_TARGET)))
USE_NT_C_SYNTAX=1
endif

ifdef DIRS
ifndef IGNORE_DIRS
$(DIRS)::
$(IGNORE_ERROR)@$(MAKE) -C $@ $(MAKECMDGOALS)
@$(CLICK_STOPWATCH)
endif
endif

#
# IMPORTS will always be associated with a component. Therefore,
# the "import" rule will always change directory to the top-level
Expand Down Expand Up @@ -60,19 +99,15 @@ ifeq ($(OS_TARGET),Darwin)
find $(SOURCE_MD_DIR)/lib -name "*.a" -exec $(RANLIB) {} \;
endif

export::
+$(LOOP_OVER_DIRS)
export:: $(DIRS)

private_export::
+$(LOOP_OVER_DIRS)
private_export:: $(DIRS)

release_export::
+$(LOOP_OVER_DIRS)
release_export:: $(DIRS)

release_classes::
+$(LOOP_OVER_DIRS)
release_classes:: $(DIRS)

libs program install:: $(TARGETS)
libs program install:: $(DIRS) $(TARGETS)
ifneq ($(LIBRARY),)
$(INSTALL) -m 664 $(LIBRARY) $(SOURCE_LIB_DIR)
endif
Expand All @@ -98,18 +133,14 @@ endif
ifneq ($(PROGRAMS),)
$(INSTALL) -m 775 $(PROGRAMS) $(SOURCE_BIN_DIR)
endif
+$(LOOP_OVER_DIRS)

tests::
+$(LOOP_OVER_DIRS)
check:: $(DIRS)

clean clobber::
clean clobber:: $(DIRS)
rm -rf $(ALL_TRASH)
+$(LOOP_OVER_DIRS)

realclean clobber_all::
realclean clobber_all:: $(DIRS)
rm -rf $(wildcard *.OBJ) dist $(ALL_TRASH)
+$(LOOP_OVER_DIRS)

#######################################################################
# Double-Colon rules for populating the binary release model. #
Expand Down Expand Up @@ -202,8 +233,7 @@ endif

endif

release_policy::
+$(LOOP_OVER_DIRS)
release_policy:: $(DIRS)

ifndef NO_MD_RELEASE
ifdef LIBRARY
Expand All @@ -223,15 +253,13 @@ ifndef NO_MD_RELEASE
endif
endif

release_md::
release_md:: $(DIRS)
ifneq ($(MD_LIB_RELEASE_FILES),)
$(INSTALL) -m 444 $(MD_LIB_RELEASE_FILES) $(SOURCE_RELEASE_PREFIX)/$(SOURCE_RELEASE_LIB_DIR)
endif
ifneq ($(MD_BIN_RELEASE_FILES),)
$(INSTALL) -m 555 $(MD_BIN_RELEASE_FILES) $(SOURCE_RELEASE_PREFIX)/$(SOURCE_RELEASE_BIN_DIR)
endif
+$(LOOP_OVER_DIRS)


alltags:
rm -f TAGS
Expand Down Expand Up @@ -768,38 +796,30 @@ endif
PUBLIC_EXPORT_DIR = $(SOURCE_XP_DIR)/public/$(MODULE)

ifneq ($(EXPORTS),)
$(PUBLIC_EXPORT_DIR)::
@if test ! -d $@; then \
echo Creating $@; \
$(NSINSTALL) -D $@; \
fi

export:: $(PUBLIC_EXPORT_DIR)
$(PUBLIC_EXPORT_DIR)/d:
@$(MAKE_OBJDIR)

export:: $(EXPORTS)
$(PUBLIC_EXPORT_DIR)/%: %
$(INSTALL) -m 444 $^ $(PUBLIC_EXPORT_DIR)

export:: $(BUILT_SRCS)
export:: $(addprefix $(PUBLIC_EXPORT_DIR)/,$(EXPORTS)) | $(PUBLIC_EXPORT_DIR)/d
endif

# Duplicate export rule for private exports, with different directories

PRIVATE_EXPORT_DIR = $(SOURCE_XP_DIR)/private/$(MODULE)

ifneq ($(PRIVATE_EXPORTS),)
$(PRIVATE_EXPORT_DIR)::
@if test ! -d $@; then \
echo Creating $@; \
$(NSINSTALL) -D $@; \
fi

private_export:: $(PRIVATE_EXPORT_DIR)
$(PRIVATE_EXPORT_DIR)/d:
@$(MAKE_OBJDIR)

private_export:: $(PRIVATE_EXPORTS)
$(PRIVATE_EXPORT_DIR)/%: %
$(INSTALL) -m 444 $^ $(PRIVATE_EXPORT_DIR)

private_export:: $(addprefix $(PRIVATE_EXPORT_DIR)/,$(PRIVATE_EXPORTS)) | $(PRIVATE_EXPORT_DIR)/d
else
private_export::
@echo There are no private exports.;
@echo "There are no private exports."
endif

##########################################################################
Expand All @@ -816,7 +836,7 @@ ifneq ($(BUILD_OPT),)
REGDATE = $(subst \ ,, $(shell $(PERL) $(CORE_DEPTH)/$(MODULE)/scripts/now))
endif

tests:: $(REGRESSION_SPEC)
check:: $(REGRESSION_SPEC)
cd $(PLATFORM); \
../$(SOURCE_MD_DIR)/bin/regress$(PROG_SUFFIX) specfile=../$(REGRESSION_SPEC) progress $(EXTRA_REGRESS_OPTIONS); \
if test ! -d $(TESTS_DIR); then \
Expand All @@ -830,8 +850,8 @@ ifneq ($(BUILD_OPT),)
echo "then run 'reporter specfile=$(RESULTS_DIR)/rptspec'"
endif
else
tests::
@echo Error: you didn't specify REGRESSION_SPEC in your manifest.mn file!;
check::
@echo "Error: you didn't specify REGRESSION_SPEC in your manifest.mn file!"
endif


Expand All @@ -855,6 +875,7 @@ endif

################################################################################

ifeq ($(MAKECMDGOALS),clean)
-include $(DEPENDENCIES)

ifneq (,$(filter-out OS2 WIN%,$(OS_TARGET)))
Expand Down Expand Up @@ -913,21 +934,20 @@ $(MKDEPEND):: $(MKDEPEND_DIR)/*.c $(MKDEPEND_DIR)/*.h
$(MAKE) -C $(MKDEPEND_DIR)

ifdef OBJS
depend:: $(MKDEPEND) $(MKDEPENDENCIES)
depend:: $(DIRS) $(MKDEPEND) $(MKDEPENDENCIES)
else
depend::
depend:: $(DIRS)
endif
+$(LOOP_OVER_DIRS)

dependclean::
dependclean:: $(DIRS)
rm -f $(MKDEPENDENCIES)
+$(LOOP_OVER_DIRS)

#-include $(NSINSTALL_DIR)/$(OBJDIR)/depend.mk

else
depend::
endif
endif

#
# HACK ALERT
Expand Down Expand Up @@ -972,5 +992,5 @@ $(filter $(OBJDIR)/%$(OBJ_SUFFIX),$(OBJS)): $(OBJDIR)/%$(OBJ_SUFFIX): $(DUMMY_DE
# Fake targets. Always run these rules, even if a file/directory with that
# name already exists.
#
.PHONY: all all_platforms alltags boot clean clobber clobber_all export install libs program realclean release $(OBJDIR)
.PHONY: all all_platforms alltags boot clean clobber clobber_all export install libs program realclean release

11 changes: 0 additions & 11 deletions coreconf/ruleset.mk
Expand Up @@ -203,15 +203,4 @@ ifdef SYSTEM_INCL_DIR
YOPT = -Y$(SYSTEM_INCL_DIR)
endif

ifdef DIRS
define SUBMAKE
+@echo "cd $2; $(MAKE) $1"
$(IGNORE_ERROR)@$(MAKE) -C $(2) $(1)
@$(CLICK_STOPWATCH)

endef

LOOP_OVER_DIRS = $(foreach dir,$(DIRS),$(call SUBMAKE,$@,$(dir)))
endif

MK_RULESET = included
32 changes: 16 additions & 16 deletions lib/Makefile
Expand Up @@ -54,22 +54,6 @@ ifndef NSS_DISABLE_LIBPKIX
LIBPKIX_SRCDIR = libpkix # Add the libpkix directory to DIRS.
endif

#######################################################################
# (5) Execute "global" rules. (OPTIONAL) #
#######################################################################

include $(CORE_DEPTH)/coreconf/rules.mk

#######################################################################
# (6) Execute "component" rules. (OPTIONAL) #
#######################################################################



#######################################################################
# (7) Execute "local" rules. (OPTIONAL). #
#######################################################################

ifeq ($(NSS_BUILD_UTIL_ONLY),1)
UTIL_SRCDIR = util
FREEBL_SRCDIR =
Expand All @@ -95,3 +79,19 @@ else
endif
endif
endif

#######################################################################
# (5) Execute "global" rules. (OPTIONAL) #
#######################################################################

include $(CORE_DEPTH)/coreconf/rules.mk

#######################################################################
# (6) Execute "component" rules. (OPTIONAL) #
#######################################################################



#######################################################################
# (7) Execute "local" rules. (OPTIONAL). #
#######################################################################
3 changes: 2 additions & 1 deletion lib/ckfw/Makefile
Expand Up @@ -6,7 +6,6 @@
include manifest.mn
include $(CORE_DEPTH)/coreconf/config.mk
include config.mk
include $(CORE_DEPTH)/coreconf/rules.mk

ifdef NOTDEF # was ifdef MOZILLA_CLIENT
NSS_BUILD_CAPI = 1
Expand All @@ -33,3 +32,5 @@ ifdef NSS_BUILD_CAPI
DIRS += capi
endif
endif

include $(CORE_DEPTH)/coreconf/rules.mk

0 comments on commit 0a164dd

Please sign in to comment.