Bug 757339 - Make expandlibs commands generate dependencies like gcc …

…does. r=ted

config/config.mk

@@ -731,9 +731,8 @@ OPTIMIZE_JARS_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/optimizeja
 
 CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
 
-EXPAND_LIBS_DEPS = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_deps.py
-EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_exec.py
-EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_gen.py
+EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_exec.py $(if $@,--depend $(MDDEPDIR)/$(basename $(@F)).pp --target $@)
+EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_gen.py $(if $@,--depend $(MDDEPDIR)/$(basename $(@F)).pp)
 EXPAND_AR = $(EXPAND_LIBS_EXEC) --extract -- $(AR)
 EXPAND_CC = $(EXPAND_LIBS_EXEC) --uselist -- $(CC)
 EXPAND_CCC = $(EXPAND_LIBS_EXEC) --uselist -- $(CCC)

config/expandlibs.py

@@ -27,9 +27,19 @@
   rules.
 '''
 from __future__ import with_statement
-import sys, os
+import sys, os, errno
 import expandlibs_config as conf
 
+def ensureParentDir(file):
+    '''Ensures the directory parent to the given file exists'''
+    dir = os.path.dirname(file)
+    if dir and not os.path.exists(dir):
+        try:
+            os.makedirs(dir)
+        except OSError, error:
+            if error.errno != errno.EEXIST:
+                raise
+
 def relativize(path):
     '''Returns a path relative to the current working directory, if it is
     shorter than the given path'''
@@ -116,5 +126,13 @@ def _expand_desc(self, arg):
             return objs
         return [arg]
 
+class ExpandLibsDeps(ExpandArgs):
+    '''Same as ExpandArgs, but also adds the library descriptor to the list'''
+    def _expand_desc(self, arg):
+        objs = super(ExpandLibsDeps, self)._expand_desc(arg)
+        if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
+            objs += [relativize(arg + conf.LIBS_DESC_SUFFIX)]
+        return objs
+
 if __name__ == '__main__':
     print " ".join(ExpandArgs(sys.argv[1:]))

config/expandlibs_exec.py

@@ -23,7 +23,7 @@
 from __future__ import with_statement
 import sys
 import os
-from expandlibs import ExpandArgs, relativize, isObject
+from expandlibs import ExpandArgs, relativize, isObject, ensureParentDir, ExpandLibsDeps
 import expandlibs_config as conf
 from optparse import OptionParser
 import subprocess
@@ -269,6 +269,10 @@ def _getSymbols(obj):
 
 def main():
     parser = OptionParser()
+    parser.add_option("--depend", dest="depend", metavar="FILE",
+        help="generate dependencies for the given execution and store it in the given file")
+    parser.add_option("--target", dest="target", metavar="FILE",
+        help="designate the target for dependencies")
     parser.add_option("--extract", action="store_true", dest="extract",
         help="when a library has no descriptor file, extract it first, when possible")
     parser.add_option("--uselist", action="store_true", dest="uselist",
@@ -280,6 +284,15 @@ def main():
 
     (options, args) = parser.parse_args()
 
+    if not options.target:
+        options.depend = False
+    if options.depend:
+        deps = ExpandLibsDeps(args)
+        # Filter out common command wrappers
+        while os.path.basename(deps[0]) in ['ccache', 'distcc']:
+            deps.pop(0)
+        # Remove command
+        deps.pop(0)
     with ExpandArgsMore(args) as args:
         if options.extract:
             args.extract()
@@ -295,7 +308,15 @@ def main():
                 with open(tmp) as file:
                     print >>sys.stderr, "".join(["    " + l for l in file.readlines()])
             sys.stderr.flush()
-        exit(subprocess.call(args))
+        ret = subprocess.call(args)
+        if ret:
+            exit(ret)
+    if not options.depend:
+        return
+    ensureParentDir(options.depend)
+    with open(options.depend, 'w') as depfile:
+        depfile.write("%s : %s\n" % (options.target, ' '.join(dep for dep in deps if os.path.isfile(dep) and dep != options.target)))
+
 
 if __name__ == '__main__':
     main()

config/expandlibs_gen.py

@@ -5,10 +5,12 @@
 '''Given a list of object files and library names, prints a library
 descriptor to standard output'''
 
+from __future__ import with_statement
 import sys
 import os
 import expandlibs_config as conf
-from expandlibs import LibDescriptor, isObject
+from expandlibs import LibDescriptor, isObject, ensureParentDir, ExpandLibsDeps
+from optparse import OptionParser
 
 def generate(args):
     desc = LibDescriptor()
@@ -26,4 +28,20 @@ def generate(args):
     return desc
 
 if __name__ == '__main__':
-    print generate(sys.argv[1:])
+    parser = OptionParser()
+    parser.add_option("--depend", dest="depend", metavar="FILE",
+        help="generate dependencies for the given execution and store it in the given file")
+    parser.add_option("-o", dest="output", metavar="FILE",
+        help="send output to the given file")
+
+    (options, args) = parser.parse_args()
+    if not options.output:
+        raise Exception("Missing option: -o")
+
+    ensureParentDir(options.output)
+    with open(options.output, 'w') as outfile:
+        print >>outfile, generate(args)
+    if options.depend:
+        ensureParentDir(options.depend)
+        with open(options.depend, 'w') as depfile:
+            depfile.write("%s : %s\n" % (options.output, ' '.join(ExpandLibsDeps(args))))

config/rules.mk

@@ -643,23 +643,6 @@ ifneq (,$(filter-out %.$(LIB_SUFFIX),$(SHARED_LIBRARY_LIBS)))
 $(error SHARED_LIBRARY_LIBS must contain .$(LIB_SUFFIX) files only)
 endif
 
-# Create dependencies on static (and shared EXTRA_DSO_LIBS) libraries
-ifneq (,$(strip $(filter %.$(LIB_SUFFIX),$(LIBS) $(EXTRA_DSO_LDOPTS)) $(SHARED_LIBRARY_LIBS) $(EXTRA_DSO_LIBS)))
-$(MDDEPDIR)/libs: Makefile.in
-	@mkdir -p $(MDDEPDIR)
-	@$(EXPAND_LIBS_DEPS) LIBS_DEPS = $(filter %.$(LIB_SUFFIX),$(LIBS)) , \
-	                     SHARED_LIBRARY_LIBS_DEPS = $(SHARED_LIBRARY_LIBS) , \
-	                     DSO_LDOPTS_DEPS = $(EXTRA_DSO_LIBS) $(filter %.$(LIB_SUFFIX), $(EXTRA_DSO_LDOPTS)) > $@
-
-ifneq (,$(wildcard $(MDDEPDIR)/libs))
-include $(MDDEPDIR)/libs
-endif
-
-$(MDDEPDIR)/libs: $(wildcard $(filter %.$(LIBS_DESC_SUFFIX),$(LIBS_DEPS) $(SHARED_LIBRARY_LIBS_DEPS) $(DSO_LDOPTS_DEPS)))
-
-EXTRA_DEPS += $(MDDEPDIR)/libs
-endif
-
 HOST_LIBS_DEPS = $(filter %.$(LIB_SUFFIX),$(HOST_LIBS))
 
 # Dependencies which, if modified, should cause everything to rebuild
@@ -752,7 +735,7 @@ alltags:
 # PROGRAM = Foo
 # creates OBJS, links with LIBS to create Foo
 #
-$(PROGRAM): $(PROGOBJS) $(LIBS_DEPS) $(EXTRA_DEPS) $(EXE_DEF_FILE) $(RESFILE) $(GLOBAL_DEPS)
+$(PROGRAM): $(PROGOBJS) $(EXTRA_DEPS) $(EXE_DEF_FILE) $(RESFILE) $(GLOBAL_DEPS)
 	@$(RM) $@.manifest
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
 	$(EXPAND_LD) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_GLUE_PROGRAM_LDFLAGS) $(PROGOBJS) $(RESFILE) $(LIBS) $(EXTRA_LIBS) $(OS_LIBS)
@@ -820,7 +803,7 @@ endif
 # SIMPLE_PROGRAMS = Foo Bar
 # creates Foo.o Bar.o, links with LIBS to create Foo, Bar.
 #
-$(SIMPLE_PROGRAMS): %$(BIN_SUFFIX): %.$(OBJ_SUFFIX) $(LIBS_DEPS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
+$(SIMPLE_PROGRAMS): %$(BIN_SUFFIX): %.$(OBJ_SUFFIX) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
 	$(EXPAND_LD) -nologo -out:$@ -pdb:$(LINK_PDBFILE) $< $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_GLUE_PROGRAM_LDFLAGS) $(LIBS) $(EXTRA_LIBS) $(OS_LIBS)
 ifdef MSMANIFEST_TOOL
@@ -857,15 +840,15 @@ EXTRA_DEPS += $(DTRACE_PROBE_OBJ)
 OBJS += $(DTRACE_PROBE_OBJ)
 endif
 
-$(filter %.$(LIB_SUFFIX),$(LIBRARY)): $(OBJS) $(LOBJS) $(SHARED_LIBRARY_LIBS_DEPS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
+$(filter %.$(LIB_SUFFIX),$(LIBRARY)): $(OBJS) $(LOBJS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(RM) $(LIBRARY)
 	$(EXPAND_AR) $(AR_FLAGS) $(OBJS) $(LOBJS) $(SHARED_LIBRARY_LIBS)
 	$(RANLIB) $@
 
-$(filter-out %.$(LIB_SUFFIX),$(LIBRARY)): $(filter %.$(LIB_SUFFIX),$(LIBRARY)) $(OBJS) $(LOBJS) $(SHARED_LIBRARY_LIBS_DEPS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
+$(filter-out %.$(LIB_SUFFIX),$(LIBRARY)): $(filter %.$(LIB_SUFFIX),$(LIBRARY)) $(OBJS) $(LOBJS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 # When we only build a library descriptor, blow out any existing library
 	$(if $(filter %.$(LIB_SUFFIX),$(LIBRARY)),,$(RM) $(REAL_LIBRARY) $(EXPORT_LIBRARY:%=%/$(REAL_LIBRARY)))
-	$(EXPAND_LIBS_GEN) $(OBJS) $(LOBJS) $(SHARED_LIBRARY_LIBS) > $@
+	$(EXPAND_LIBS_GEN) -o $@ $(OBJS) $(LOBJS) $(SHARED_LIBRARY_LIBS)
 
 ifeq ($(OS_ARCH),WINNT)
 $(IMPORT_LIBRARY): $(SHARED_LIBRARY)
@@ -910,7 +893,7 @@ endif
 # symlinks back to the originals. The symlinks are a no-op for stabs debugging,
 # so no need to conditionalize on OS version or debugging format.
 
-$(SHARED_LIBRARY): $(OBJS) $(LOBJS) $(DEF_FILE) $(RESFILE) $(SHARED_LIBRARY_LIBS_DEPS) $(LIBRARY) $(EXTRA_DEPS) $(DSO_LDOPTS_DEPS) $(GLOBAL_DEPS)
+$(SHARED_LIBRARY): $(OBJS) $(LOBJS) $(DEF_FILE) $(RESFILE) $(LIBRARY) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 ifndef INCREMENTAL_LINKER
 	$(RM) $@
 endif

config/tests/unit-expandlibs.py

@@ -36,8 +36,7 @@
 
 config = sys.modules['expandlibs_config'] = imp.new_module('expandlibs_config')
 
-from expandlibs import LibDescriptor, ExpandArgs, relativize
-from expandlibs_deps import ExpandLibsDeps, split_args
+from expandlibs import LibDescriptor, ExpandArgs, relativize, ExpandLibsDeps
 from expandlibs_gen import generate
 from expandlibs_exec import ExpandArgsMore, SectionFinder
 
@@ -213,12 +212,6 @@ def test_expandlibsdeps(self):
         args = self.arg_files + [self.tmpfile('liby', Lib('y'))]
         self.assertRelEqual(ExpandLibsDeps(args), ExpandArgs(args))
 
-class TestSplitArgs(unittest.TestCase):
-    def test_split_args(self):
-        self.assertEqual(split_args(['a', '=', 'b', 'c']), {'a': ['b', 'c']})
-        self.assertEqual(split_args(['a', '=', 'b', 'c', ',', 'd', '=', 'e', 'f', 'g', ',', 'h', '=', 'i']),
-                         {'a': ['b', 'c'], 'd': ['e', 'f', 'g'], 'h': ['i']})
-
 class TestExpandArgsMore(TestExpandInit):
     def test_makelist(self):
         '''Test grouping object files in lists'''

js/src/config/config.mk

@@ -731,9 +731,8 @@ OPTIMIZE_JARS_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/optimizeja
 
 CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
 
-EXPAND_LIBS_DEPS = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_deps.py
-EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_exec.py
-EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_gen.py
+EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_exec.py $(if $@,--depend $(MDDEPDIR)/$(basename $(@F)).pp --target $@)
+EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_gen.py $(if $@,--depend $(MDDEPDIR)/$(basename $(@F)).pp)
 EXPAND_AR = $(EXPAND_LIBS_EXEC) --extract -- $(AR)
 EXPAND_CC = $(EXPAND_LIBS_EXEC) --uselist -- $(CC)
 EXPAND_CCC = $(EXPAND_LIBS_EXEC) --uselist -- $(CCC)

js/src/config/expandlibs.py

@@ -27,9 +27,19 @@
   rules.
 '''
 from __future__ import with_statement
-import sys, os
+import sys, os, errno
 import expandlibs_config as conf
 
+def ensureParentDir(file):
+    '''Ensures the directory parent to the given file exists'''
+    dir = os.path.dirname(file)
+    if dir and not os.path.exists(dir):
+        try:
+            os.makedirs(dir)
+        except OSError, error:
+            if error.errno != errno.EEXIST:
+                raise
+
 def relativize(path):
     '''Returns a path relative to the current working directory, if it is
     shorter than the given path'''
@@ -116,5 +126,13 @@ def _expand_desc(self, arg):
             return objs
         return [arg]
 
+class ExpandLibsDeps(ExpandArgs):
+    '''Same as ExpandArgs, but also adds the library descriptor to the list'''
+    def _expand_desc(self, arg):
+        objs = super(ExpandLibsDeps, self)._expand_desc(arg)
+        if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
+            objs += [relativize(arg + conf.LIBS_DESC_SUFFIX)]
+        return objs
+
 if __name__ == '__main__':
     print " ".join(ExpandArgs(sys.argv[1:]))