Bug 603370 - Add an option to expandlibs-exec to allow to reorder the…

… symbols when linking. r=khuey

aclocal.m4

@@ -18,6 +18,7 @@ builtin(include, build/autoconf/lto.m4)dnl
 builtin(include, build/autoconf/gcc-pr49911.m4)dnl
 builtin(include, build/autoconf/frameptr.m4)dnl
 builtin(include, build/autoconf/compiler-opts.m4)dnl
+builtin(include, build/autoconf/expandlibs.m4)dnl
 
 MOZ_PROG_CHECKMSYS()
 

build/autoconf/expandlibs.m4

@@ -0,0 +1,56 @@
+AC_DEFUN([MOZ_EXPAND_LIBS],
+[
+dnl ========================================================
+dnl =
+dnl = Check what kind of list files are supported by the
+dnl = linker
+dnl =
+dnl ========================================================
+
+AC_CACHE_CHECK(what kind of list files are supported by the linker,
+    EXPAND_LIBS_LIST_STYLE,
+    [echo "int main() {return 0;}" > conftest.${ac_ext}
+     if AC_TRY_COMMAND(${CC-cc} -o conftest.${OBJ_SUFFIX} -c $CFLAGS $CPPFLAGS conftest.${ac_ext} 1>&5) && test -s conftest.${OBJ_SUFFIX}; then
+         echo "INPUT(conftest.${OBJ_SUFFIX})" > conftest.list
+         if AC_TRY_COMMAND(${CC-cc} -o conftest${ac_exeext} $LDFLAGS conftest.list $LIBS 1>&5) && test -s conftest${ac_exeext}; then
+             EXPAND_LIBS_LIST_STYLE=linkerscript
+         else
+             echo "conftest.${OBJ_SUFFIX}" > conftest.list
+             if AC_TRY_COMMAND(${CC-cc} -o conftest${ac_exeext} $LDFLAGS @conftest.list $LIBS 1>&5) && test -s conftest${ac_exeext}; then
+                 EXPAND_LIBS_LIST_STYLE=list
+             else
+                 EXPAND_LIBS_LIST_STYLE=none
+             fi
+         fi
+     else
+         dnl We really don't expect to get here, but just in case
+         AC_ERROR([couldn't compile a simple C file])
+     fi
+     rm -rf conftest*])
+
+LIBS_DESC_SUFFIX=desc
+AC_SUBST(LIBS_DESC_SUFFIX)
+AC_SUBST(EXPAND_LIBS_LIST_STYLE)
+
+if test "$GCC_USE_GNU_LD"; then
+    AC_CACHE_CHECK(what kind of ordering can be done with the linker,
+        EXPAND_LIBS_ORDER_STYLE,
+        [> conftest.order
+         _SAVE_LDFLAGS="$LDFLAGS"
+         LDFLAGS="${LDFLAGS} -Wl,--section-ordering-file,conftest.order"
+         AC_TRY_LINK([], [],
+             EXPAND_LIBS_ORDER_STYLE=section-ordering-file,
+             EXPAND_LIBS_ORDER_STYLE=)
+         LDFLAGS="$_SAVE_LDFLAGS"
+         if test -z "$EXPAND_LIBS_ORDER_STYLE"; then
+             if AC_TRY_COMMAND(${CC-cc} ${DSO_LDOPTS} ${LDFLAGS} -o ${DLL_PREFIX}conftest${DLL_SUFFIX} -Wl,--verbose 2> /dev/null | sed -n '/^===/,/^===/p' | grep '\.text'); then
+                 EXPAND_LIBS_ORDER_STYLE=linkerscript
+             else
+                 EXPAND_LIBS_ORDER_STYLE=none
+             fi
+             rm -f ${DLL_PREFIX}conftest${DLL_SUFFIX}
+         fi])
+fi
+AC_SUBST(EXPAND_LIBS_ORDER_STYLE)
+
+])

config/config.mk

@@ -790,7 +790,11 @@ EXPAND_AR = $(EXPAND_LIBS_EXEC) --extract -- $(AR)
 EXPAND_CC = $(EXPAND_LIBS_EXEC) --uselist -- $(CC)
 EXPAND_CCC = $(EXPAND_LIBS_EXEC) --uselist -- $(CCC)
 EXPAND_LD = $(EXPAND_LIBS_EXEC) --uselist -- $(LD)
-EXPAND_MKSHLIB = $(EXPAND_LIBS_EXEC) --uselist -- $(MKSHLIB)
+EXPAND_MKSHLIB_ARGS = --uselist
+ifdef SYMBOL_ORDER
+EXPAND_MKSHLIB_ARGS += --symbol-order $(SYMBOL_ORDER)
+endif
+EXPAND_MKSHLIB = $(EXPAND_LIBS_EXEC) $(EXPAND_MKSHLIB_ARGS) -- $(MKSHLIB)
 
 ifdef STDCXX_COMPAT
 ifneq ($(OS_ARCH),Darwin)

config/expandlibs_config.py.in

@@ -54,3 +54,4 @@ DLL_SUFFIX = normalize_suffix("@DLL_SUFFIX@")
 IMPORT_LIB_SUFFIX = normalize_suffix("@IMPORT_LIB_SUFFIX@")
 LIBS_DESC_SUFFIX = normalize_suffix("@LIBS_DESC_SUFFIX@")
 EXPAND_LIBS_LIST_STYLE = "@EXPAND_LIBS_LIST_STYLE@"
+EXPAND_LIBS_ORDER_STYLE = "@EXPAND_LIBS_ORDER_STYLE@"

config/expandlibs_exec.py

@@ -48,6 +48,10 @@
 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
 or 'linkerscript' for GNU ld linker scripts.
 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
+
+With the --symbol-order argument, followed by a file name, it will add the
+relevant linker options to change the order in which the linker puts the
+symbols appear in the resulting binary. Only works for ELF targets.
 '''
 from __future__ import with_statement
 import sys
@@ -58,6 +62,18 @@
 import subprocess
 import tempfile
 import shutil
+import subprocess
+import re
+
+# The are the insert points for a GNU ld linker script, assuming a more
+# or less "standard" default linker script. This is not a dict because
+# order is important.
+SECTION_INSERT_BEFORE = [
+  ('.text', '.fini'),
+  ('.rodata', '.rodata1'),
+  ('.data.rel.ro', '.dynamic'),
+  ('.data', '.data1'),
+]
 
 class ExpandArgsMore(ExpandArgs):
     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
@@ -125,6 +141,126 @@ def makelist(self):
         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
         self[0:] = newlist
 
+    def _getOrderedSections(self, ordered_symbols):
+        '''Given an ordered list of symbols, returns the corresponding list
+        of sections following the order.'''
+        if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
+            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
+        finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
+        sections = set()
+        ordered_sections = []
+        for symbol in ordered_symbols:
+            for section in finder.getSections(symbol):
+                if not section in sections:
+                    ordered_sections.append(section)
+                    sections.add(section)
+        return ordered_sections
+
+    def orderSymbols(self, order):
+        '''Given a file containing a list of symbols, adds the appropriate
+        argument to make the linker put the symbols in that order.'''
+        with open(order) as file:
+            sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
+        split_sections = {}
+        linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
+        for s in sections:
+            for linked_section in linked_sections:
+                if s.startswith(linked_section):
+                    if linked_section in split_sections:
+                        split_sections[linked_section].append(s)
+                    else:
+                        split_sections[linked_section] = [s]
+                    break
+        content = []
+        # Order is important
+        linked_sections = [s for s in linked_sections if s in split_sections]
+
+        if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
+            option = '-Wl,--section-ordering-file,%s'
+            content = sections
+            for linked_section in linked_sections:
+                content.extend(split_sections[linked_section])
+                content.append('%s.*' % linked_section)
+                content.append(linked_section)
+
+        elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
+            option = '-Wl,-T,%s'
+            section_insert_before = dict(SECTION_INSERT_BEFORE)
+            for linked_section in linked_sections:
+                content.append('SECTIONS {')
+                content.append('  %s : {' % linked_section)
+                content.extend('    *(%s)' % s for s in split_sections[linked_section])
+                content.append('  }')
+                content.append('}')
+                content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
+        else:
+            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
+
+        fd, tmp = tempfile.mkstemp(dir=os.curdir)
+        f = os.fdopen(fd, "w")
+        f.write('\n'.join(content)+'\n')
+        f.close()
+        self.tmp.append(tmp)
+        self.append(option % tmp)
+
+class SectionFinder(object):
+    '''Instances of this class allow to map symbol names to sections in
+    object files.'''
+
+    def __init__(self, objs):
+        '''Creates an instance, given a list of object files.'''
+        if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
+            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
+        self.mapping = {}
+        for obj in objs:
+            if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
+                raise Exception('%s is not an object nor a static library' % obj)
+            for symbol, section in SectionFinder._getSymbols(obj):
+                sym = SectionFinder._normalize(symbol)
+                if sym in self.mapping:
+                    if not section in self.mapping[sym]:
+                        self.mapping[sym].append(section)
+                else:
+                    self.mapping[sym] = [section]
+
+    def getSections(self, symbol):
+        '''Given a symbol, returns a list of sections containing it or the
+        corresponding thunks. When the given symbol is a thunk, returns the
+        list of sections containing its corresponding normal symbol and the
+        other thunks for that symbol.'''
+        sym = SectionFinder._normalize(symbol)
+        if sym in self.mapping:
+            return self.mapping[sym]
+        return []
+
+    @staticmethod
+    def _normalize(symbol):
+        '''For normal symbols, return the given symbol. For thunks, return
+        the corresponding normal symbol.'''
+        if re.match('^_ZThn[0-9]+_', symbol):
+            return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
+        return symbol
+
+    @staticmethod
+    def _getSymbols(obj):
+        '''Returns a list of (symbol, section) contained in the given object
+        file.'''
+        proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
+        (stdout, stderr) = proc.communicate()
+        syms = []
+        for line in stdout.splitlines():
+            # Each line has the following format:
+            # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
+            tmp = line.split(' ',1)
+            # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
+            # We only need to consider cases where "<section>\t<length> <symbol>" is present,
+            # and where the [FfO] flag is either F (function) or O (object).
+            if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
+                tmp = tmp[1][8:].split()
+                # That gives us ["<section>","<length>", "<symbol>"]
+                syms.append((tmp[-1], tmp[0]))
+        return syms
+
 def main():
     parser = OptionParser()
     parser.add_option("--extract", action="store_true", dest="extract",
@@ -133,12 +269,16 @@ def main():
         help="use a list file for objects when executing a command")
     parser.add_option("--verbose", action="store_true", dest="verbose",
         help="display executed command and temporary files content")
+    parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
+        help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
 
     (options, args) = parser.parse_args()
 
     with ExpandArgsMore(args) as args:
         if options.extract:
             args.extract()
+        if options.symbol_order:
+            args.orderSymbols(options.symbol_order)
         if options.uselist:
             args.makelist()
 

config/tests/unit-expandlibs.py

@@ -38,7 +38,7 @@
 
 from expandlibs import LibDescriptor, ExpandArgs, relativize
 from expandlibs_gen import generate
-from expandlibs_exec import ExpandArgsMore
+from expandlibs_exec import ExpandArgsMore, SectionFinder
 
 def Lib(name):
     return config.LIB_PREFIX + name + config.LIB_SUFFIX
@@ -267,5 +267,68 @@ def call(args, **kargs):
         # Restore subprocess.call
         subprocess.call = subprocess_call
 
+class FakeProcess(object):
+    def __init__(self, out):
+        self.out = out
+
+    def communicate(self):
+        return (self.out, '')
+
+OBJDUMPS = {
+'foo.o': '''
+00000000 g     F .text\t00000001 foo
+00000000 g     F .text._Z6foobarv\t00000001 _Z6foobarv
+00000000 g     F .text.hello\t00000001 hello
+00000000 g     F .text._ZThn4_6foobarv\t00000001 _ZThn4_6foobarv
+''',
+'bar.o': '''
+00000000 g     F .text.hi\t00000001 hi
+00000000 g     F .text.hot._Z6barbazv\t00000001 .hidden _Z6barbazv
+''',
+}
+
+class ObjdumpSubprocessPopen(object):
+    def __init__(self, test):
+        self.test = test
+
+    def __call__(self, args, stdout = None, stderr = None):
+        self.test.assertEqual(stdout, subprocess.PIPE)
+        self.test.assertEqual(stderr, subprocess.PIPE)
+        self.test.assertEqual(args[0], 'objdump')
+        self.test.assertEqual(args[1], '-t')
+        self.test.assertTrue(args[2] in OBJDUMPS)
+
+        return FakeProcess(OBJDUMPS[args[2]])
+
+class TestSectionFinder(unittest.TestCase):
+    def test_getSections(self):
+        '''Test SectionFinder'''
+        # Divert subprocess.Popen
+        subprocess_popen = subprocess.Popen
+        subprocess.Popen = ObjdumpSubprocessPopen(self)
+        config.EXPAND_LIBS_ORDER_STYLE = 'linkerscript'
+        config.OBJ_SUFFIX = '.o'
+        config.LIB_SUFFIX = '.a'
+        finder = SectionFinder(['foo.o', 'bar.o'])
+        self.assertEqual(finder.getSections('foobar'), [])
+        self.assertEqual(finder.getSections('_Z6barbazv'), ['.text.hot._Z6barbazv'])
+        self.assertEqual(finder.getSections('_Z6foobarv'), ['.text._Z6foobarv', '.text._ZThn4_6foobarv'])
+        self.assertEqual(finder.getSections('_ZThn4_6foobarv'), ['.text._Z6foobarv', '.text._ZThn4_6foobarv'])
+        subprocess.Popen = subprocess_popen
+
+class TestSymbolOrder(unittest.TestCase):
+    def test_getOrderedSections(self):
+        '''Test ExpandMoreArgs' _getOrderedSections'''
+        # Divert subprocess.Popen
+        subprocess_popen = subprocess.Popen
+        subprocess.Popen = ObjdumpSubprocessPopen(self)
+        config.EXPAND_LIBS_ORDER_STYLE = 'linkerscript'
+        config.OBJ_SUFFIX = '.o'
+        config.LIB_SUFFIX = '.a'
+        args = ExpandArgsMore(['foo', '-bar', 'bar.o', 'foo.o'])
+        self.assertEqual(args._getOrderedSections(['_Z6foobarv', '_Z6barbazv']), ['.text._Z6foobarv', '.text._ZThn4_6foobarv', '.text.hot._Z6barbazv'])
+        self.assertEqual(args._getOrderedSections(['_ZThn4_6foobarv', '_Z6barbazv']), ['.text._Z6foobarv', '.text._ZThn4_6foobarv', '.text.hot._Z6barbazv'])
+        subprocess.Popen = subprocess_popen
+
 if __name__ == '__main__':
     unittest.main(testRunner=MozTestRunner())

configure.in

@@ -8000,37 +8000,7 @@ fi # ! SKIP_COMPILER_CHECKS
 AC_DEFINE(CPP_THROW_NEW, [throw()])
 AC_LANG_C
 
-dnl ========================================================
-dnl =
-dnl = Check what kind of list files are supported by the
-dnl = linker
-dnl =
-dnl ========================================================
-
-AC_CACHE_CHECK(what kind of list files are supported by the linker,
-    EXPAND_LIBS_LIST_STYLE,
-    [echo "int main() {return 0;}" > conftest.${ac_ext}
-     if AC_TRY_COMMAND(${CC-cc} -o conftest.${OBJ_SUFFIX} -c $CFLAGS $CPPFLAGS conftest.${ac_ext} 1>&2) && test -s conftest.${OBJ_SUFFIX}; then
-         echo "INPUT(conftest.${OBJ_SUFFIX})" > conftest.list
-         if AC_TRY_COMMAND(${CC-cc} -o conftest${ac_exeext} $LDFLAGS conftest.list $LIBS 1>&2) && test -s conftest${ac_exeext}; then
-             EXPAND_LIBS_LIST_STYLE=linkerscript
-         else
-             echo "conftest.${OBJ_SUFFIX}" > conftest.list
-             if AC_TRY_COMMAND(${CC-cc} -o conftest${ac_exeext} $LDFLAGS @conftest.list $LIBS 1>&2) && test -s conftest${ac_exeext}; then
-                 EXPAND_LIBS_LIST_STYLE=list
-             else
-                 EXPAND_LIBS_LIST_STYLE=none
-             fi
-         fi
-     else
-         dnl We really don't expect to get here, but just in case
-         AC_ERROR([couldn't compile a simple C file])
-     fi
-     rm -rf conftest*])
-
-LIBS_DESC_SUFFIX=desc
-AC_SUBST(LIBS_DESC_SUFFIX)
-AC_SUBST(EXPAND_LIBS_LIST_STYLE)
+MOZ_EXPAND_LIBS
 
 dnl ========================================================
 dnl =

js/src/aclocal.m4

@@ -15,5 +15,6 @@ builtin(include, build/autoconf/lto.m4)dnl
 builtin(include, build/autoconf/gcc-pr49911.m4)dnl
 builtin(include, build/autoconf/frameptr.m4)dnl
 builtin(include, build/autoconf/compiler-opts.m4)dnl
+builtin(include, build/autoconf/expandlibs.m4)dnl
 
 MOZ_PROG_CHECKMSYS()