expandlibs_exec.py 15 KB
Newer Older
1 2 3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 5 6 7 8 9 10 11 12 13 14 15 16 17

'''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
a given command line, and executes that command line with the expanded
arguments.

With the --extract argument (useful for e.g. $(AR)), it extracts object files
from static libraries (or use those listed in library descriptors directly).

With the --uselist argument (useful for e.g. $(CC)), it replaces all object
files with a list file. This can be used to avoid limitations in the length
of a command line. The kind of list file format used depends on the
EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
or 'linkerscript' for GNU ld linker scripts.
See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
18 19 20 21

With the --symbol-order argument, followed by a file name, it will add the
relevant linker options to change the order in which the linker puts the
symbols appear in the resulting binary. Only works for ELF targets.
22
'''
23
from __future__ import with_statement
24 25
import sys
import os
26 27 28 29 30 31
from expandlibs import (
    ExpandArgs,
    relativize,
    isDynamicLib,
    isObject,
)
32 33 34 35 36
import expandlibs_config as conf
from optparse import OptionParser
import subprocess
import tempfile
import shutil
37 38
import subprocess
import re
39
from mozbuild.makeutil import Makefile
40 41 42 43 44 45 46 47 48 49

# The are the insert points for a GNU ld linker script, assuming a more
# or less "standard" default linker script. This is not a dict because
# order is important.
SECTION_INSERT_BEFORE = [
  ('.text', '.fini'),
  ('.rodata', '.rodata1'),
  ('.data.rel.ro', '.dynamic'),
  ('.data', '.data1'),
]
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74

class ExpandArgsMore(ExpandArgs):
    ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
    def __enter__(self):
        self.tmp = []
        return self
        
    def __exit__(self, type, value, tb):
        '''Automatically remove temporary files'''
        for tmp in self.tmp:
            if os.path.isdir(tmp):
                shutil.rmtree(tmp, True)
            else:
                os.remove(tmp)

    def extract(self):
        self[0:] = self._extract(self)

    def _extract(self, args):
        '''When a static library name is found, either extract its contents
        in a temporary directory or use the information found in the
        corresponding lib descriptor.
        '''
        ar_extract = conf.AR_EXTRACT.split()
        newlist = []
75 76 77 78 79 80

        def lookup(base, f):
            for root, dirs, files in os.walk(base):
                if f in files:
                    return os.path.join(root, f)

81 82 83 84
        for arg in args:
            if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
                if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
                    newlist += self._extract(self._expand_desc(arg))
85 86
                    continue
                elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
87 88
                    tmp = tempfile.mkdtemp(dir=os.curdir)
                    self.tmp.append(tmp)
89 90
                    if conf.AR == 'lib':
                        out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
91 92 93 94 95 96 97 98
                        files = out.splitlines()
                        # If lib -list returns a list full of dlls, it's an
                        # import lib.
                        if all(isDynamicLib(f) for f in files):
                            newlist += [arg]
                            continue
                        for f in files:
                            subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)
99 100
                    else:
                        subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
101
                    objs = []
102
                    basedir = os.path.dirname(arg)
103
                    for root, dirs, files in os.walk(tmp):
104 105 106 107 108 109
                        for f in files:
                            if isObject(f):
                                # If the file extracted from the library also
                                # exists in the directory containing the
                                # library, or one of its subdirectories, use
                                # that instead.
110
                                maybe_obj = lookup(os.path.join(basedir, os.path.relpath(root, tmp)), f)
111 112 113 114
                                if maybe_obj:
                                    objs.append(relativize(maybe_obj))
                                else:
                                    objs.append(relativize(os.path.join(root, f)))
115 116 117
                    newlist += sorted(objs)
                    continue
            newlist += [arg]
118 119 120 121 122 123
        return newlist

    def makelist(self):
        '''Replaces object file names with a temporary list file, using a
        list format depending on the EXPAND_LIBS_LIST_STYLE variable
        '''
124
        objs = [o for o in self if isObject(o)]
125 126 127
        if not len(objs): return
        fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
        if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
128
            content = ['INPUT("%s")\n' % obj for obj in objs]
129
            ref = tmp
130 131 132
        elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
            content = ["%s\n" % obj for obj in objs]
            ref = "-Wl,-filelist," + tmp
133
        elif conf.EXPAND_LIBS_LIST_STYLE == "list":
134
            content = ["%s\n" % obj for obj in objs]
135 136
            ref = "@" + tmp
        else:
137
            os.close(fd)
138 139 140 141 142 143 144
            os.remove(tmp)
            return
        self.tmp.append(tmp)
        f = os.fdopen(fd, "w")
        f.writelines(content)
        f.close()
        idx = self.index(objs[0])
145
        newlist = self[0:idx] + [ref] + [os.path.normpath(item) for item in self[idx:] if item not in objs]
146 147
        self[0:] = newlist

148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
    def _getFoldedSections(self):
        '''Returns a dict about folded sections.
        When section A and B are folded into section C, the dict contains:
        { 'A': 'C',
          'B': 'C',
          'C': ['A', 'B'] }'''
        if not conf.LD_PRINT_ICF_SECTIONS:
            return {}

        proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
        (stdout, stderr) = proc.communicate()
        result = {}
        # gold's --print-icf-sections output looks like the following:
        # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
        # In terms of words, chances are this will change in the future,
        # especially considering "into" is misplaced. Splitting on quotes
        # seems safer.
        for l in stderr.split('\n'):
            quoted = l.split("'")
            if len(quoted) > 5 and quoted[1] != quoted[5]:
168
                result[quoted[1]] = [quoted[5]]
169 170 171 172 173 174
                if quoted[5] in result:
                    result[quoted[5]].append(quoted[1])
                else:
                    result[quoted[5]] = [quoted[1]]
        return result

175 176 177
    def _getOrderedSections(self, ordered_symbols):
        '''Given an ordered list of symbols, returns the corresponding list
        of sections following the order.'''
178 179 180
        if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
        finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
181
        folded = self._getFoldedSections()
182 183 184
        sections = set()
        ordered_sections = []
        for symbol in ordered_symbols:
185 186 187 188 189 190 191 192 193 194 195
            symbol_sections = finder.getSections(symbol)
            all_symbol_sections = []
            for section in symbol_sections:
                if section in folded:
                    if isinstance(folded[section], str):
                        section = folded[section]
                    all_symbol_sections.append(section)
                    all_symbol_sections.extend(folded[section])
                else:
                    all_symbol_sections.append(section)
            for section in all_symbol_sections:
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
                if not section in sections:
                    ordered_sections.append(section)
                    sections.add(section)
        return ordered_sections

    def orderSymbols(self, order):
        '''Given a file containing a list of symbols, adds the appropriate
        argument to make the linker put the symbols in that order.'''
        with open(order) as file:
            sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
        split_sections = {}
        linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
        for s in sections:
            for linked_section in linked_sections:
                if s.startswith(linked_section):
                    if linked_section in split_sections:
                        split_sections[linked_section].append(s)
                    else:
                        split_sections[linked_section] = [s]
                    break
        content = []
        # Order is important
        linked_sections = [s for s in linked_sections if s in split_sections]

        if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
221
            option = '-Wl,--section-ordering-file,%s'
222 223 224
            content = sections
            for linked_section in linked_sections:
                content.extend(split_sections[linked_section])
225
                content.append('%s.*' % linked_section)
226 227 228
                content.append(linked_section)

        elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
229
            option = '-Wl,-T,%s'
230 231
            section_insert_before = dict(SECTION_INSERT_BEFORE)
            for linked_section in linked_sections:
232 233 234 235 236 237
                content.append('SECTIONS {')
                content.append('  %s : {' % linked_section)
                content.extend('    *(%s)' % s for s in split_sections[linked_section])
                content.append('  }')
                content.append('}')
                content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
238
        else:
239
            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
240 241 242 243 244 245

        fd, tmp = tempfile.mkstemp(dir=os.curdir)
        f = os.fdopen(fd, "w")
        f.write('\n'.join(content)+'\n')
        f.close()
        self.tmp.append(tmp)
246
        self.append(option % tmp)
247 248 249 250 251 252 253 254

class SectionFinder(object):
    '''Instances of this class allow to map symbol names to sections in
    object files.'''

    def __init__(self, objs):
        '''Creates an instance, given a list of object files.'''
        if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
255
            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
256 257 258
        self.mapping = {}
        for obj in objs:
            if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
259
                raise Exception('%s is not an object nor a static library' % obj)
260 261 262
            for symbol, section in SectionFinder._getSymbols(obj):
                sym = SectionFinder._normalize(symbol)
                if sym in self.mapping:
263
                    if not section in self.mapping[sym]:
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
                        self.mapping[sym].append(section)
                else:
                    self.mapping[sym] = [section]

    def getSections(self, symbol):
        '''Given a symbol, returns a list of sections containing it or the
        corresponding thunks. When the given symbol is a thunk, returns the
        list of sections containing its corresponding normal symbol and the
        other thunks for that symbol.'''
        sym = SectionFinder._normalize(symbol)
        if sym in self.mapping:
            return self.mapping[sym]
        return []

    @staticmethod
    def _normalize(symbol):
        '''For normal symbols, return the given symbol. For thunks, return
        the corresponding normal symbol.'''
        if re.match('^_ZThn[0-9]+_', symbol):
            return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
        return symbol

    @staticmethod
    def _getSymbols(obj):
        '''Returns a list of (symbol, section) contained in the given object
        file.'''
        proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
        (stdout, stderr) = proc.communicate()
        syms = []
        for line in stdout.splitlines():
            # Each line has the following format:
            # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
            tmp = line.split(' ',1)
            # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
            # We only need to consider cases where "<section>\t<length> <symbol>" is present,
            # and where the [FfO] flag is either F (function) or O (object).
            if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
                tmp = tmp[1][8:].split()
                # That gives us ["<section>","<length>", "<symbol>"]
                syms.append((tmp[-1], tmp[0]))
        return syms

306
def print_command(out, args):
307
    print >>out, "Executing: " + " ".join(args)
308
    for tmp in [f for f in args.tmp if os.path.isfile(f)]:
309
        print >>out, tmp + ":"
310
        with open(tmp) as file:
311
            print >>out, "".join(["    " + l for l in file.readlines()])
312 313
    out.flush()

314
def main(args, proc_callback=None):
315 316 317 318 319
    parser = OptionParser()
    parser.add_option("--extract", action="store_true", dest="extract",
        help="when a library has no descriptor file, extract it first, when possible")
    parser.add_option("--uselist", action="store_true", dest="uselist",
        help="use a list file for objects when executing a command")
320 321
    parser.add_option("--verbose", action="store_true", dest="verbose",
        help="display executed command and temporary files content")
322 323
    parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
        help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
324

325
    (options, args) = parser.parse_args(args)
326 327

    with ExpandArgsMore(args) as args:
328
        if options.extract:
329
            args.extract()
330 331
        if options.symbol_order:
            args.orderSymbols(options.symbol_order)
332 333 334
        if options.uselist:
            args.makelist()

335
        if options.verbose:
336
            print_command(sys.stderr, args)
337 338
        try:
            proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
339 340
            if proc_callback:
                proc_callback(proc)
341 342 343
        except Exception, e:
            print >>sys.stderr, 'error: Launching', args, ':', e
            raise e
344 345 346 347 348 349
        (stdout, stderr) = proc.communicate()
        if proc.returncode and not options.verbose:
            print_command(sys.stderr, args)
        sys.stderr.write(stdout)
        sys.stderr.flush()
        if proc.returncode:
350 351
            return proc.returncode
        return 0
352 353

if __name__ == '__main__':
354
    exit(main(sys.argv[1:]))