Bug 924331 - Move config/utils.py into mozbuild; r=mshal

config/JarMaker.py

@@ -17,7 +17,11 @@
 from MozZipFile import ZipFile
 from cStringIO import StringIO
 
-from utils import pushback_iter, lockFile
+from mozbuild.util import (
+    lock_file,
+    PushbackIter,
+)
+
 from Preprocessor import Preprocessor
 from mozbuild.action.buildlist import addEntriesToListFile
 if sys.platform == "win32":
@@ -172,7 +176,7 @@ def updateManifest(self, manifestPath, chromebasepath, register):
     '''updateManifest replaces the % in the chrome registration entries
     with the given chrome base path, and updates the given manifest file.
     '''
-    lock = lockFile(manifestPath + '.lck')
+    lock = lock_file(manifestPath + '.lck')
     try:
       myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
                                      register.iterkeys()))
@@ -213,7 +217,7 @@ def makeJar(self, infile, jardir):
     pp = self.pp.clone()
     pp.out = StringIO()
     pp.do_include(infile)
-    lines = pushback_iter(pp.out.getvalue().splitlines())
+    lines = PushbackIter(pp.out.getvalue().splitlines())
     try:
       while True:
         l = lines.next()
@@ -250,8 +254,8 @@ def processJarSection(self, jarfile, lines, jardir):
     '''Internal method called by makeJar to actually process a section
     of a jar.mn file.
 
-    jarfile is the basename of the jarfile or the directory name for 
-    flat output, lines is a pushback_iterator of the lines of jar.mn,
+    jarfile is the basename of the jarfile or the directory name for
+    flat output, lines is a PushbackIter of the lines of jar.mn,
     the remaining options are carried over from makeJar.
     '''
 

config/MozZipFile.py

@@ -2,12 +2,12 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import zipfile
-import time
-import binascii, struct
-import zlib
 import os
-from utils import lockFile
+import time
+import zipfile
+
+from mozbuild.util import lock_file
+
 
 class ZipFile(zipfile.ZipFile):
   """ Class with methods to open, read, write, close, list zip files.
@@ -19,7 +19,7 @@ def __init__(self, file, mode="r", compression=zipfile.ZIP_STORED,
                lock = False):
     if lock:
       assert isinstance(file, basestring)
-      self.lockfile = lockFile(file + '.lck')
+      self.lockfile = lock_file(file + '.lck')
     else:
       self.lockfile = None
 

python/mozbuild/mozbuild/action/buildlist.py

@@ -11,13 +11,14 @@
 
 import sys
 import os
-from utils import lockFile
+
+from mozbuild.util import lock_file
 
 def addEntriesToListFile(listFile, entries):
   """Given a file |listFile| containing one entry per line,
   add each entry in |entries| to the file, unless it is already
   present."""
-  lock = lockFile(listFile + ".lck")
+  lock = lock_file(listFile + ".lck")
   try:
     if os.path.exists(listFile):
       f = open(listFile)

python/mozbuild/mozbuild/util.py

@@ -11,10 +11,13 @@
 import errno
 import hashlib
 import os
+import stat
 import sys
+import time
 
 from StringIO import StringIO
 
+
 if sys.version_info[0] == 3:
     str_type = str
 else:
@@ -370,3 +373,120 @@ def _check_list(self, value):
             if not isinstance(v, str_type):
                 raise ValueError(
                     'Expected a list of strings, not an element of %s' % type(v))
+
+
+class LockFile(object):
+    """LockFile is used by the lock_file method to hold the lock.
+
+    This object should not be used directly, but only through
+    the lock_file method below.
+    """
+
+    def __init__(self, lockfile):
+        self.lockfile = lockfile
+
+    def __del__(self):
+        while True:
+            try:
+                os.remove(self.lockfile)
+                break
+            except OSError as e:
+                if e.errno == errno.EACCES:
+                    # Another process probably has the file open, we'll retry.
+                    # Just a short sleep since we want to drop the lock ASAP
+                    # (but we need to let some other process close the file
+                    # first).
+                    time.sleep(0.1)
+            else:
+                # Re-raise unknown errors
+                raise
+
+
+def lock_file(lockfile, max_wait = 600):
+    """Create and hold a lockfile of the given name, with the given timeout.
+
+    To release the lock, delete the returned object.
+    """
+
+    # FUTURE This function and object could be written as a context manager.
+
+    while True:
+        try:
+            fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT)
+            # We created the lockfile, so we're the owner
+            break
+        except OSError as e:
+            if (e.errno == errno.EEXIST or
+                (sys.platform == "win32" and e.errno == errno.EACCES)):
+                pass
+            else:
+                # Should not occur
+                raise
+
+        try:
+            # The lock file exists, try to stat it to get its age
+            # and read its contents to report the owner PID
+            f = open(lockfile, 'r')
+            s = os.stat(lockfile)
+        except EnvironmentError as e:
+            if e.errno == errno.ENOENT or e.errno == errno.EACCES:
+            # We didn't create the lockfile, so it did exist, but it's
+            # gone now. Just try again
+                continue
+
+            raise Exception('{0} exists but stat() failed: {1}'.format(
+                lockfile, e.strerror))
+
+        # We didn't create the lockfile and it's still there, check
+        # its age
+        now = int(time.time())
+        if now - s[stat.ST_MTIME] > max_wait:
+            pid = f.readline().rstrip()
+            raise Exception('{0} has been locked for more than '
+                '{1} seconds (PID {2})'.format(lockfile, max_wait, pid))
+
+        # It's not been locked too long, wait a while and retry
+        f.close()
+        time.sleep(1)
+
+    # if we get here. we have the lockfile. Convert the os.open file
+    # descriptor into a Python file object and record our PID in it
+    f = os.fdopen(fd, 'w')
+    f.write('{0}\n'.format(os.getpid()))
+    f.close()
+
+    return LockFile(lockfile)
+
+
+class PushbackIter(object):
+    '''Utility iterator that can deal with pushed back elements.
+
+    This behaves like a regular iterable, just that you can call
+    iter.pushback(item) to get the given item as next item in the
+    iteration.
+    '''
+    def __init__(self, iterable):
+        self.it = iter(iterable)
+        self.pushed_back = []
+
+    def __iter__(self):
+        return self
+
+    def __nonzero__(self):
+        if self.pushed_back:
+            return True
+
+        try:
+            self.pushed_back.insert(0, self.it.next())
+        except StopIteration:
+            return False
+        else:
+            return True
+
+    def next(self):
+        if self.pushed_back:
+            return self.pushed_back.pop()
+        return self.it.next()
+
+    def pushback(self, item):
+        self.pushed_back.append(item)