Package flumotion :: Package common :: Module package
[hide private]

Source Code for Module flumotion.common.package

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_package -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """objects and functions used in dealing with packages 
 23  """ 
 24   
 25  import ihooks 
 26  import glob 
 27  import os 
 28  import sys 
 29   
 30  from twisted.python import rebuild, reflect 
 31   
 32  from flumotion.common import log, common 
 33  from flumotion.configure import configure 
 34   
 35  __version__ = "$Rev: 7927 $" 
 36   
 37   
38 -class _PatchedModuleImporter(ihooks.ModuleImporter):
39 """ 40 I am overriding ihook's ModuleImporter's import_module() method to 41 accept (and ignore) the 'level' keyword argument that appeared in 42 the built-in __import__() function in python2.5. 43 44 While no built-in modules in python2.5 seem to use that keyword 45 argument, 'encodings' module in python2.6 does and so it breaks if 46 used together with ihooks. 47 48 I make no attempt to properly support the 'level' argument - 49 ihooks didn't make it into py3k, and the only use in python2.6 50 we've seen so far, in 'encodings', serves as a performance hint 51 and it seems that can be ignored with no difference in behaviour. 52 """ 53
54 - def import_module(self, name, globals=None, locals=None, fromlist=None, 55 level=-1):
56 # all we do is drop 'level' as ihooks don't support it, anyway 57 return ihooks.ModuleImporter.import_module(self, name, globals, 58 locals, fromlist)
59 60
61 -class PackageHooks(ihooks.Hooks):
62 """ 63 I am an import Hooks object that makes sure that every package that gets 64 loaded has every necessary path in the module's __path__ list. 65 66 @type packager: L{Packager} 67 """ 68 packager = None 69
70 - def load_package(self, name, filename, file=None):
71 # this is only ever called the first time a package is imported 72 log.log('packager', 'load_package %s' % name) 73 ret = ihooks.Hooks.load_package(self, name, filename, file) 74 75 m = sys.modules[name] 76 77 packagePaths = self.packager.getPathsForPackage(name) 78 if not packagePaths: 79 return ret 80 81 # get full paths to the package 82 paths = [os.path.join(path, name.replace('.', os.sep)) 83 for path in packagePaths] 84 for path in paths: 85 if not path in m.__path__: 86 log.log('packager', 'adding path %s for package %s' % ( 87 path, name)) 88 m.__path__.append(path) 89 90 return ret
91 92
93 -class Packager(log.Loggable):
94 """ 95 I am an object through which package paths can be registered, to support 96 the partitioning of the module import namespace across bundles. 97 """ 98 99 logCategory = 'packager' 100
101 - def __init__(self):
102 self._paths = {} # key -> package path registered with that key 103 self._packages = {} # package name -> keys for that package 104 self.install()
105
106 - def install(self):
107 """ 108 Install our custom importer that uses bundled packages. 109 """ 110 self.debug('installing custom importer') 111 self._hooks = PackageHooks() 112 self._hooks.packager = self 113 if sys.version_info < (2, 6): 114 self._importer = ihooks.ModuleImporter() 115 else: 116 self.debug('python2.6 or later detected - using patched' 117 ' ModuleImporter') 118 self._importer = _PatchedModuleImporter() 119 self._importer.set_hooks(self._hooks) 120 self._importer.install()
121
122 - def getPathsForPackage(self, packageName):
123 """ 124 Return all absolute paths to the top level of a tree from which 125 (part of) the given package name can be imported. 126 """ 127 if packageName not in self._packages: 128 return None 129 130 return [self._paths[key] for key in self._packages[packageName]]
131
132 - def registerPackagePath(self, packagePath, key, prefix=configure.PACKAGE):
133 """ 134 Register a given path as a path that can be imported from. 135 Used to support partition of bundled code or import code from various 136 uninstalled location. 137 138 sys.path will also be changed to include this, and remove references 139 to older packagePath's for the same bundle. 140 141 @param packagePath: path to add under which the module namespaces live, 142 (ending in an md5sum, for flumotion purposes) 143 @type packagePath: string 144 @param key a unique id for the package being registered 145 @type key: string 146 @param prefix: prefix of the packages to be considered 147 @type prefix: string 148 """ 149 150 new = True 151 packagePath = os.path.abspath(packagePath) 152 if not os.path.exists(packagePath): 153 log.warning('bundle', 154 'registering a non-existing package path %s' % packagePath) 155 156 self.log('registering packagePath %s' % packagePath) 157 158 # check if a packagePath for this bundle was already registered 159 if key in self._paths: 160 oldPath = self._paths[key] 161 if packagePath == oldPath: 162 self.log('already registered %s for key %s' % ( 163 packagePath, key)) 164 return 165 new = False 166 167 # Find the packages in the path and sort them, 168 # the following algorithm only works if they're sorted. 169 # By sorting the list we can ensure that a parent package 170 # is always processed before one of its children 171 if not os.path.isdir(packagePath): 172 log.warning('bundle', 'package path not a dir: %s', 173 packagePath) 174 packageNames = [] 175 else: 176 packageNames = _findPackageCandidates(packagePath, prefix) 177 178 if not packageNames: 179 log.log('bundle', 180 'packagePath %s does not have candidates starting with %s' % 181 (packagePath, prefix)) 182 return 183 packageNames.sort() 184 185 self.log('package candidates %r' % packageNames) 186 187 if not new: 188 # it already existed, and now it's a different path 189 log.log('bundle', 190 'replacing old path %s with new path %s for key %s' % ( 191 oldPath, packagePath, key)) 192 193 if oldPath in sys.path: 194 log.log('bundle', 195 'removing old packagePath %s from sys.path' % oldPath) 196 sys.path.remove(oldPath) 197 198 # clear this key from our name -> key cache 199 for keys in self._packages.values(): 200 if key in keys: 201 keys.remove(key) 202 203 self._paths[key] = packagePath 204 205 # put packagePath at the top of sys.path if not in there 206 if not packagePath in sys.path: 207 self.log('adding packagePath %s to sys.path' % packagePath) 208 sys.path.insert(0, packagePath) 209 210 # update our name->keys cache 211 for name in packageNames: 212 if name not in self._packages: 213 self._packages[name] = [key] 214 else: 215 self._packages[name].insert(0, key) 216 217 self.log('packagePath %s has packageNames %r' % ( 218 packagePath, packageNames)) 219 # since we want sub-modules to be fixed up before parent packages, 220 # we reverse the list 221 packageNames.reverse() 222 223 for packageName in packageNames: 224 if packageName not in sys.modules: 225 continue 226 self.log('fixing up %s ...' % packageName) 227 228 # the package is imported, so mess with __path__ and rebuild 229 package = sys.modules.get(packageName) 230 for path in package.__path__: 231 if not new and path.startswith(oldPath): 232 self.log('%s.__path__ before remove %r' % ( 233 packageName, package.__path__)) 234 self.log('removing old %s from %s.__path__' % ( 235 path, name)) 236 package.__path__.remove(path) 237 self.log('%s.__path__ after remove %r' % ( 238 packageName, package.__path__)) 239 240 # move the new path to the top 241 # insert at front because FLU_REGISTRY_PATH paths should override 242 # base components, and because subsequent reload() should prefer 243 # the latest registered path 244 newPath = os.path.join(packagePath, 245 packageName.replace('.', os.sep)) 246 247 # if path already at position 0, everything's fine 248 # if it's in there at another place, it needs to move to front 249 # if not in there, it needs to be put in front 250 if len(package.__path__) == 0: 251 # FIXME: this seems to happen to e.g. flumotion.component.base 252 # even when it was just rebuilt and had the __path__ set 253 # can be triggered by choosing a admin_gtk depending on 254 # the base admin_gtk where the base admin_gtk changes 255 self.debug('WARN: package %s does not have __path__ values' % ( 256 packageName)) 257 elif package.__path__[0] == newPath: 258 self.log('path %s already at start of %s.__path__' % ( 259 newPath, packageName)) 260 continue 261 262 if newPath in package.__path__: 263 package.__path__.remove(newPath) 264 self.log('moving %s to front of %s.__path__' % ( 265 newPath, packageName)) 266 else: 267 self.log('inserting new %s into %s.__path__' % ( 268 newPath, packageName)) 269 package.__path__.insert(0, newPath) 270 271 # Rebuilding these packages just to get __path__ fixed in 272 # seems not necessary - but re-enable it if it breaks 273 # self.log('rebuilding package %s from paths %r' % (packageName, 274 # package.__path__)) 275 # rebuild.rebuild(package) 276 # self.log('rebuilt package %s with paths %r' % (packageName, 277 # package.__path__)) 278 self.log('fixed up %s, __path__ %s ...' % ( 279 packageName, package.__path__)) 280 281 # now rebuild all non-package modules in this packagePath if this 282 # is not a new package 283 if not new: 284 self.log('finding end module candidates') 285 if not os.path.isdir(packagePath): 286 log.warning('bundle', 'package path not a dir: %s', 287 path) 288 moduleNames = [] 289 else: 290 moduleNames = findEndModuleCandidates(packagePath, prefix) 291 self.log('end module candidates to rebuild: %r' % moduleNames) 292 for name in moduleNames: 293 if name in sys.modules: 294 # fixme: isn't sys.modules[name] sufficient? 295 self.log("rebuilding non-package module %s" % name) 296 try: 297 module = reflect.namedAny(name) 298 except AttributeError: 299 log.warning('bundle', 300 "could not reflect non-package module %s" % name) 301 continue 302 303 if hasattr(module, '__path__'): 304 self.log('rebuilding module %s with paths %r' % (name, 305 module.__path__)) 306 rebuild.rebuild(module) 307 #if paths: 308 # module.__path__ = paths 309 310 self.log('registered packagePath %s for key %s' % (packagePath, key))
311
312 - def unregister(self):
313 """ 314 Unregister all previously registered package paths, and uninstall 315 the custom importer. 316 """ 317 for path in self._paths.values(): 318 if path in sys.path: 319 self.log('removing packagePath %s from sys.path' % path) 320 sys.path.remove(path) 321 self._paths = {} 322 self._packages = {} 323 self.debug('uninstalling custom importer') 324 self._importer.uninstall()
325 326
327 -def _listDirRecursively(path):
328 """ 329 I'm similar to os.listdir, but I work recursively and only return 330 directories containing python code. 331 332 @param path: the path 333 @type path: string 334 """ 335 retval = [] 336 try: 337 files = os.listdir(path) 338 except OSError: 339 pass 340 else: 341 for f in files: 342 # this only adds directories since files are not returned 343 p = os.path.join(path, f) 344 if os.path.isdir(p) and f != '.svn': 345 retval += _listDirRecursively(p) 346 347 if glob.glob(os.path.join(path, '*.py*')): 348 retval.append(path) 349 350 return retval
351 352
353 -def _listPyFileRecursively(path):
354 """ 355 I'm similar to os.listdir, but I work recursively and only return 356 files representing python non-package modules. 357 358 @param path: the path 359 @type path: string 360 361 @rtype: list 362 @returns: list of files underneath the given path containing python code 363 """ 364 retval = [] 365 366 # get all the dirs containing python code 367 dirs = _listDirRecursively(path) 368 369 for directory in dirs: 370 pyfiles = glob.glob(os.path.join(directory, '*.py*')) 371 dontkeep = glob.glob(os.path.join(directory, '*__init__.py*')) 372 for f in dontkeep: 373 if f in pyfiles: 374 pyfiles.remove(f) 375 376 retval.extend(pyfiles) 377 378 return retval
379 380
381 -def _findPackageCandidates(path, prefix=configure.PACKAGE):
382 """ 383 I take a directory and return a list of candidate python packages 384 under that directory that start with the given prefix. 385 A package is a module containing modules; typically the directory 386 with the same name as the package contains __init__.py 387 388 @param path: the path 389 @type path: string 390 """ 391 # this function also "guesses" candidate packages when __init__ is missing 392 # so a bundle with only a subpackage is also detected 393 dirs = _listDirRecursively(os.path.join(path, prefix)) 394 395 # chop off the base path to get a list of "relative" bundlespace paths 396 bundlePaths = [x[len(path) + 1:] for x in dirs] 397 398 # remove some common candidates, like .svn subdirs, or containing - 399 bundlePaths = [path for path in bundlePaths if path.find('.svn') == -1] 400 bundlePaths = [path for path in bundlePaths if path.find('-') == -1] 401 402 # convert paths to module namespace 403 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths] 404 405 # now make sure that all parent packages for each package are listed 406 # as well 407 packages = {} 408 for name in bundlePackages: 409 packages[name] = 1 410 parts = name.split(".") 411 build = None 412 for p in parts: 413 if not build: 414 build = p 415 else: 416 build = build + "." + p 417 packages[build] = 1 418 419 bundlePackages = packages.keys() 420 421 # sort them so that depending packages are after higher-up packages 422 bundlePackages.sort() 423 424 return bundlePackages
425 426
427 -def findEndModuleCandidates(path, prefix=configure.PACKAGE):
428 """ 429 I take a directory and return a list of candidate python end modules 430 (i.e., non-package modules) for the given module prefix. 431 432 @param path: the path under which to search for end modules 433 @type path: string 434 @param prefix: module prefix to check candidates under 435 @type prefix: string 436 """ 437 pathPrefix = "/".join(prefix.split(".")) 438 files = _listPyFileRecursively(os.path.join(path, pathPrefix)) 439 440 # chop off the base path to get a list of "relative" import space paths 441 importPaths = [x[len(path) + 1:] for x in files] 442 443 # remove some common candidates, like .svn subdirs, or containing - 444 importPaths = [path for path in importPaths if path.find('.svn') == -1] 445 importPaths = [path for path in importPaths if path.find('-') == -1] 446 447 # convert paths to module namespace 448 endModules = [common.pathToModuleName(x) for x in importPaths] 449 450 # remove all not starting with prefix 451 endModules = [module for module in endModules 452 if module and module.startswith(prefix)] 453 454 # sort them so that depending packages are after higher-up packages 455 endModules.sort() 456 457 # make unique 458 res = {} 459 for b in endModules: 460 res[b] = 1 461 462 return res.keys()
463 464 # singleton factory function 465 __packager = None 466 467
468 -def getPackager():
469 """ 470 Return the (unique) packager. 471 472 @rtype: L{Packager} 473 """ 474 global __packager 475 if not __packager: 476 __packager = Packager() 477 478 return __packager
479