Package flumotion :: Package common :: Module bundle
[hide private]

Source Code for Module flumotion.common.bundle

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_bundle -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007,2008 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  bundles of files used to implement caching over the network 
 24  """ 
 25   
 26  import StringIO 
 27  import errno 
 28  import os 
 29  import sys 
 30  import tempfile 
 31  import zipfile 
 32   
 33  from flumotion.common import errors, dag, python 
 34  from flumotion.common.python import makedirs 
 35   
 36  __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket'] 
 37  __version__ = "$Rev$" 
 38   
 39   
40 -def rename(source, dest):
41 return os.rename(source, dest)
42 43
44 -def _win32Rename(source, dest):
45 # rename a source to dest. 46 # ignores the destination if it already exists 47 # removes source if destination already exists 48 try: 49 return os.rename(source, dest) 50 except WindowsError, e: 51 import winerror 52 if e.errno == winerror.ERROR_ALREADY_EXISTS: 53 os.unlink(source)
54 55 56 if sys.platform == 'win32': 57 rename = _win32Rename 58 59
60 -class BundledFile:
61 """ 62 I represent one file as managed by a bundler. 63 """ 64
65 - def __init__(self, source, destination):
66 self.source = source 67 self.destination = destination 68 self._last_md5sum = None 69 self._last_timestamp = None 70 self.zipped = False
71
72 - def md5sum(self):
73 """ 74 Calculate the md5sum of the given file. 75 76 @returns: the md5 sum a 32 character string of hex characters. 77 """ 78 data = open(self.source, "r").read() 79 return python.md5(data).hexdigest()
80
81 - def timestamp(self):
82 """ 83 @returns: the last modified timestamp for the file. 84 """ 85 return os.path.getmtime(self.source)
86
87 - def hasChanged(self):
88 """ 89 Check if the file has changed since it was last checked. 90 91 @rtype: boolean 92 """ 93 94 # if it wasn't zipped yet, it needs zipping, so we pretend it 95 # was changed 96 # FIXME: move this out here 97 if not self.zipped: 98 return True 99 100 try: 101 timestamp = self.timestamp() 102 except OSError: 103 return True 104 # if file still has an old timestamp, it hasn't changed 105 # FIXME: looks bogus, shouldn't this check be != instead of <= ? 106 if self._last_timestamp and timestamp <= self._last_timestamp: 107 return False 108 self._last_timestamp = timestamp 109 110 # if the md5sum has changed, it has changed 111 md5sum = self.md5sum() 112 if self._last_md5sum != md5sum: 113 self._last_md5sum = md5sum 114 return True 115 116 return False
117
118 - def pack(self, zip):
119 self._last_timestamp = self.timestamp() 120 self._last_md5sum = self.md5sum() 121 zip.write(self.source, self.destination) 122 self.zipped = True
123 124
125 -class Bundle:
126 """ 127 I am a bundle of files, represented by a zip file and md5sum. 128 """ 129
130 - def __init__(self, name):
131 self.zip = None 132 self.md5sum = None 133 self.name = name
134
135 - def setZip(self, zip):
136 """ 137 Set the bundle to the given data representation of the zip file. 138 """ 139 self.zip = zip 140 self.md5sum = python.md5(self.zip).hexdigest()
141
142 - def getZip(self):
143 """ 144 Get the bundle's zip data. 145 """ 146 return self.zip
147 148
149 -class Unbundler:
150 """ 151 I unbundle bundles by unpacking them in the given directory 152 under directories with the bundle's md5sum. 153 """ 154
155 - def __init__(self, directory):
156 self._undir = directory
157
158 - def unbundlePathByInfo(self, name, md5sum):
159 """ 160 Return the full path where a bundle with the given name and md5sum 161 would be unbundled to. 162 """ 163 return os.path.join(self._undir, name, md5sum)
164
165 - def unbundlePath(self, bundle):
166 """ 167 Return the full path where this bundle will/would be unbundled to. 168 """ 169 return self.unbundlePathByInfo(bundle.name, bundle.md5sum)
170
171 - def unbundle(self, bundle):
172 """ 173 Unbundle the given bundle. 174 175 @type bundle: L{flumotion.common.bundle.Bundle} 176 177 @rtype: string 178 @returns: the full path to the directory where it was unpacked 179 """ 180 directory = self.unbundlePath(bundle) 181 182 filelike = StringIO.StringIO(bundle.getZip()) 183 zipFile = zipfile.ZipFile(filelike, "r") 184 zipFile.testzip() 185 186 filepaths = zipFile.namelist() 187 for filepath in filepaths: 188 path = os.path.join(directory, filepath) 189 parent = os.path.split(path)[0] 190 try: 191 makedirs(parent) 192 except OSError, err: 193 # Reraise error unless if it's an already existing 194 if err.errno != errno.EEXIST or not os.path.isdir(parent): 195 raise 196 data = zipFile.read(filepath) 197 198 # atomically write to path, see #373 199 fd, tempname = tempfile.mkstemp(dir=parent) 200 handle = os.fdopen(fd, 'wb') 201 handle.write(data) 202 handle.close() 203 rename(tempname, path) 204 return directory
205 206
207 -class Bundler:
208 """ 209 I bundle files into a bundle so they can be cached remotely easily. 210 """ 211
212 - def __init__(self, name):
213 """ 214 Create a new bundle. 215 """ 216 self._bundledFiles = {} # dictionary of BundledFile's indexed on path 217 self.name = name 218 self._bundle = Bundle(name)
219
220 - def add(self, source, destination = None):
221 """ 222 Add files to the bundle. 223 224 @param source: the path to the file to add to the bundle. 225 @param destination: a relative path to store this file in the bundle. 226 If unspecified, this will be stored in the top level. 227 228 @returns: the path the file got stored as 229 """ 230 if destination == None: 231 destination = os.path.split(source)[1] 232 self._bundledFiles[source] = BundledFile(source, destination) 233 return destination
234
235 - def bundle(self):
236 """ 237 Bundle the files registered with the bundler. 238 239 @rtype: L{flumotion.common.bundle.Bundle} 240 """ 241 # rescan files registered in the bundle, and check if we need to 242 # rebuild the internal zip 243 if not self._bundle.getZip(): 244 self._bundle.setZip(self._buildzip()) 245 return self._bundle 246 247 update = False 248 for bundledFile in self._bundledFiles.values(): 249 if bundledFile.hasChanged(): 250 update = True 251 break 252 253 if update: 254 self._bundle.setZip(self._buildzip()) 255 256 return self._bundle
257 258 # build the zip file containing the files registered in the bundle 259 # and return the zip file data 260
261 - def _buildzip(self):
262 filelike = StringIO.StringIO() 263 zipFile = zipfile.ZipFile(filelike, "w") 264 for bundledFile in self._bundledFiles.values(): 265 bundledFile.pack(zipFile) 266 zipFile.close() 267 data = filelike.getvalue() 268 filelike.close() 269 return data
270 271
272 -class BundlerBasket:
273 """ 274 I manage bundlers that are registered through me. 275 """ 276
277 - def __init__(self, mtime=None):
278 """ 279 Create a new bundler basket. 280 """ 281 self._bundlers = {} # bundler name -> bundle 282 283 self._files = {} # filename -> bundle name 284 self._imports = {} # import statements -> bundle name 285 286 self._graph = dag.DAG() 287 288 self._mtime = mtime # Registry modifcation time when the basket was
289 # created 290
291 - def isUptodate(self, mtime):
292 return self._mtime >= mtime
293
294 - def add(self, bundleName, source, destination=None):
295 """ 296 Add files to the bundler basket for the given bundle. 297 298 @param bundleName: the name of the bundle this file is a part of 299 @param source: the path to the file to add to the bundle 300 @param destination: a relative path to store this file in the bundle. 301 If unspecified, this will be stored in the top level 302 """ 303 # get the bundler and create it if need be 304 if not bundleName in self._bundlers: 305 bundler = Bundler(bundleName) 306 self._bundlers[bundleName] = bundler 307 else: 308 bundler = self._bundlers[bundleName] 309 310 # add the file to the bundle and register 311 location = bundler.add(source, destination) 312 if location in self._files: 313 raise Exception("Cannot add %s to bundle %s, already in %s" % ( 314 location, bundleName, self._files[location])) 315 self._files[location] = bundleName 316 317 # add possible imports from this file 318 package = None 319 if location.endswith('.py'): 320 package = location[:-3] 321 elif location.endswith('.pyc'): 322 package = location[:-4] 323 324 if package: 325 if package.endswith('__init__'): 326 package = os.path.split(package)[0] 327 328 package = ".".join(package.split('/')) # win32 fixme 329 if package in self._imports: 330 raise Exception("Bundler %s already has import %s" % ( 331 bundleName, package)) 332 self._imports[package] = bundleName
333
334 - def depend(self, depender, *dependencies):
335 """ 336 Make the given bundle depend on the other given bundles. 337 338 @type depender: string 339 @type dependencies: list of strings 340 """ 341 # note that a bundler doesn't necessarily need to be registered yet 342 if not self._graph.hasNode(depender): 343 self._graph.addNode(depender) 344 for dep in dependencies: 345 if not self._graph.hasNode(dep): 346 self._graph.addNode(dep) 347 self._graph.addEdge(depender, dep)
348
349 - def getDependencies(self, bundlerName):
350 """ 351 Return names of all the dependencies of this bundle, including this 352 bundle itself. 353 The dependencies are returned in a correct depending order. 354 """ 355 if not bundlerName in self._bundlers: 356 raise errors.NoBundleError('Unknown bundle %s' % bundlerName) 357 elif not self._graph.hasNode(bundlerName): 358 return [bundlerName] 359 else: 360 return [bundlerName] + self._graph.getOffspring(bundlerName)
361
362 - def getBundlerByName(self, bundlerName):
363 """ 364 Return the bundle by name, or None if not found. 365 """ 366 if bundlerName in self._bundlers: 367 return self._bundlers[bundlerName] 368 return None
369
370 - def getBundlerNameByImport(self, importString):
371 """ 372 Return the bundler name by import statement, or None if not found. 373 """ 374 if importString in self._imports: 375 return self._imports[importString] 376 return None
377
378 - def getBundlerNameByFile(self, filename):
379 """ 380 Return the bundler name by filename, or None if not found. 381 """ 382 if filename in self._files: 383 return self._files[filename] 384 return None
385
386 - def getBundlerNames(self):
387 """ 388 Get all bundler names. 389 390 @rtype: list of str 391 @returns: a list of all bundler names in this basket. 392 """ 393 return self._bundlers.keys()
394 395
396 -class MergedBundler(Bundler):
397 """ 398 I am a bundler, with the extension that I can also bundle other 399 bundlers. 400 401 The effect is that when you call bundle() on a me, you get one 402 bundle with a union of all subbundlers' files, in addition to any 403 loose files that you added to me. 404 """ 405
406 - def __init__(self, name='merged-bundle'):
407 Bundler.__init__(self, name) 408 self._subbundlers = {}
409
410 - def addBundler(self, bundler):
411 """Add to me all of the files managed by another bundler. 412 413 @param bundler: The bundler whose files you want in this 414 bundler. 415 @type bundler: L{Bundler} 416 """ 417 if bundler.name not in self._subbundlers: 418 self._subbundlers[bundler.name] = bundler 419 for bfile in bundler._files.values(): 420 self.add(bfile.source, bfile.destination)
421
422 - def getSubBundlers(self):
423 """ 424 @returns: A list of all of the bundlers that have been added to 425 me. 426 """ 427 return self._subbundlers.values()
428