Package instant :: Module build
[hide private]
[frames] | no frames]

Source Code for Module instant.build

  1  """This module contains the main part of Instant, the build_module function.""" 
  2   
  3  import os, sys, shutil, glob 
  4  from itertools import chain 
  5   
  6  # TODO: Import only the official interface 
  7  from output import * 
  8  from config import header_and_libs_from_pkgconfig 
  9  from paths import * 
 10  from signatures import * 
 11  from cache import * 
 12  from codegeneration import * 
 13  from locking import get_lock, release_lock 
 14       
15 -def assert_is_str(x):
16 instant_assert(isinstance(x, str), 17 "In instant.build_module: Expecting string.")
18
19 -def assert_is_bool(x):
20 instant_assert(isinstance(x, bool), 21 "In instant.build_module: Expecting bool.")
22
23 -def assert_is_str_list(x):
24 instant_assert(isinstance(x, (list, tuple)), 25 "In instant.build_module: Expecting sequence.") 26 instant_assert(all(isinstance(i, str) for i in x), 27 "In instant.build_module: Expecting sequence of strings.")
28
29 -def strip_strings(x):
30 assert_is_str_list(x) 31 return [s.strip() for s in x]
32
33 -def arg_strings(x):
34 if isinstance(x, str): 35 x = x.split() 36 return strip_strings(x)
37 38
39 -def copy_files(source, dest, files):
40 """Copy a list of files from a source directory to a destination directory. 41 This may seem a bit complicated, but a lot of this code is error checking.""" 42 if os.path.exists(dest): 43 overwriting = set(files) & set(glob.glob(os.path.join(dest, "*"))) 44 if overwriting: 45 instant_warning("In instant.copy_files: Path '%s' already exists, "\ 46 "overwriting existing files: %r." % (dest, list(overwriting))) 47 else: 48 os.mkdir(dest) 49 50 if source != dest: 51 instant_debug("In instant.copy_files: Copying files %r from %r to %r"\ 52 % (files, source, dest)) 53 54 for f in files: 55 a = os.path.join(source, f) 56 b = os.path.join(dest, f) 57 instant_assert(a != b, "In instant.copy_files: Seems like the "\ 58 "input files are absolute paths, should be relative to "\ 59 "source. (%r, %r)" % (a, b)) 60 instant_assert(os.path.isfile(a), "In instant.copy_files: "\ 61 "Missing source file '%s'." % a) 62 if os.path.isfile(b): 63 os.remove(b) 64 shutil.copyfile(a, b)
65 66
67 -def recompile(modulename, module_path, setup_name, new_compilation_checksum):
68 """Recompile module if the new checksum is different from 69 the one in the checksum file in the module directory.""" 70 # Check if the old checksum matches the new one 71 compilation_checksum_filename = "%s.checksum" % modulename 72 if os.path.exists(compilation_checksum_filename): 73 checksum_file = open(compilation_checksum_filename) 74 old_compilation_checksum = checksum_file.readline() 75 checksum_file.close() 76 if old_compilation_checksum == new_compilation_checksum: 77 return 78 79 # Verify that SWIG is on the system 80 (swig_stat, swig_out) = get_status_output("swig -version") 81 if swig_stat != 0: 82 instant_error("In instant.recompile: Could not find swig!"\ 83 " You can download swig from http://www.swig.org") 84 85 # Create log file for logging of compilation errors 86 compile_log_filename = os.path.join(module_path, "compile.log") 87 compile_log_file = open(compile_log_filename, "w") 88 try: 89 # Build module 90 cmd = "python %s build_ext" % setup_name 91 instant_info("--- Instant: compiling ---") 92 instant_debug("cmd = %s" % cmd) 93 ret, output = get_status_output(cmd) 94 compile_log_file.write(output) 95 compile_log_file.flush() 96 if ret != 0: 97 if os.path.exists(compilation_checksum_filename): 98 os.remove(compilation_checksum_filename) 99 instant_error("In instant.recompile: The module did not "\ 100 "compile, see '%s'" % compile_log_filename) 101 102 # 'Install' module 103 cmd = "python %s install --install-platlib=." % setup_name 104 instant_debug("cmd = %s" % cmd) 105 ret, output = get_status_output(cmd) 106 compile_log_file.write(output) 107 compile_log_file.flush() 108 if ret != 0: 109 if os.path.exists(compilation_checksum_filename): 110 os.remove(compilation_checksum_filename) 111 instant_error("In instant.recompile: Could not 'install' "\ 112 "the module, see '%s'" % compile_log_filename) 113 finally: 114 compile_log_file.close() 115 116 # Compilation succeeded, write new_compilation_checksum to checksum_file 117 write_file(compilation_checksum_filename, new_compilation_checksum)
118 119
120 -def copy_to_cache(module_path, cache_dir, modulename):
121 "Copy module directory to cache." 122 # Get lock, check if the module exists, _otherwise_ copy the 123 # finished compiled module from /tmp/foo to the cache directory, 124 # and then release lock 125 lock = get_lock(cache_dir, modulename) 126 127 # Validate the path 128 cache_module_path = os.path.join(cache_dir, modulename) 129 if os.path.exists(cache_module_path): 130 # This indicates a race condition has happened (and is being avoided!). 131 instant_warning("In instant.build_module: Path '%s' already exists,"\ 132 " but module wasn't found in cache previously. Not overwriting,"\ 133 " assuming this module is valid." % cache_module_path) 134 135 release_lock(lock) 136 return cache_module_path 137 138 # Not deleting anymore, relying on locking system 139 #shutil.rmtree(cache_module_path, ignore_errors=True) 140 141 # Error checks 142 instant_assert(os.path.isdir(module_path), "In instant.build_module:"\ 143 " Cannot copy non-existing directory %r!" % module_path) 144 instant_assert(not os.path.isdir(cache_module_path), 145 "In instant.build_module: Cache directory %r shouldn't exist "\ 146 "at this point!" % cache_module_path) 147 instant_debug("In instant.build_module: Copying built module from %r"\ 148 " to cache at %r" % (module_path, cache_module_path)) 149 150 # Do the copying 151 shutil.copytree(module_path, cache_module_path) 152 delete_temp_dir() 153 release_lock(lock) 154 return cache_module_path
155 156
157 -def build_module(modulename=None, source_directory=".", 158 code="", init_code="", 159 additional_definitions="", additional_declarations="", 160 sources=[], wrap_headers=[], 161 local_headers=[], system_headers=[], 162 include_dirs=['.'], library_dirs=[], libraries=[], 163 swigargs=['-c++', '-fcompact', '-O', '-I.', '-small'], 164 swig_include_dirs = [], 165 cppargs=['-O2'], lddargs=[], 166 object_files=[], arrays=[], 167 generate_interface=True, generate_setup=True, 168 signature=None, cache_dir=None):
169 """Generate and compile a module from C/C++ code using SWIG. 170 171 Arguments: 172 ========== 173 The keyword arguments are as follows: 174 - B{modulename}: 175 - The name you want for the module. 176 If specified, the module will not be cached. 177 If missing, a name will be constructed based on 178 a checksum of the other arguments, and the module 179 will be placed in the global cache. String. 180 - B{source_directory}: 181 - The directory where user supplied files reside. The files 182 given in B{sources}, B{wrap_headers}, and B{local_headers} 183 are expected to exist in this directory. String. 184 - B{code}: 185 - A string containing C or C++ code to be compiled and wrapped. String. 186 - B{init_code}: 187 - Code that should be executed when the Instant module is 188 imported. This code is inserted in the SWIG interface file, and is 189 used for instance for calling C{import_array()} used for the 190 initialization of NumPy arrays. String. 191 - B{additional_definitions}: 192 - Additional definitions (typically needed for inheritance) 193 for interface file. These definitions should be given as triple-quoted 194 strings in the case they span multiple lines, and are placed both in the 195 initial block for C/C++ code (C{%{,%}}-block), and the main section 196 of the interface file. String. 197 - B{additional_declarations}: 198 - Additional declarations (typically needed for inheritance) 199 for interface file. These declarations should be given as triple-quoted 200 strings in the case they span multiple lines, and are plaves in the main 201 section of the interface file. String. 202 - B{sources}: 203 - Source files to compile and link with the module. These 204 files are compiled togehter with the SWIG-generated wrapper file into 205 the final library file. Should reside in directory specified in 206 B{source_directory}. List of strings. 207 - B{wrap_headers}: 208 - Local header files that should be wrapped by SWIG. The 209 files specified will be included both in the initial block for C/C++ code 210 (with a C directive) and in the main section of the interface file (with 211 a SWIG directive). Should reside in directory specified in 212 B{source_directory}. List of strings. 213 - B{local_headers}: 214 - Local header files required to compile the wrapped 215 code. The files specified will be included in the initial block for 216 C/C++ code (with a C directive). Should reside in directory specified in 217 B{source_directory}. List of strings. 218 - B{system_headers}: 219 - System header files required to compile the wrapped 220 code. The files specified will be included in the initial block for C/C++ 221 code (with a C directive). List of strings. 222 - B{include_dirs}: 223 - Directories to search for header files for building the 224 extension module. Needs to be absolute path names. List of strings. 225 - B{library_dirs}: 226 - Directories to search for libraries (C{-l}) for building 227 the extension module. Needs to be absolute paths. List of strings. 228 - B{libraries}: 229 - Libraries needed by the Instant module. The libraries will 230 be linked in from the shared object file. The initial C{-l} is added 231 automatically. List of strings. 232 - B{swigargs}: 233 - List of arguments to swig, e.g. C{["-lpointers.i"]} 234 to include the SWIG pointers.i library. 235 - B{swig_include_dirs}: 236 - A list of directories to include in the 'swig' command. 237 - B{cppargs}: 238 - List of arguments to the compiler, e.g. C{["-Wall", "-fopenmp"]}. 239 - B{lddargs}: 240 - List of arguments to the linker, e.g. C{["-E", "-U"]}. 241 - B{object_files}: 242 - If you want to compile the files yourself. TODO: Not yet supported. 243 - B{arrays}: 244 - A nested list describing the C arrays to be made from NumPy arrays. 245 If the NumPy array is 1D, the inner list should contain strings with 246 the variable names for length of the array and the array itself. 247 If the NumPy array is a matrix or a tensor, the inner list should 248 contain strings with variable names for the number of dimensions, 249 the length in each dimension, and the array itself, respectively. 250 - B{generate_interface}: 251 - A bool to indicate if you want to generate the interface files. 252 - B{generate_setup}: 253 - A bool to indicate if you want to generate the setup.py file. 254 - B{signature}: 255 - A signature string to identify the form instead of the source code. 256 - B{cache_dir}: 257 - A directory to look for cached modules and place new ones. 258 If missing, a default directory is used. Note that the module 259 will not be cached if B{modulename} is specified. 260 The cache directory should not be used for anything else. 261 """ 262 263 # Store original directory to be able to restore later 264 original_path = os.getcwd() 265 266 # --- Validate arguments 267 268 instant_assert(modulename is None or isinstance(modulename, str), 269 "In instant.build_module: Expecting modulename to be string or None.") 270 assert_is_str(source_directory) 271 source_directory = os.path.abspath(source_directory) 272 assert_is_str(code) 273 assert_is_str(init_code) 274 assert_is_str(additional_definitions) 275 assert_is_str(additional_declarations) 276 sources = strip_strings(sources) 277 wrap_headers = strip_strings(wrap_headers) 278 local_headers = strip_strings(local_headers) 279 system_headers = strip_strings(system_headers) 280 include_dirs = strip_strings(include_dirs) 281 library_dirs = strip_strings(library_dirs) 282 libraries = strip_strings(libraries) 283 swigargs = arg_strings(swigargs) 284 swig_include_dirs = strip_strings(swig_include_dirs) 285 cppargs = arg_strings(cppargs) 286 lddargs = arg_strings(lddargs) 287 object_files = strip_strings(object_files) 288 arrays = [strip_strings(a) for a in arrays] 289 assert_is_bool(generate_interface) 290 assert_is_bool(generate_setup) 291 instant_assert( signature is None \ 292 or isinstance(signature, str) \ 293 or hasattr(signature, "signature"), 294 "In instant.build_module: Expecting modulename to be string or None.") 295 instant_assert(not (signature is not None and modulename is not None), 296 "In instant.build_module: Can't have both modulename and signature.") 297 298 # --- Replace arguments with defaults if necessary 299 300 cache_dir = validate_cache_dir(cache_dir) 301 302 # Split sources by file-suffix (.c or .cpp) 303 csrcs = [f for f in sources if f.endswith('.c') or f.endswith('.C')] 304 cppsrcs = [f for f in sources if f.endswith('.cpp') or f.endswith('.cxx')] 305 instant_assert(len(csrcs) + len(cppsrcs) == len(sources), 306 "In instant.build_module: Source files must have '.c' or '.cpp' suffix") 307 308 # --- Debugging code 309 instant_debug('In instant.build_module:') 310 instant_debug('::: Begin Arguments :::') 311 instant_debug(' modulename: %r' % modulename) 312 instant_debug(' source_directory: %r' % source_directory) 313 instant_debug(' code: %r' % code) 314 instant_debug(' init_code: %r' % init_code) 315 instant_debug(' additional_definitions: %r' % additional_definitions) 316 instant_debug(' additional_declarations: %r' % additional_declarations) 317 instant_debug(' sources: %r' % sources) 318 instant_debug(' csrcs: %r' % csrcs) 319 instant_debug(' cppsrcs: %r' % cppsrcs) 320 instant_debug(' wrap_headers: %r' % wrap_headers) 321 instant_debug(' local_headers: %r' % local_headers) 322 instant_debug(' system_headers: %r' % system_headers) 323 instant_debug(' include_dirs: %r' % include_dirs) 324 instant_debug(' library_dirs: %r' % library_dirs) 325 instant_debug(' libraries: %r' % libraries) 326 instant_debug(' swigargs: %r' % swigargs) 327 instant_debug(' swig_include_dirs: %r' % swig_include_dirs) 328 instant_debug(' cppargs: %r' % cppargs) 329 instant_debug(' lddargs: %r' % lddargs) 330 instant_debug(' object_files: %r' % object_files) 331 instant_debug(' arrays: %r' % arrays) 332 instant_debug(' generate_interface: %r' % generate_interface) 333 instant_debug(' generate_setup: %r' % generate_setup) 334 instant_debug(' signature: %r' % signature) 335 instant_debug(' cache_dir: %r' % cache_dir) 336 instant_debug('::: End Arguments :::') 337 338 # --- Setup module directory, making it and copying 339 # files to it if necessary, and compute a modulename 340 # if it isn't specified explicitly 341 342 if modulename is None: 343 # Compute a signature if we have none passed by the user: 344 if signature is None: 345 # Collect arguments used for checksum creation, 346 # including everything that affects the interface 347 # file generation and module compilation. 348 checksum_args = ( \ 349 # We don't care about the modulename, that's what we're trying to construct! 350 #modulename, 351 # We don't care where the user code resides: 352 #source_directory, 353 code, init_code, 354 additional_definitions, 355 additional_declarations, 356 # Skipping filenames, since we use the file contents: 357 #sources, wrap_headers, 358 #local_headers, 359 system_headers, 360 include_dirs, library_dirs, libraries, 361 swig_include_dirs, swigargs, cppargs, lddargs, 362 object_files, arrays, 363 generate_interface, generate_setup, 364 # The signature isn't defined, and the cache_dir doesn't affect the module: 365 #signature, cache_dir) 366 ) 367 allfiles = sources + wrap_headers + local_headers 368 allfiles = [os.path.join(source_directory, file) for file in allfiles] 369 text = "\n".join((str(a) for a in checksum_args)) 370 signature = modulename_from_checksum(compute_checksum(text, allfiles)) 371 modulename = signature 372 moduleids = [signature] 373 else: 374 module, moduleids = check_memory_cache(signature) 375 if module: return module 376 modulename = moduleids[-1] 377 378 # Look for module in disk cache 379 module = check_disk_cache(modulename, cache_dir, moduleids) 380 if module: return module 381 382 # Make a temporary module path for compilation 383 module_path = os.path.join(get_temp_dir(), modulename) 384 instant_assert(not os.path.exists(module_path), 385 "In instant.build_module: Not expecting module_path to exist: '%s'"\ 386 % module_path) 387 os.mkdir(module_path) 388 use_cache = True 389 else: 390 use_cache = False 391 moduleids = [] 392 module_path = os.path.join(original_path, modulename) 393 if not os.path.exists(module_path): 394 os.mkdir(module_path) 395 396 ## Look for module in memory cache 397 #module, moduleids = check_memory_cache(modulename) 398 #if module: return module 399 #instant_assert(modulename == moduleids[-1] and len(moduleids) == 1, "Logic breach.") 400 ## Look for module in local directory 401 #module = check_disk_cache(modulename, original_path, moduleids) 402 #if module: return module 403 404 # Wrapping rest of code in try-block to 405 # clean up at the end if something fails. 406 try: 407 # --- Copy user-supplied files to module path 408 409 module_path = os.path.abspath(module_path) 410 files_to_copy = sources + wrap_headers + local_headers + object_files 411 copy_files(source_directory, module_path, files_to_copy) 412 # At this point, all user input files should reside in module_path. 413 414 # --- Generate additional files in module directory 415 os.chdir(module_path) 416 417 # Generate __init__.py which imports compiled module contents 418 write_file("__init__.py", "from %s import *" % modulename) 419 420 # Generate SWIG interface if wanted 421 ifile_name = "%s.i" % modulename 422 if generate_interface: 423 write_interfacefile(ifile_name, modulename, code, init_code, 424 additional_definitions, additional_declarations, system_headers, 425 local_headers, wrap_headers, arrays) 426 427 # Generate setup.py if wanted 428 setup_name = "setup.py" 429 if generate_setup: 430 write_setup(setup_name, modulename, csrcs, cppsrcs, local_headers, \ 431 include_dirs, library_dirs, libraries, swig_include_dirs, \ 432 swigargs, cppargs, lddargs) 433 434 # --- Build module 435 436 # At this point we have all the files, and can make the 437 # total checksum from all file contents. This is used to 438 # decide whether the module needs recompilation or not. 439 440 # Compute new_compilation_checksum 441 # Collect arguments used for checksum creation, 442 # including everything that affects the module compilation. 443 # Since the interface file is included in allfiles, 444 # we don't need stuff that modifies it here. 445 checksum_args = ( \ 446 # We don't care about the modulename, that's what 447 # we're trying to construct! 448 #modulename, 449 # We don't care where the user code resides: 450 #source_directory, 451 #code, init_code, 452 #additional_definitions, additional_declarations, 453 # Skipping filenames, since we use the file contents: 454 #sources, wrap_headers, 455 #local_headers, 456 system_headers, 457 include_dirs, library_dirs, libraries, 458 swigargs, swig_include_dirs, cppargs, lddargs, 459 object_files, #arrays, 460 #generate_interface, generate_setup, 461 # The signature isn't defined, and the 462 # cache_dir doesn't affect the module: 463 #signature, cache_dir) 464 ) 465 text = "\n".join((str(a) for a in checksum_args)) 466 allfiles = sources + wrap_headers + local_headers + [ifile_name] 467 new_compilation_checksum = compute_checksum(text, allfiles) 468 469 # Recompile if necessary 470 recompile(modulename, module_path, setup_name, new_compilation_checksum) 471 472 # --- Load, cache, and return module 473 474 # Copy compiled module to cache 475 if use_cache: 476 module_path = copy_to_cache(module_path, cache_dir, modulename) 477 478 # Import module and place in memory cache 479 # Do not use locks if use_cache is False: 480 if use_cache: 481 lock = get_lock(cache_dir, modulename) 482 module = import_and_cache_module(module_path, modulename, moduleids) 483 if use_cache: 484 release_lock(lock) 485 if not module: 486 instant_error("Failed to import newly compiled module!") 487 488 instant_debug("In instant.build_module: Returning %s from build_module."\ 489 % module) 490 return module 491 # The end! 492 493 finally: 494 # Always get back to original directory. 495 os.chdir(original_path) 496 497 instant_error("In instant.build_module: Should never reach this point!")
498 # end build_module 499