0001 """ 0002 Import utilities 0003 0004 Exported classes: 0005 ImportManager Manage the import process 0006 0007 Importer Base class for replacing standard import functions 0008 BuiltinImporter Emulate the import mechanism for builtin and frozen modules 0009 0010 DynLoadSuffixImporter 0011 """ 0012 0013 # note: avoid importing non-builtin modules 0014 import imp ### not available in JPython? 0015 import sys 0016 import __builtin__ 0017 0018 # for the DirectoryImporter 0019 import struct 0020 import marshal 0021 0022 __all__ = ["ImportManager","Importer","BuiltinImporter"] 0023 0024 _StringType = type('') 0025 _ModuleType = type(sys) ### doesn't work in JPython... 0026 0027 class ImportManager: 0028 "Manage the import process." 0029 0030 def install(self, namespace=vars(__builtin__)): 0031 "Install this ImportManager into the specified namespace." 0032 0033 if isinstance(namespace, _ModuleType): 0034 namespace = vars(namespace) 0035 0036 # Note: we have no notion of "chaining" 0037 0038 # Record the previous import hook, then install our own. 0039 self.previous_importer = namespace['__import__'] 0040 self.namespace = namespace 0041 namespace['__import__'] = self._import_hook 0042 0043 ### fix this 0044 #namespace['reload'] = self._reload_hook 0045 0046 def uninstall(self): 0047 "Restore the previous import mechanism." 0048 self.namespace['__import__'] = self.previous_importer 0049 0050 def add_suffix(self, suffix, importFunc): 0051 assert callable(importFunc) 0052 self.fs_imp.add_suffix(suffix, importFunc) 0053 0054 ###################################################################### 0055 # 0056 # PRIVATE METHODS 0057 # 0058 0059 clsFilesystemImporter = None 0060 0061 def __init__(self, fs_imp=None): 0062 # we're definitely going to be importing something in the future, 0063 # so let's just load the OS-related facilities. 0064 if not _os_stat: 0065 _os_bootstrap() 0066 0067 # This is the Importer that we use for grabbing stuff from the 0068 # filesystem. It defines one more method (import_from_dir) for our use. 0069 if fs_imp is None: 0070 cls = self.clsFilesystemImporter or _FilesystemImporter 0071 fs_imp = cls() 0072 self.fs_imp = fs_imp 0073 0074 # Initialize the set of suffixes that we recognize and import. 0075 # The default will import dynamic-load modules first, followed by 0076 # .py files (or a .py file's cached bytecode) 0077 for desc in imp.get_suffixes(): 0078 if desc[2] == imp.C_EXTENSION: 0079 self.add_suffix(desc[0], 0080 DynLoadSuffixImporter(desc).import_file) 0081 self.add_suffix('.py', py_suffix_importer) 0082 0083 def _import_hook(self, fqname, globals=None, locals=None, fromlist=None): 0084 """Python calls this hook to locate and import a module.""" 0085 0086 parts = fqname.split('.') 0087 0088 # determine the context of this import 0089 parent = self._determine_import_context(globals) 0090 0091 # if there is a parent, then its importer should manage this import 0092 if parent: 0093 module = parent.__importer__._do_import(parent, parts, fromlist) 0094 if module: 0095 return module 0096 0097 # has the top module already been imported? 0098 try: 0099 top_module = sys.modules[parts[0]] 0100 except KeyError: 0101 0102 # look for the topmost module 0103 top_module = self._import_top_module(parts[0]) 0104 if not top_module: 0105 # the topmost module wasn't found at all. 0106 raise ImportError, 'No module named ' + fqname 0107 0108 # fast-path simple imports 0109 if len(parts) == 1: 0110 if not fromlist: 0111 return top_module 0112 0113 if not top_module.__dict__.get('__ispkg__'): 0114 # __ispkg__ isn't defined (the module was not imported by us), 0115 # or it is zero. 0116 # 0117 # In the former case, there is no way that we could import 0118 # sub-modules that occur in the fromlist (but we can't raise an 0119 # error because it may just be names) because we don't know how 0120 # to deal with packages that were imported by other systems. 0121 # 0122 # In the latter case (__ispkg__ == 0), there can't be any sub- 0123 # modules present, so we can just return. 0124 # 0125 # In both cases, since len(parts) == 1, the top_module is also 0126 # the "bottom" which is the defined return when a fromlist 0127 # exists. 0128 return top_module 0129 0130 importer = top_module.__dict__.get('__importer__') 0131 if importer: 0132 return importer._finish_import(top_module, parts[1:], fromlist) 0133 0134 # Grrr, some people "import os.path" 0135 if len(parts) == 2 and hasattr(top_module, parts[1]): 0136 return top_module 0137 0138 # If the importer does not exist, then we have to bail. A missing 0139 # importer means that something else imported the module, and we have 0140 # no knowledge of how to get sub-modules out of the thing. 0141 raise ImportError, 'No module named ' + fqname 0142 0143 def _determine_import_context(self, globals): 0144 """Returns the context in which a module should be imported. 0145 0146 The context could be a loaded (package) module and the imported module 0147 will be looked for within that package. The context could also be None, 0148 meaning there is no context -- the module should be looked for as a 0149 "top-level" module. 0150 """ 0151 0152 if not globals or not globals.get('__importer__'): 0153 # globals does not refer to one of our modules or packages. That 0154 # implies there is no relative import context (as far as we are 0155 # concerned), and it should just pick it off the standard path. 0156 return None 0157 0158 # The globals refer to a module or package of ours. It will define 0159 # the context of the new import. Get the module/package fqname. 0160 parent_fqname = globals['__name__'] 0161 0162 # if a package is performing the import, then return itself (imports 0163 # refer to pkg contents) 0164 if globals['__ispkg__']: 0165 parent = sys.modules[parent_fqname] 0166 assert globals is parent.__dict__ 0167 return parent 0168 0169 i = parent_fqname.rfind('.') 0170 0171 # a module outside of a package has no particular import context 0172 if i == -1: 0173 return None 0174 0175 # if a module in a package is performing the import, then return the 0176 # package (imports refer to siblings) 0177 parent_fqname = parent_fqname[:i] 0178 parent = sys.modules[parent_fqname] 0179 assert parent.__name__ == parent_fqname 0180 return parent 0181 0182 def _import_top_module(self, name): 0183 # scan sys.path looking for a location in the filesystem that contains 0184 # the module, or an Importer object that can import the module. 0185 for item in sys.path: 0186 if isinstance(item, _StringType): 0187 module = self.fs_imp.import_from_dir(item, name) 0188 else: 0189 module = item.import_top(name) 0190 if module: 0191 return module 0192 return None 0193 0194 def _reload_hook(self, module): 0195 "Python calls this hook to reload a module." 0196 0197 # reloading of a module may or may not be possible (depending on the 0198 # importer), but at least we can validate that it's ours to reload 0199 importer = module.__dict__.get('__importer__') 0200 if not importer: 0201 ### oops. now what... 0202 pass 0203 0204 # okay. it is using the imputil system, and we must delegate it, but 0205 # we don't know what to do (yet) 0206 ### we should blast the module dict and do another get_code(). need to 0207 ### flesh this out and add proper docco... 0208 raise SystemError, "reload not yet implemented" 0209 0210 0211 class Importer: 0212 "Base class for replacing standard import functions." 0213 0214 def import_top(self, name): 0215 "Import a top-level module." 0216 return self._import_one(None, name, name) 0217 0218 ###################################################################### 0219 # 0220 # PRIVATE METHODS 0221 # 0222 def _finish_import(self, top, parts, fromlist): 0223 # if "a.b.c" was provided, then load the ".b.c" portion down from 0224 # below the top-level module. 0225 bottom = self._load_tail(top, parts) 0226 0227 # if the form is "import a.b.c", then return "a" 0228 if not fromlist: 0229 # no fromlist: return the top of the import tree 0230 return top 0231 0232 # the top module was imported by self. 0233 # 0234 # this means that the bottom module was also imported by self (just 0235 # now, or in the past and we fetched it from sys.modules). 0236 # 0237 # since we imported/handled the bottom module, this means that we can 0238 # also handle its fromlist (and reliably use __ispkg__). 0239 0240 # if the bottom node is a package, then (potentially) import some 0241 # modules. 0242 # 0243 # note: if it is not a package, then "fromlist" refers to names in 0244 # the bottom module rather than modules. 0245 # note: for a mix of names and modules in the fromlist, we will 0246 # import all modules and insert those into the namespace of 0247 # the package module. Python will pick up all fromlist names 0248 # from the bottom (package) module; some will be modules that 0249 # we imported and stored in the namespace, others are expected 0250 # to be present already. 0251 if bottom.__ispkg__: 0252 self._import_fromlist(bottom, fromlist) 0253 0254 # if the form is "from a.b import c, d" then return "b" 0255 return bottom 0256 0257 def _import_one(self, parent, modname, fqname): 0258 "Import a single module." 0259 0260 # has the module already been imported? 0261 try: 0262 return sys.modules[fqname] 0263 except KeyError: 0264 pass 0265 0266 # load the module's code, or fetch the module itself 0267 result = self.get_code(parent, modname, fqname) 0268 if result is None: 0269 return None 0270 0271 module = self._process_result(result, fqname) 0272 0273 # insert the module into its parent 0274 if parent: 0275 setattr(parent, modname, module) 0276 return module 0277 0278 def _process_result(self, (ispkg, code, values), fqname): 0279 # did get_code() return an actual module? (rather than a code object) 0280 is_module = isinstance(code, _ModuleType) 0281 0282 # use the returned module, or create a new one to exec code into 0283 if is_module: 0284 module = code 0285 else: 0286 module = imp.new_module(fqname) 0287 0288 ### record packages a bit differently?? 0289 module.__importer__ = self 0290 module.__ispkg__ = ispkg 0291 0292 # insert additional values into the module (before executing the code) 0293 module.__dict__.update(values) 0294 0295 # the module is almost ready... make it visible 0296 sys.modules[fqname] = module 0297 0298 # execute the code within the module's namespace 0299 if not is_module: 0300 try: 0301 exec code in module.__dict__ 0302 except: 0303 if fqname in sys.modules: 0304 del sys.modules[fqname] 0305 raise 0306 0307 # fetch from sys.modules instead of returning module directly. 0308 # also make module's __name__ agree with fqname, in case 0309 # the "exec code in module.__dict__" played games on us. 0310 module = sys.modules[fqname] 0311 module.__name__ = fqname 0312 return module 0313 0314 def _load_tail(self, m, parts): 0315 """Import the rest of the modules, down from the top-level module. 0316 0317 Returns the last module in the dotted list of modules. 0318 """ 0319 for part in parts: 0320 fqname = "%s.%s" % (m.__name__, part) 0321 m = self._import_one(m, part, fqname) 0322 if not m: 0323 raise ImportError, "No module named " + fqname 0324 return m 0325 0326 def _import_fromlist(self, package, fromlist): 0327 'Import any sub-modules in the "from" list.' 0328 0329 # if '*' is present in the fromlist, then look for the '__all__' 0330 # variable to find additional items (modules) to import. 0331 if '*' in fromlist: 0332 fromlist = list(fromlist) + \ 0333 list(package.__dict__.get('__all__', [])) 0334 0335 for sub in fromlist: 0336 # if the name is already present, then don't try to import it (it 0337 # might not be a module!). 0338 if sub != '*' and not hasattr(package, sub): 0339 subname = "%s.%s" % (package.__name__, sub) 0340 submod = self._import_one(package, sub, subname) 0341 if not submod: 0342 raise ImportError, "cannot import name " + subname 0343 0344 def _do_import(self, parent, parts, fromlist): 0345 """Attempt to import the module relative to parent. 0346 0347 This method is used when the import context specifies that <self> 0348 imported the parent module. 0349 """ 0350 top_name = parts[0] 0351 top_fqname = parent.__name__ + '.' + top_name 0352 top_module = self._import_one(parent, top_name, top_fqname) 0353 if not top_module: 0354 # this importer and parent could not find the module (relatively) 0355 return None 0356 0357 return self._finish_import(top_module, parts[1:], fromlist) 0358 0359 ###################################################################### 0360 # 0361 # METHODS TO OVERRIDE 0362 # 0363 def get_code(self, parent, modname, fqname): 0364 """Find and retrieve the code for the given module. 0365 0366 parent specifies a parent module to define a context for importing. It 0367 may be None, indicating no particular context for the search. 0368 0369 modname specifies a single module (not dotted) within the parent. 0370 0371 fqname specifies the fully-qualified module name. This is a 0372 (potentially) dotted name from the "root" of the module namespace 0373 down to the modname. 0374 If there is no parent, then modname==fqname. 0375 0376 This method should return None, or a 3-tuple. 0377 0378 * If the module was not found, then None should be returned. 0379 0380 * The first item of the 2- or 3-tuple should be the integer 0 or 1, 0381 specifying whether the module that was found is a package or not. 0382 0383 * The second item is the code object for the module (it will be 0384 executed within the new module's namespace). This item can also 0385 be a fully-loaded module object (e.g. loaded from a shared lib). 0386 0387 * The third item is a dictionary of name/value pairs that will be 0388 inserted into new module before the code object is executed. This 0389 is provided in case the module's code expects certain values (such 0390 as where the module was found). When the second item is a module 0391 object, then these names/values will be inserted *after* the module 0392 has been loaded/initialized. 0393 """ 0394 raise RuntimeError, "get_code not implemented" 0395 0396 0397 ###################################################################### 0398 # 0399 # Some handy stuff for the Importers 0400 # 0401 0402 # byte-compiled file suffix character 0403 _suffix_char = __debug__ and 'c' or 'o' 0404 0405 # byte-compiled file suffix 0406 _suffix = '.py' + _suffix_char 0407 0408 def _compile(pathname, timestamp): 0409 """Compile (and cache) a Python source file. 0410 0411 The file specified by <pathname> is compiled to a code object and 0412 returned. 0413 0414 Presuming the appropriate privileges exist, the bytecodes will be 0415 saved back to the filesystem for future imports. The source file's 0416 modification timestamp must be provided as a Long value. 0417 """ 0418 codestring = open(pathname, 'rU').read() 0419 if codestring and codestring[-1] != '\n': 0420 codestring = codestring + '\n' 0421 code = __builtin__.compile(codestring, pathname, 'exec') 0422 0423 # try to cache the compiled code 0424 try: 0425 f = open(pathname + _suffix_char, 'wb') 0426 except IOError: 0427 pass 0428 else: 0429 f.write('\0\0\0\0') 0430 f.write(struct.pack('<I', timestamp)) 0431 marshal.dump(code, f) 0432 f.flush() 0433 f.seek(0, 0) 0434 f.write(imp.get_magic()) 0435 f.close() 0436 0437 return code 0438 0439 _os_stat = _os_path_join = None 0440 def _os_bootstrap(): 0441 "Set up 'os' module replacement functions for use during import bootstrap." 0442 0443 names = sys.builtin_module_names 0444 0445 join = None 0446 if 'posix' in names: 0447 sep = '/' 0448 from posix import stat 0449 elif 'nt' in names: 0450 sep = '\\' 0451 from nt import stat 0452 elif 'dos' in names: 0453 sep = '\\' 0454 from dos import stat 0455 elif 'os2' in names: 0456 sep = '\\' 0457 from os2 import stat 0458 elif 'mac' in names: 0459 from mac import stat 0460 def join(a, b): 0461 if a == '': 0462 return b 0463 if ':' not in a: 0464 a = ':' + a 0465 if a[-1:] != ':': 0466 a = a + ':' 0467 return a + b 0468 else: 0469 raise ImportError, 'no os specific module found' 0470 0471 if join is None: 0472 def join(a, b, sep=sep): 0473 if a == '': 0474 return b 0475 lastchar = a[-1:] 0476 if lastchar == '/' or lastchar == sep: 0477 return a + b 0478 return a + sep + b 0479 0480 global _os_stat 0481 _os_stat = stat 0482 0483 global _os_path_join 0484 _os_path_join = join 0485 0486 def _os_path_isdir(pathname): 0487 "Local replacement for os.path.isdir()." 0488 try: 0489 s = _os_stat(pathname) 0490 except OSError: 0491 return None 0492 return (s.st_mode & 0170000) == 0040000 0493 0494 def _timestamp(pathname): 0495 "Return the file modification time as a Long." 0496 try: 0497 s = _os_stat(pathname) 0498 except OSError: 0499 return None 0500 return long(s.st_mtime) 0501 0502 0503 ###################################################################### 0504 # 0505 # Emulate the import mechanism for builtin and frozen modules 0506 # 0507 class BuiltinImporter(Importer): 0508 def get_code(self, parent, modname, fqname): 0509 if parent: 0510 # these modules definitely do not occur within a package context 0511 return None 0512 0513 # look for the module 0514 if imp.is_builtin(modname): 0515 type = imp.C_BUILTIN 0516 elif imp.is_frozen(modname): 0517 type = imp.PY_FROZEN 0518 else: 0519 # not found 0520 return None 0521 0522 # got it. now load and return it. 0523 module = imp.load_module(modname, None, modname, ('', '', type)) 0524 return 0, module, { } 0525 0526 0527 ###################################################################### 0528 # 0529 # Internal importer used for importing from the filesystem 0530 # 0531 class _FilesystemImporter(Importer): 0532 def __init__(self): 0533 self.suffixes = [ ] 0534 0535 def add_suffix(self, suffix, importFunc): 0536 assert callable(importFunc) 0537 self.suffixes.append((suffix, importFunc)) 0538 0539 def import_from_dir(self, dir, fqname): 0540 result = self._import_pathname(_os_path_join(dir, fqname), fqname) 0541 if result: 0542 return self._process_result(result, fqname) 0543 return None 0544 0545 def get_code(self, parent, modname, fqname): 0546 # This importer is never used with an empty parent. Its existence is 0547 # private to the ImportManager. The ImportManager uses the 0548 # import_from_dir() method to import top-level modules/packages. 0549 # This method is only used when we look for a module within a package. 0550 assert parent 0551 0552 return self._import_pathname(_os_path_join(parent.__pkgdir__, modname), 0553 fqname) 0554 0555 def _import_pathname(self, pathname, fqname): 0556 if _os_path_isdir(pathname): 0557 result = self._import_pathname(_os_path_join(pathname, '__init__'), 0558 fqname) 0559 if result: 0560 values = result[2] 0561 values['__pkgdir__'] = pathname 0562 values['__path__'] = [ pathname ] 0563 return 1, result[1], values 0564 return None 0565 0566 for suffix, importFunc in self.suffixes: 0567 filename = pathname + suffix 0568 try: 0569 finfo = _os_stat(filename) 0570 except OSError: 0571 pass 0572 else: 0573 return importFunc(filename, finfo, fqname) 0574 return None 0575 0576 ###################################################################### 0577 # 0578 # SUFFIX-BASED IMPORTERS 0579 # 0580 0581 def py_suffix_importer(filename, finfo, fqname): 0582 file = filename[:-3] + _suffix 0583 t_py = long(finfo[8]) 0584 t_pyc = _timestamp(file) 0585 0586 code = None 0587 if t_pyc is not None and t_pyc >= t_py: 0588 f = open(file, 'rb') 0589 if f.read(4) == imp.get_magic(): 0590 t = struct.unpack('<I', f.read(4))[0] 0591 if t == t_py: 0592 code = marshal.load(f) 0593 f.close() 0594 if code is None: 0595 file = filename 0596 code = _compile(file, t_py) 0597 0598 return 0, code, { '__file__' : file } 0599 0600 class DynLoadSuffixImporter: 0601 def __init__(self, desc): 0602 self.desc = desc 0603 0604 def import_file(self, filename, finfo, fqname): 0605 fp = open(filename, self.desc[1]) 0606 module = imp.load_module(fqname, fp, filename, self.desc) 0607 module.__file__ = filename 0608 return 0, module, { } 0609 0610 0611 ###################################################################### 0612 0613 def _print_importers(): 0614 items = sys.modules.items() 0615 items.sort() 0616 for name, module in items: 0617 if module: 0618 print name, module.__dict__.get('__importer__', '-- no importer') 0619 else: 0620 print name, '-- non-existent module' 0621 0622 def _test_revamp(): 0623 ImportManager().install() 0624 sys.path.insert(0, BuiltinImporter()) 0625 0626 ###################################################################### 0627 0628 # 0629 # TODO 0630 # 0631 # from Finn Bock: 0632 # type(sys) is not a module in JPython. what to use instead? 0633 # imp.C_EXTENSION is not in JPython. same for get_suffixes and new_module 0634 # 0635 # given foo.py of: 0636 # import sys 0637 # sys.modules['foo'] = sys 0638 # 0639 # ---- standard import mechanism 0640 # >>> import foo 0641 # >>> foo 0642 # <module 'sys' (built-in)> 0643 # 0644 # ---- revamped import mechanism 0645 # >>> import imputil 0646 # >>> imputil._test_revamp() 0647 # >>> import foo 0648 # >>> foo 0649 # <module 'foo' from 'foo.py'> 0650 # 0651 # 0652 # from MAL: 0653 # should BuiltinImporter exist in sys.path or hard-wired in ImportManager? 0654 # need __path__ processing 0655 # performance 0656 # move chaining to a subclass [gjs: it's been nuked] 0657 # deinstall should be possible 0658 # query mechanism needed: is a specific Importer installed? 0659 # py/pyc/pyo piping hooks to filter/process these files 0660 # wish list: 0661 # distutils importer hooked to list of standard Internet repositories 0662 # module->file location mapper to speed FS-based imports 0663 # relative imports 0664 # keep chaining so that it can play nice with other import hooks 0665 # 0666 # from Gordon: 0667 # push MAL's mapper into sys.path[0] as a cache (hard-coded for apps) 0668 # 0669 # from Guido: 0670 # need to change sys.* references for rexec environs 0671 # need hook for MAL's walk-me-up import strategy, or Tim's absolute strategy 0672 # watch out for sys.modules[...] is None 0673 # flag to force absolute imports? (speeds _determine_import_context and 0674 # checking for a relative module) 0675 # insert names of archives into sys.path (see quote below) 0676 # note: reload does NOT blast module dict 0677 # shift import mechanisms and policies around; provide for hooks, overrides 0678 # (see quote below) 0679 # add get_source stuff 0680 # get_topcode and get_subcode 0681 # CRLF handling in _compile 0682 # race condition in _compile 0683 # refactoring of os.py to deal with _os_bootstrap problem 0684 # any special handling to do for importing a module with a SyntaxError? 0685 # (e.g. clean up the traceback) 0686 # implement "domain" for path-type functionality using pkg namespace 0687 # (rather than FS-names like __path__) 0688 # don't use the word "private"... maybe "internal" 0689 # 0690 # 0691 # Guido's comments on sys.path caching: 0692 # 0693 # We could cache this in a dictionary: the ImportManager can have a 0694 # cache dict mapping pathnames to importer objects, and a separate 0695 # method for coming up with an importer given a pathname that's not yet 0696 # in the cache. The method should do a stat and/or look at the 0697 # extension to decide which importer class to use; you can register new 0698 # importer classes by registering a suffix or a Boolean function, plus a 0699 # class. If you register a new importer class, the cache is zapped. 0700 # The cache is independent from sys.path (but maintained per 0701 # ImportManager instance) so that rearrangements of sys.path do the 0702 # right thing. If a path is dropped from sys.path the corresponding 0703 # cache entry is simply no longer used. 0704 # 0705 # My/Guido's comments on factoring ImportManager and Importer: 0706 # 0707 # > However, we still have a tension occurring here: 0708 # > 0709 # > 1) implementing policy in ImportManager assists in single-point policy 0710 # > changes for app/rexec situations 0711 # > 2) implementing policy in Importer assists in package-private policy 0712 # > changes for normal, operating conditions 0713 # > 0714 # > I'll see if I can sort out a way to do this. Maybe the Importer class will 0715 # > implement the methods (which can be overridden to change policy) by 0716 # > delegating to ImportManager. 0717 # 0718 # Maybe also think about what kind of policies an Importer would be 0719 # likely to want to change. I have a feeling that a lot of the code 0720 # there is actually not so much policy but a *necessity* to get things 0721 # working given the calling conventions for the __import__ hook: whether 0722 # to return the head or tail of a dotted name, or when to do the "finish 0723 # fromlist" stuff. 0724 # 0725
Generated by PyXR 0.9.4