1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21 :type PY_SOURCE_EXTS: tuple(str)
22 :var PY_SOURCE_EXTS: list of possible python source file extension
23
24 :type STD_LIB_DIR: str
25 :var STD_LIB_DIR: directory where standard modules are located
26
27 :type BUILTIN_MODULES: dict
28 :var BUILTIN_MODULES: dictionary with builtin module names has key
29 """
30
31
32 __docformat__ = "restructuredtext en"
33
34 import sys
35 import os
36 from os.path import splitext, join, abspath, isdir, dirname, exists, basename
37 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
38 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
39 from distutils.errors import DistutilsPlatformError
40
41 try:
42 import zipimport
43 except ImportError:
44 zipimport = None
45
46 ZIPFILE = object()
47
48 from logilab.common import STD_BLACKLIST, _handle_blacklist
49
50
51
52
53
54
55
56 if sys.platform.startswith('win'):
57 PY_SOURCE_EXTS = ('py', 'pyw')
58 PY_COMPILED_EXTS = ('dll', 'pyd')
59 else:
60 PY_SOURCE_EXTS = ('py',)
61 PY_COMPILED_EXTS = ('so',)
62
63 try:
64 STD_LIB_DIR = get_python_lib(standard_lib=1)
65
66
67 except DistutilsPlatformError:
68 STD_LIB_DIR = '//'
69
70 EXT_LIB_DIR = get_python_lib()
71
72 BUILTIN_MODULES = dict(list(zip(sys.builtin_module_names,
73 [1]*len(sys.builtin_module_names))))
74
75
77 """exception raised when we are not able to get a python
78 source file for a precompiled file
79 """
80
83 self.module = module
84 self.obj = obj
85 self._imported = None
86
88 if self._imported is None:
89 self._imported = getattr(load_module_from_name(self.module),
90 self.obj)
91 return self._imported
92
94 try:
95 return super(LazyObject, self).__getattribute__(attr)
96 except AttributeError as ex:
97 return getattr(self._getobj(), attr)
98
100 return self._getobj()(*args, **kwargs)
101
102
104 """Load a Python module from its name.
105
106 :type dotted_name: str
107 :param dotted_name: python name of a module or package
108
109 :type path: list or None
110 :param path:
111 optional list of path where the module or package should be
112 searched (use sys.path if nothing or None is given)
113
114 :type use_sys: bool
115 :param use_sys:
116 boolean indicating whether the sys.modules dictionary should be
117 used or not
118
119
120 :raise ImportError: if the module or package is not found
121
122 :rtype: module
123 :return: the loaded module
124 """
125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126
127
129 """Load a python module from its splitted name.
130
131 :type parts: list(str) or tuple(str)
132 :param parts:
133 python name of a module or package splitted on '.'
134
135 :type path: list or None
136 :param path:
137 optional list of path where the module or package should be
138 searched (use sys.path if nothing or None is given)
139
140 :type use_sys: bool
141 :param use_sys:
142 boolean indicating whether the sys.modules dictionary should be used or not
143
144 :raise ImportError: if the module or package is not found
145
146 :rtype: module
147 :return: the loaded module
148 """
149 if use_sys:
150 try:
151 return sys.modules['.'.join(parts)]
152 except KeyError:
153 pass
154 modpath = []
155 prevmodule = None
156 for part in parts:
157 modpath.append(part)
158 curname = '.'.join(modpath)
159 module = None
160 if len(modpath) != len(parts):
161
162 module = sys.modules.get(curname)
163 elif use_sys:
164
165 module = sys.modules.get(curname)
166 if module is None:
167 mp_file, mp_filename, mp_desc = find_module(part, path)
168 module = load_module(curname, mp_file, mp_filename, mp_desc)
169 if prevmodule:
170 setattr(prevmodule, part, module)
171 _file = getattr(module, '__file__', '')
172 if not _file and len(modpath) != len(parts):
173 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
174 path = [dirname( _file )]
175 prevmodule = module
176 return module
177
178
180 """Load a Python module from it's path.
181
182 :type filepath: str
183 :param filepath: path to the python module or package
184
185 :type path: list or None
186 :param path:
187 optional list of path where the module or package should be
188 searched (use sys.path if nothing or None is given)
189
190 :type use_sys: bool
191 :param use_sys:
192 boolean indicating whether the sys.modules dictionary should be
193 used or not
194
195
196 :raise ImportError: if the module or package is not found
197
198 :rtype: module
199 :return: the loaded module
200 """
201 modpath = modpath_from_file(filepath, extrapath)
202 return load_module_from_modpath(modpath, path, use_sys)
203
204
206 """check there are some __init__.py all along the way"""
207 for part in mod_path:
208 path = join(path, part)
209 if not _has_init(path):
210 return False
211 return True
212
213
215 """given a file path return the corresponding splitted module's name
216 (i.e name of a module or package splitted on '.')
217
218 :type filename: str
219 :param filename: file's path for which we want the module's name
220
221 :type extrapath: dict
222 :param extrapath:
223 optional extra search path, with path as key and package name for the path
224 as value. This is usually useful to handle package splitted in multiple
225 directories using __path__ trick.
226
227
228 :raise ImportError:
229 if the corresponding module's name has not been found
230
231 :rtype: list(str)
232 :return: the corresponding splitted module's name
233 """
234 base = splitext(abspath(filename))[0]
235 if extrapath is not None:
236 for path_ in extrapath:
237 path = abspath(path_)
238 if path and base[:len(path)] == path:
239 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
240 if pkg]
241 if _check_init(path, submodpath[:-1]):
242 return extrapath[path_].split('.') + submodpath
243 for path in sys.path:
244 path = abspath(path)
245 if path and base.startswith(path):
246 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
247 if _check_init(path, modpath[:-1]):
248 return modpath
249 raise ImportError('Unable to find module for %s in %s' % (
250 filename, ', \n'.join(sys.path)))
251
252
253
255 """given a mod path (i.e. splitted module / package name), return the
256 corresponding file, giving priority to source file over precompiled
257 file if it exists
258
259 :type modpath: list or tuple
260 :param modpath:
261 splitted module's name (i.e name of a module or package splitted
262 on '.')
263 (this means explicit relative imports that start with dots have
264 empty strings in this list!)
265
266 :type path: list or None
267 :param path:
268 optional list of path where the module or package should be
269 searched (use sys.path if nothing or None is given)
270
271 :type context_file: str or None
272 :param context_file:
273 context file to consider, necessary if the identifier has been
274 introduced using a relative import unresolvable in the actual
275 context (i.e. modutils)
276
277 :raise ImportError: if there is no such module in the directory
278
279 :rtype: str or None
280 :return:
281 the path to the module's file or None if it's an integrated
282 builtin module such as 'sys'
283 """
284 if context_file is not None:
285 context = dirname(context_file)
286 else:
287 context = context_file
288 if modpath[0] == 'xml':
289
290 try:
291 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
292 except ImportError:
293 return _file_from_modpath(modpath, path, context)
294 elif modpath == ['os', 'path']:
295
296 return os.path.__file__
297 return _file_from_modpath(modpath, path, context)
298
299
300
302 """given a dotted name return the module part of the name :
303
304 >>> get_module_part('logilab.common.modutils.get_module_part')
305 'logilab.common.modutils'
306
307 :type dotted_name: str
308 :param dotted_name: full name of the identifier we are interested in
309
310 :type context_file: str or None
311 :param context_file:
312 context file to consider, necessary if the identifier has been
313 introduced using a relative import unresolvable in the actual
314 context (i.e. modutils)
315
316
317 :raise ImportError: if there is no such module in the directory
318
319 :rtype: str or None
320 :return:
321 the module part of the name or None if we have not been able at
322 all to import the given name
323
324 XXX: deprecated, since it doesn't handle package precedence over module
325 (see #10066)
326 """
327
328 if dotted_name.startswith('os.path'):
329 return 'os.path'
330 parts = dotted_name.split('.')
331 if context_file is not None:
332
333
334 if parts[0] in BUILTIN_MODULES:
335 if len(parts) > 2:
336 raise ImportError(dotted_name)
337 return parts[0]
338
339 path = None
340 starti = 0
341 if parts[0] == '':
342 assert context_file is not None, \
343 'explicit relative import, but no context_file?'
344 path = []
345 starti = 1
346 while parts[starti] == '':
347 starti += 1
348 context_file = dirname(context_file)
349 for i in range(starti, len(parts)):
350 try:
351 file_from_modpath(parts[starti:i+1],
352 path=path, context_file=context_file)
353 except ImportError:
354 if not i >= max(1, len(parts) - 2):
355 raise
356 return '.'.join(parts[:i])
357 return dotted_name
358
359
361 """given a package directory return a list of all available python
362 modules in the package and its subpackages
363
364 :type package: str
365 :param package: the python name for the package
366
367 :type src_directory: str
368 :param src_directory:
369 path of the directory corresponding to the package
370
371 :type blacklist: list or tuple
372 :param blacklist:
373 optional list of files or directory to ignore, default to
374 the value of `logilab.common.STD_BLACKLIST`
375
376 :rtype: list
377 :return:
378 the list of all available python modules in the package and its
379 subpackages
380 """
381 modules = []
382 for directory, dirnames, filenames in os.walk(src_directory):
383 _handle_blacklist(blacklist, dirnames, filenames)
384
385 if not '__init__.py' in filenames:
386 dirnames[:] = ()
387 continue
388 if directory != src_directory:
389 dir_package = directory[len(src_directory):].replace(os.sep, '.')
390 modules.append(package + dir_package)
391 for filename in filenames:
392 if _is_python_file(filename) and filename != '__init__.py':
393 src = join(directory, filename)
394 module = package + src[len(src_directory):-3]
395 modules.append(module.replace(os.sep, '.'))
396 return modules
397
398
399
401 """given a package directory return a list of all available python
402 module's files in the package and its subpackages
403
404 :type src_directory: str
405 :param src_directory:
406 path of the directory corresponding to the package
407
408 :type blacklist: list or tuple
409 :param blacklist:
410 optional list of files or directory to ignore, default to the value of
411 `logilab.common.STD_BLACKLIST`
412
413 :rtype: list
414 :return:
415 the list of all available python module's files in the package and
416 its subpackages
417 """
418 files = []
419 for directory, dirnames, filenames in os.walk(src_directory):
420 _handle_blacklist(blacklist, dirnames, filenames)
421
422 if not '__init__.py' in filenames:
423 dirnames[:] = ()
424 continue
425 for filename in filenames:
426 if _is_python_file(filename):
427 src = join(directory, filename)
428 files.append(src)
429 return files
430
431
433 """given a python module's file name return the matching source file
434 name (the filename will be returned identically if it's a already an
435 absolute path to a python source file...)
436
437 :type filename: str
438 :param filename: python module's file name
439
440
441 :raise NoSourceFile: if no source file exists on the file system
442
443 :rtype: str
444 :return: the absolute path of the source file if it exists
445 """
446 base, orig_ext = splitext(abspath(filename))
447 for ext in PY_SOURCE_EXTS:
448 source_path = '%s.%s' % (base, ext)
449 if exists(source_path):
450 return source_path
451 if include_no_ext and not orig_ext and exists(base):
452 return base
453 raise NoSourceFile(filename)
454
455
457 """remove submodules of `directories` from `sys.modules`"""
458 for modname, module in list(sys.modules.items()):
459 modfile = getattr(module, '__file__', None)
460 if modfile:
461 for directory in directories:
462 if modfile.startswith(directory):
463 del sys.modules[modname]
464 break
465
466
468 """
469 rtype: bool
470 return: True if the filename is a python source file
471 """
472 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
473
474
475
477 """try to guess if a module is a standard python module (by default,
478 see `std_path` parameter's description)
479
480 :type modname: str
481 :param modname: name of the module we are interested in
482
483 :type std_path: list(str) or tuple(str)
484 :param std_path: list of path considered has standard
485
486
487 :rtype: bool
488 :return:
489 true if the module:
490 - is located on the path listed in one of the directory in `std_path`
491 - is a built-in module
492 """
493 modname = modname.split('.')[0]
494 try:
495 filename = file_from_modpath([modname])
496 except ImportError as ex:
497
498
499 return 0
500
501
502 if filename is None:
503 return 1
504 filename = abspath(filename)
505 if filename.startswith(EXT_LIB_DIR):
506 return 0
507 for path in std_path:
508 if filename.startswith(abspath(path)):
509 return 1
510 return False
511
512
513
515 """return true if the given module name is relative to the given
516 file name
517
518 :type modname: str
519 :param modname: name of the module we are interested in
520
521 :type from_file: str
522 :param from_file:
523 path of the module from which modname has been imported
524
525 :rtype: bool
526 :return:
527 true if the module has been imported relatively to `from_file`
528 """
529 if not isdir(from_file):
530 from_file = dirname(from_file)
531 if from_file in sys.path:
532 return False
533 try:
534 find_module(modname.split('.')[0], [from_file])
535 return True
536 except ImportError:
537 return False
538
539
540
541
543 """given a mod path (i.e. splitted module / package name), return the
544 corresponding file
545
546 this function is used internally, see `file_from_modpath`'s
547 documentation for more information
548 """
549 assert len(modpath) > 0
550 if context is not None:
551 try:
552 mtype, mp_filename = _module_file(modpath, [context])
553 except ImportError:
554 mtype, mp_filename = _module_file(modpath, path)
555 else:
556 mtype, mp_filename = _module_file(modpath, path)
557 if mtype == PY_COMPILED:
558 try:
559 return get_source_file(mp_filename)
560 except NoSourceFile:
561 return mp_filename
562 elif mtype == C_BUILTIN:
563
564 return None
565 elif mtype == PKG_DIRECTORY:
566 mp_filename = _has_init(mp_filename)
567 return mp_filename
568
570 for filepath, importer in list(pic.items()):
571 if importer is not None:
572 if importer.find_module(modpath[0]):
573 if not importer.find_module('/'.join(modpath)):
574 raise ImportError('No module named %s in %s/%s' % (
575 '.'.join(modpath[1:]), filepath, modpath))
576 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
577 raise ImportError('No module named %s' % '.'.join(modpath))
578
579 try:
580 import pkg_resources
581 except ImportError:
582 pkg_resources = None
583
585 """get a module type / file path
586
587 :type modpath: list or tuple
588 :param modpath:
589 splitted module's name (i.e name of a module or package splitted
590 on '.'), with leading empty strings for explicit relative import
591
592 :type path: list or None
593 :param path:
594 optional list of path where the module or package should be
595 searched (use sys.path if nothing or None is given)
596
597
598 :rtype: tuple(int, str)
599 :return: the module type flag and the file path for a module
600 """
601
602 try:
603 pic = sys.path_importer_cache
604 _path = (path is None and sys.path or path)
605 for __path in _path:
606 if not __path in pic:
607 try:
608 pic[__path] = zipimport.zipimporter(__path)
609 except zipimport.ZipImportError:
610 pic[__path] = None
611 checkeggs = True
612 except AttributeError:
613 checkeggs = False
614
615 if pkg_resources is not None and modpath[0] in pkg_resources._namespace_packages and len(modpath) > 1:
616
617
618 module = sys.modules[modpath.pop(0)]
619 path = module.__path__
620 imported = []
621 while modpath:
622 modname = modpath[0]
623
624
625
626
627
628
629
630
631
632 try:
633 _, mp_filename, mp_desc = find_module(modname, path)
634 except ImportError:
635 if checkeggs:
636 return _search_zip(modpath, pic)[:2]
637 raise
638 else:
639 if checkeggs and mp_filename:
640 fullabspath = [abspath(x) for x in _path]
641 try:
642 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
643 emtype, emp_filename, zippath = _search_zip(modpath, pic)
644 if pathindex > _path.index(zippath):
645
646 return emtype, emp_filename
647 except ValueError:
648
649 pass
650 except ImportError:
651 pass
652 checkeggs = False
653 imported.append(modpath.pop(0))
654 mtype = mp_desc[2]
655 if modpath:
656 if mtype != PKG_DIRECTORY:
657 raise ImportError('No module %s in %s' % ('.'.join(modpath),
658 '.'.join(imported)))
659
660
661 try:
662 with open(join(mp_filename, '__init__.py')) as stream:
663 data = stream.read(4096)
664 except IOError:
665 path = [mp_filename]
666 else:
667 if 'pkgutil' in data and 'extend_path' in data:
668
669
670 path = [join(p, *imported) for p in sys.path
671 if isdir(join(p, *imported))]
672 else:
673 path = [mp_filename]
674 return mtype, mp_filename
675
677 """return true if the given filename should be considered as a python file
678
679 .pyc and .pyo are ignored
680 """
681 for ext in ('.py', '.so', '.pyd', '.pyw'):
682 if filename.endswith(ext):
683 return True
684 return False
685
686
688 """if the given directory has a valid __init__ file, return its path,
689 else return None
690 """
691 mod_or_pack = join(directory, '__init__')
692 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
693 if exists(mod_or_pack + '.' + ext):
694 return mod_or_pack + '.' + ext
695 return None
696