1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21 :type PY_SOURCE_EXTS: tuple(str)
22 :var PY_SOURCE_EXTS: list of possible python source file extension
23
24 :type STD_LIB_DIR: str
25 :var STD_LIB_DIR: directory where standard modules are located
26
27 :type BUILTIN_MODULES: dict
28 :var BUILTIN_MODULES: dictionary with builtin module names as key
29 """
30
31 __docformat__ = "restructuredtext en"
32
33 import sys
34 import os
35 from os.path import splitext, join, abspath, isdir, dirname, exists, basename
36 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
37 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
38 from distutils.errors import DistutilsPlatformError
39
40 from six.moves import range
41
42 try:
43 import zipimport
44 except ImportError:
45 zipimport = None
46
47 ZIPFILE = object()
48
49 from logilab.common import STD_BLACKLIST, _handle_blacklist
50
51
52
53
54
55
56
57 if sys.platform.startswith('win'):
58 PY_SOURCE_EXTS = ('py', 'pyw')
59 PY_COMPILED_EXTS = ('dll', 'pyd')
60 else:
61 PY_SOURCE_EXTS = ('py',)
62 PY_COMPILED_EXTS = ('so',)
63
64 try:
65 STD_LIB_DIR = get_python_lib(standard_lib=True)
66
67
68 except DistutilsPlatformError:
69 STD_LIB_DIR = '//'
70
71 EXT_LIB_DIR = get_python_lib()
72
73 BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
74
75
77 """exception raised when we are not able to get a python
78 source file for a precompiled file
79 """
80
83 self.module = module
84 self.obj = obj
85 self._imported = None
86
88 if self._imported is None:
89 self._imported = getattr(load_module_from_name(self.module),
90 self.obj)
91 return self._imported
92
94 try:
95 return super(LazyObject, self).__getattribute__(attr)
96 except AttributeError as ex:
97 return getattr(self._getobj(), attr)
98
100 return self._getobj()(*args, **kwargs)
101
102
104 """Load a Python module from its name.
105
106 :type dotted_name: str
107 :param dotted_name: python name of a module or package
108
109 :type path: list or None
110 :param path:
111 optional list of path where the module or package should be
112 searched (use sys.path if nothing or None is given)
113
114 :type use_sys: bool
115 :param use_sys:
116 boolean indicating whether the sys.modules dictionary should be
117 used or not
118
119
120 :raise ImportError: if the module or package is not found
121
122 :rtype: module
123 :return: the loaded module
124 """
125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126
127
129 """Load a python module from its splitted name.
130
131 :type parts: list(str) or tuple(str)
132 :param parts:
133 python name of a module or package splitted on '.'
134
135 :type path: list or None
136 :param path:
137 optional list of path where the module or package should be
138 searched (use sys.path if nothing or None is given)
139
140 :type use_sys: bool
141 :param use_sys:
142 boolean indicating whether the sys.modules dictionary should be used or not
143
144 :raise ImportError: if the module or package is not found
145
146 :rtype: module
147 :return: the loaded module
148 """
149 if use_sys:
150 try:
151 return sys.modules['.'.join(parts)]
152 except KeyError:
153 pass
154 modpath = []
155 prevmodule = None
156 for part in parts:
157 modpath.append(part)
158 curname = '.'.join(modpath)
159 module = None
160 if len(modpath) != len(parts):
161
162 module = sys.modules.get(curname)
163 elif use_sys:
164
165 module = sys.modules.get(curname)
166 if module is None:
167 mp_file, mp_filename, mp_desc = find_module(part, path)
168 try:
169 module = load_module(curname, mp_file, mp_filename, mp_desc)
170 finally:
171 if mp_file is not None:
172 mp_file.close()
173 if prevmodule:
174 setattr(prevmodule, part, module)
175 _file = getattr(module, '__file__', '')
176 prevmodule = module
177 if not _file and _is_namespace(curname):
178 continue
179 if not _file and len(modpath) != len(parts):
180 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
181 path = [dirname( _file )]
182 return module
183
184
186 """Load a Python module from it's path.
187
188 :type filepath: str
189 :param filepath: path to the python module or package
190
191 :type path: list or None
192 :param path:
193 optional list of path where the module or package should be
194 searched (use sys.path if nothing or None is given)
195
196 :type use_sys: bool
197 :param use_sys:
198 boolean indicating whether the sys.modules dictionary should be
199 used or not
200
201
202 :raise ImportError: if the module or package is not found
203
204 :rtype: module
205 :return: the loaded module
206 """
207 modpath = modpath_from_file(filepath, extrapath)
208 return load_module_from_modpath(modpath, path, use_sys)
209
210
212 """check there are some __init__.py all along the way"""
213 modpath = []
214 for part in mod_path:
215 modpath.append(part)
216 path = join(path, part)
217 if not _is_namespace('.'.join(modpath)) and not _has_init(path):
218 return False
219 return True
220
221
223 """given a file path return the corresponding splitted module's name
224 (i.e name of a module or package splitted on '.')
225
226 :type filename: str
227 :param filename: file's path for which we want the module's name
228
229 :type extrapath: dict
230 :param extrapath:
231 optional extra search path, with path as key and package name for the path
232 as value. This is usually useful to handle package splitted in multiple
233 directories using __path__ trick.
234
235
236 :raise ImportError:
237 if the corresponding module's name has not been found
238
239 :rtype: list(str)
240 :return: the corresponding splitted module's name
241 """
242 base = splitext(abspath(filename))[0]
243 if extrapath is not None:
244 for path_ in extrapath:
245 path = abspath(path_)
246 if path and base[:len(path)] == path:
247 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
248 if pkg]
249 if _check_init(path, submodpath[:-1]):
250 return extrapath[path_].split('.') + submodpath
251 for path in sys.path:
252 path = abspath(path)
253 if path and base.startswith(path):
254 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
255 if _check_init(path, modpath[:-1]):
256 return modpath
257 raise ImportError('Unable to find module for %s in %s' % (
258 filename, ', \n'.join(sys.path)))
259
260
261
263 """given a mod path (i.e. splitted module / package name), return the
264 corresponding file, giving priority to source file over precompiled
265 file if it exists
266
267 :type modpath: list or tuple
268 :param modpath:
269 splitted module's name (i.e name of a module or package splitted
270 on '.')
271 (this means explicit relative imports that start with dots have
272 empty strings in this list!)
273
274 :type path: list or None
275 :param path:
276 optional list of path where the module or package should be
277 searched (use sys.path if nothing or None is given)
278
279 :type context_file: str or None
280 :param context_file:
281 context file to consider, necessary if the identifier has been
282 introduced using a relative import unresolvable in the actual
283 context (i.e. modutils)
284
285 :raise ImportError: if there is no such module in the directory
286
287 :rtype: str or None
288 :return:
289 the path to the module's file or None if it's an integrated
290 builtin module such as 'sys'
291 """
292 if context_file is not None:
293 context = dirname(context_file)
294 else:
295 context = context_file
296 if modpath[0] == 'xml':
297
298 try:
299 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
300 except ImportError:
301 return _file_from_modpath(modpath, path, context)
302 elif modpath == ['os', 'path']:
303
304 return os.path.__file__
305 return _file_from_modpath(modpath, path, context)
306
307
308
310 """given a dotted name return the module part of the name :
311
312 >>> get_module_part('logilab.common.modutils.get_module_part')
313 'logilab.common.modutils'
314
315 :type dotted_name: str
316 :param dotted_name: full name of the identifier we are interested in
317
318 :type context_file: str or None
319 :param context_file:
320 context file to consider, necessary if the identifier has been
321 introduced using a relative import unresolvable in the actual
322 context (i.e. modutils)
323
324
325 :raise ImportError: if there is no such module in the directory
326
327 :rtype: str or None
328 :return:
329 the module part of the name or None if we have not been able at
330 all to import the given name
331
332 XXX: deprecated, since it doesn't handle package precedence over module
333 (see #10066)
334 """
335
336 if dotted_name.startswith('os.path'):
337 return 'os.path'
338 parts = dotted_name.split('.')
339 if context_file is not None:
340
341
342 if parts[0] in BUILTIN_MODULES:
343 if len(parts) > 2:
344 raise ImportError(dotted_name)
345 return parts[0]
346
347 path = None
348 starti = 0
349 if parts[0] == '':
350 assert context_file is not None, \
351 'explicit relative import, but no context_file?'
352 path = []
353 starti = 1
354 while parts[starti] == '':
355 starti += 1
356 context_file = dirname(context_file)
357 for i in range(starti, len(parts)):
358 try:
359 file_from_modpath(parts[starti:i+1],
360 path=path, context_file=context_file)
361 except ImportError:
362 if not i >= max(1, len(parts) - 2):
363 raise
364 return '.'.join(parts[:i])
365 return dotted_name
366
367
369 """given a package directory return a list of all available python
370 modules in the package and its subpackages
371
372 :type package: str
373 :param package: the python name for the package
374
375 :type src_directory: str
376 :param src_directory:
377 path of the directory corresponding to the package
378
379 :type blacklist: list or tuple
380 :param blacklist:
381 optional list of files or directory to ignore, default to
382 the value of `logilab.common.STD_BLACKLIST`
383
384 :rtype: list
385 :return:
386 the list of all available python modules in the package and its
387 subpackages
388 """
389 modules = []
390 for directory, dirnames, filenames in os.walk(src_directory):
391 _handle_blacklist(blacklist, dirnames, filenames)
392
393 if not '__init__.py' in filenames:
394 dirnames[:] = ()
395 continue
396 if directory != src_directory:
397 dir_package = directory[len(src_directory):].replace(os.sep, '.')
398 modules.append(package + dir_package)
399 for filename in filenames:
400 if _is_python_file(filename) and filename != '__init__.py':
401 src = join(directory, filename)
402 module = package + src[len(src_directory):-3]
403 modules.append(module.replace(os.sep, '.'))
404 return modules
405
406
407
409 """given a package directory return a list of all available python
410 module's files in the package and its subpackages
411
412 :type src_directory: str
413 :param src_directory:
414 path of the directory corresponding to the package
415
416 :type blacklist: list or tuple
417 :param blacklist:
418 optional list of files or directory to ignore, default to the value of
419 `logilab.common.STD_BLACKLIST`
420
421 :rtype: list
422 :return:
423 the list of all available python module's files in the package and
424 its subpackages
425 """
426 files = []
427 for directory, dirnames, filenames in os.walk(src_directory):
428 _handle_blacklist(blacklist, dirnames, filenames)
429
430 if not '__init__.py' in filenames:
431 dirnames[:] = ()
432 continue
433 for filename in filenames:
434 if _is_python_file(filename):
435 src = join(directory, filename)
436 files.append(src)
437 return files
438
439
441 """given a python module's file name return the matching source file
442 name (the filename will be returned identically if it's a already an
443 absolute path to a python source file...)
444
445 :type filename: str
446 :param filename: python module's file name
447
448
449 :raise NoSourceFile: if no source file exists on the file system
450
451 :rtype: str
452 :return: the absolute path of the source file if it exists
453 """
454 base, orig_ext = splitext(abspath(filename))
455 for ext in PY_SOURCE_EXTS:
456 source_path = '%s.%s' % (base, ext)
457 if exists(source_path):
458 return source_path
459 if include_no_ext and not orig_ext and exists(base):
460 return base
461 raise NoSourceFile(filename)
462
463
465 """remove submodules of `directories` from `sys.modules`"""
466 cleaned = []
467 for modname, module in list(sys.modules.items()):
468 modfile = getattr(module, '__file__', None)
469 if modfile:
470 for directory in directories:
471 if modfile.startswith(directory):
472 cleaned.append(modname)
473 del sys.modules[modname]
474 break
475 return cleaned
476
477
479 """
480 rtype: bool
481 return: True if the filename is a python source file
482 """
483 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
484
485
487 """try to guess if a module is a standard python module (by default,
488 see `std_path` parameter's description)
489
490 :type modname: str
491 :param modname: name of the module we are interested in
492
493 :type std_path: list(str) or tuple(str)
494 :param std_path: list of path considered as standard
495
496
497 :rtype: bool
498 :return:
499 true if the module:
500 - is located on the path listed in one of the directory in `std_path`
501 - is a built-in module
502
503 Note: this function is known to return wrong values when inside virtualenv.
504 See https://www.logilab.org/ticket/294756.
505 """
506 modname = modname.split('.')[0]
507 try:
508 filename = file_from_modpath([modname])
509 except ImportError as ex:
510
511
512 return False
513
514
515 if filename is None:
516
517 return not _is_namespace(modname)
518 filename = abspath(filename)
519 if filename.startswith(EXT_LIB_DIR):
520 return False
521 for path in std_path:
522 if filename.startswith(abspath(path)):
523 return True
524 return False
525
526
527
529 """return true if the given module name is relative to the given
530 file name
531
532 :type modname: str
533 :param modname: name of the module we are interested in
534
535 :type from_file: str
536 :param from_file:
537 path of the module from which modname has been imported
538
539 :rtype: bool
540 :return:
541 true if the module has been imported relatively to `from_file`
542 """
543 if not isdir(from_file):
544 from_file = dirname(from_file)
545 if from_file in sys.path:
546 return False
547 try:
548 find_module(modname.split('.')[0], [from_file])
549 return True
550 except ImportError:
551 return False
552
553
554
555
557 """given a mod path (i.e. splitted module / package name), return the
558 corresponding file
559
560 this function is used internally, see `file_from_modpath`'s
561 documentation for more information
562 """
563 assert len(modpath) > 0
564 if context is not None:
565 try:
566 mtype, mp_filename = _module_file(modpath, [context])
567 except ImportError:
568 mtype, mp_filename = _module_file(modpath, path)
569 else:
570 mtype, mp_filename = _module_file(modpath, path)
571 if mtype == PY_COMPILED:
572 try:
573 return get_source_file(mp_filename)
574 except NoSourceFile:
575 return mp_filename
576 elif mtype == C_BUILTIN:
577
578 return None
579 elif mtype == PKG_DIRECTORY:
580 mp_filename = _has_init(mp_filename)
581 return mp_filename
582
584 for filepath, importer in pic.items():
585 if importer is not None:
586 if importer.find_module(modpath[0]):
587 if not importer.find_module('/'.join(modpath)):
588 raise ImportError('No module named %s in %s/%s' % (
589 '.'.join(modpath[1:]), filepath, modpath))
590 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
591 raise ImportError('No module named %s' % '.'.join(modpath))
592
593 try:
594 import pkg_resources
595 except ImportError:
596 pkg_resources = None
597
598
602
603
605 """get a module type / file path
606
607 :type modpath: list or tuple
608 :param modpath:
609 splitted module's name (i.e name of a module or package splitted
610 on '.'), with leading empty strings for explicit relative import
611
612 :type path: list or None
613 :param path:
614 optional list of path where the module or package should be
615 searched (use sys.path if nothing or None is given)
616
617
618 :rtype: tuple(int, str)
619 :return: the module type flag and the file path for a module
620 """
621
622 try:
623 pic = sys.path_importer_cache
624 _path = (path is None and sys.path or path)
625 for __path in _path:
626 if not __path in pic:
627 try:
628 pic[__path] = zipimport.zipimporter(__path)
629 except zipimport.ZipImportError:
630 pic[__path] = None
631 checkeggs = True
632 except AttributeError:
633 checkeggs = False
634
635 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules):
636
637
638 module = sys.modules[modpath.pop(0)]
639 path = module.__path__
640 if not modpath:
641 return C_BUILTIN, None
642 imported = []
643 while modpath:
644 modname = modpath[0]
645
646
647
648
649
650
651
652
653
654 try:
655 _, mp_filename, mp_desc = find_module(modname, path)
656 except ImportError:
657 if checkeggs:
658 return _search_zip(modpath, pic)[:2]
659 raise
660 else:
661 if checkeggs and mp_filename:
662 fullabspath = [abspath(x) for x in _path]
663 try:
664 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
665 emtype, emp_filename, zippath = _search_zip(modpath, pic)
666 if pathindex > _path.index(zippath):
667
668 return emtype, emp_filename
669 except ValueError:
670
671 pass
672 except ImportError:
673 pass
674 checkeggs = False
675 imported.append(modpath.pop(0))
676 mtype = mp_desc[2]
677 if modpath:
678 if mtype != PKG_DIRECTORY:
679 raise ImportError('No module %s in %s' % ('.'.join(modpath),
680 '.'.join(imported)))
681
682
683 try:
684 with open(join(mp_filename, '__init__.py')) as stream:
685 data = stream.read(4096)
686 except IOError:
687 path = [mp_filename]
688 else:
689 if 'pkgutil' in data and 'extend_path' in data:
690
691
692 path = [join(p, *imported) for p in sys.path
693 if isdir(join(p, *imported))]
694 else:
695 path = [mp_filename]
696 return mtype, mp_filename
697
699 """return true if the given filename should be considered as a python file
700
701 .pyc and .pyo are ignored
702 """
703 for ext in ('.py', '.so', '.pyd', '.pyw'):
704 if filename.endswith(ext):
705 return True
706 return False
707
708
710 """if the given directory has a valid __init__ file, return its path,
711 else return None
712 """
713 mod_or_pack = join(directory, '__init__')
714 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
715 if exists(mod_or_pack + '.' + ext):
716 return mod_or_pack + '.' + ext
717 return None
718