1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21 :type PY_SOURCE_EXTS: tuple(str)
22 :var PY_SOURCE_EXTS: list of possible python source file extension
23
24 :type STD_LIB_DIR: str
25 :var STD_LIB_DIR: directory where standard modules are located
26
27 :type BUILTIN_MODULES: dict
28 :var BUILTIN_MODULES: dictionary with builtin module names has key
29 """
30 __docformat__ = "restructuredtext en"
31
32 import sys
33 import os
34 from os.path import splitext, join, abspath, isdir, dirname, exists, basename
35 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
36 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
37 from distutils.errors import DistutilsPlatformError
38
39 try:
40 import zipimport
41 except ImportError:
42 zipimport = None
43
44 ZIPFILE = object()
45
46 from logilab.common import STD_BLACKLIST, _handle_blacklist
47
48
49
50
51
52
53
54 if sys.platform.startswith('win'):
55 PY_SOURCE_EXTS = ('py', 'pyw')
56 PY_COMPILED_EXTS = ('dll', 'pyd')
57 else:
58 PY_SOURCE_EXTS = ('py',)
59 PY_COMPILED_EXTS = ('so',)
60
61 try:
62 STD_LIB_DIR = get_python_lib(standard_lib=1)
63
64
65 except DistutilsPlatformError:
66 STD_LIB_DIR = '//'
67
68 EXT_LIB_DIR = get_python_lib()
69
70 BUILTIN_MODULES = dict(list(zip(sys.builtin_module_names,
71 [1]*len(sys.builtin_module_names))))
72
73
75 """exception raised when we are not able to get a python
76 source file for a precompiled file
77 """
78
81 self.module = module
82 self.obj = obj
83 self._imported = None
84
86 if self._imported is None:
87 self._imported = getattr(load_module_from_name(self.module),
88 self.obj)
89 return self._imported
90
92 try:
93 return super(LazyObject, self).__getattribute__(attr)
94 except AttributeError as ex:
95 return getattr(self._getobj(), attr)
96
98 return self._getobj()(*args, **kwargs)
99
100
102 """Load a Python module from its name.
103
104 :type dotted_name: str
105 :param dotted_name: python name of a module or package
106
107 :type path: list or None
108 :param path:
109 optional list of path where the module or package should be
110 searched (use sys.path if nothing or None is given)
111
112 :type use_sys: bool
113 :param use_sys:
114 boolean indicating whether the sys.modules dictionary should be
115 used or not
116
117
118 :raise ImportError: if the module or package is not found
119
120 :rtype: module
121 :return: the loaded module
122 """
123 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
124
125
127 """Load a python module from its splitted name.
128
129 :type parts: list(str) or tuple(str)
130 :param parts:
131 python name of a module or package splitted on '.'
132
133 :type path: list or None
134 :param path:
135 optional list of path where the module or package should be
136 searched (use sys.path if nothing or None is given)
137
138 :type use_sys: bool
139 :param use_sys:
140 boolean indicating whether the sys.modules dictionary should be used or not
141
142 :raise ImportError: if the module or package is not found
143
144 :rtype: module
145 :return: the loaded module
146 """
147 if use_sys:
148 try:
149 return sys.modules['.'.join(parts)]
150 except KeyError:
151 pass
152 modpath = []
153 prevmodule = None
154 for part in parts:
155 modpath.append(part)
156 curname = '.'.join(modpath)
157 module = None
158 if len(modpath) != len(parts):
159
160 module = sys.modules.get(curname)
161 elif use_sys:
162
163 module = sys.modules.get(curname)
164 if module is None:
165 mp_file, mp_filename, mp_desc = find_module(part, path)
166 module = load_module(curname, mp_file, mp_filename, mp_desc)
167 if prevmodule:
168 setattr(prevmodule, part, module)
169 _file = getattr(module, '__file__', '')
170 if not _file and len(modpath) != len(parts):
171 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
172 path = [dirname( _file )]
173 prevmodule = module
174 return module
175
176
178 """Load a Python module from it's path.
179
180 :type filepath: str
181 :param filepath: path to the python module or package
182
183 :type path: list or None
184 :param path:
185 optional list of path where the module or package should be
186 searched (use sys.path if nothing or None is given)
187
188 :type use_sys: bool
189 :param use_sys:
190 boolean indicating whether the sys.modules dictionary should be
191 used or not
192
193
194 :raise ImportError: if the module or package is not found
195
196 :rtype: module
197 :return: the loaded module
198 """
199 modpath = modpath_from_file(filepath, extrapath)
200 return load_module_from_modpath(modpath, path, use_sys)
201
202
204 """check there are some __init__.py all along the way"""
205 for part in mod_path:
206 path = join(path, part)
207 if not _has_init(path):
208 return False
209 return True
210
211
213 """given a file path return the corresponding splitted module's name
214 (i.e name of a module or package splitted on '.')
215
216 :type filename: str
217 :param filename: file's path for which we want the module's name
218
219 :type extrapath: dict
220 :param extrapath:
221 optional extra search path, with path as key and package name for the path
222 as value. This is usually useful to handle package splitted in multiple
223 directories using __path__ trick.
224
225
226 :raise ImportError:
227 if the corresponding module's name has not been found
228
229 :rtype: list(str)
230 :return: the corresponding splitted module's name
231 """
232 base = splitext(abspath(filename))[0]
233 if extrapath is not None:
234 for path_ in extrapath:
235 path = abspath(path_)
236 if path and base[:len(path)] == path:
237 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
238 if pkg]
239 if _check_init(path, submodpath[:-1]):
240 return extrapath[path_].split('.') + submodpath
241 for path in sys.path:
242 path = abspath(path)
243 if path and base.startswith(path):
244 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
245 if _check_init(path, modpath[:-1]):
246 return modpath
247 raise ImportError('Unable to find module for %s in %s' % (
248 filename, ', \n'.join(sys.path)))
249
250
251
253 """given a mod path (i.e. splitted module / package name), return the
254 corresponding file, giving priority to source file over precompiled
255 file if it exists
256
257 :type modpath: list or tuple
258 :param modpath:
259 splitted module's name (i.e name of a module or package splitted
260 on '.')
261 (this means explicit relative imports that start with dots have
262 empty strings in this list!)
263
264 :type path: list or None
265 :param path:
266 optional list of path where the module or package should be
267 searched (use sys.path if nothing or None is given)
268
269 :type context_file: str or None
270 :param context_file:
271 context file to consider, necessary if the identifier has been
272 introduced using a relative import unresolvable in the actual
273 context (i.e. modutils)
274
275 :raise ImportError: if there is no such module in the directory
276
277 :rtype: str or None
278 :return:
279 the path to the module's file or None if it's an integrated
280 builtin module such as 'sys'
281 """
282 if context_file is not None:
283 context = dirname(context_file)
284 else:
285 context = context_file
286 if modpath[0] == 'xml':
287
288 try:
289 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
290 except ImportError:
291 return _file_from_modpath(modpath, path, context)
292 elif modpath == ['os', 'path']:
293
294 return os.path.__file__
295 return _file_from_modpath(modpath, path, context)
296
297
298
300 """given a dotted name return the module part of the name :
301
302 >>> get_module_part('logilab.common.modutils.get_module_part')
303 'logilab.common.modutils'
304
305 :type dotted_name: str
306 :param dotted_name: full name of the identifier we are interested in
307
308 :type context_file: str or None
309 :param context_file:
310 context file to consider, necessary if the identifier has been
311 introduced using a relative import unresolvable in the actual
312 context (i.e. modutils)
313
314
315 :raise ImportError: if there is no such module in the directory
316
317 :rtype: str or None
318 :return:
319 the module part of the name or None if we have not been able at
320 all to import the given name
321
322 XXX: deprecated, since it doesn't handle package precedence over module
323 (see #10066)
324 """
325
326 if dotted_name.startswith('os.path'):
327 return 'os.path'
328 parts = dotted_name.split('.')
329 if context_file is not None:
330
331
332 if parts[0] in BUILTIN_MODULES:
333 if len(parts) > 2:
334 raise ImportError(dotted_name)
335 return parts[0]
336
337 path = None
338 starti = 0
339 if parts[0] == '':
340 assert context_file is not None, \
341 'explicit relative import, but no context_file?'
342 path = []
343 starti = 1
344 while parts[starti] == '':
345 starti += 1
346 context_file = dirname(context_file)
347 for i in range(starti, len(parts)):
348 try:
349 file_from_modpath(parts[starti:i+1],
350 path=path, context_file=context_file)
351 except ImportError:
352 if not i >= max(1, len(parts) - 2):
353 raise
354 return '.'.join(parts[:i])
355 return dotted_name
356
357
359 """given a package directory return a list of all available python
360 modules in the package and its subpackages
361
362 :type package: str
363 :param package: the python name for the package
364
365 :type src_directory: str
366 :param src_directory:
367 path of the directory corresponding to the package
368
369 :type blacklist: list or tuple
370 :param blacklist:
371 optional list of files or directory to ignore, default to
372 the value of `logilab.common.STD_BLACKLIST`
373
374 :rtype: list
375 :return:
376 the list of all available python modules in the package and its
377 subpackages
378 """
379 modules = []
380 for directory, dirnames, filenames in os.walk(src_directory):
381 _handle_blacklist(blacklist, dirnames, filenames)
382
383 if not '__init__.py' in filenames:
384 dirnames[:] = ()
385 continue
386 if directory != src_directory:
387 dir_package = directory[len(src_directory):].replace(os.sep, '.')
388 modules.append(package + dir_package)
389 for filename in filenames:
390 if _is_python_file(filename) and filename != '__init__.py':
391 src = join(directory, filename)
392 module = package + src[len(src_directory):-3]
393 modules.append(module.replace(os.sep, '.'))
394 return modules
395
396
397
399 """given a package directory return a list of all available python
400 module's files in the package and its subpackages
401
402 :type src_directory: str
403 :param src_directory:
404 path of the directory corresponding to the package
405
406 :type blacklist: list or tuple
407 :param blacklist:
408 optional list of files or directory to ignore, default to the value of
409 `logilab.common.STD_BLACKLIST`
410
411 :rtype: list
412 :return:
413 the list of all available python module's files in the package and
414 its subpackages
415 """
416 files = []
417 for directory, dirnames, filenames in os.walk(src_directory):
418 _handle_blacklist(blacklist, dirnames, filenames)
419
420 if not '__init__.py' in filenames:
421 dirnames[:] = ()
422 continue
423 for filename in filenames:
424 if _is_python_file(filename):
425 src = join(directory, filename)
426 files.append(src)
427 return files
428
429
431 """given a python module's file name return the matching source file
432 name (the filename will be returned identically if it's a already an
433 absolute path to a python source file...)
434
435 :type filename: str
436 :param filename: python module's file name
437
438
439 :raise NoSourceFile: if no source file exists on the file system
440
441 :rtype: str
442 :return: the absolute path of the source file if it exists
443 """
444 base, orig_ext = splitext(abspath(filename))
445 for ext in PY_SOURCE_EXTS:
446 source_path = '%s.%s' % (base, ext)
447 if exists(source_path):
448 return source_path
449 if include_no_ext and not orig_ext and exists(base):
450 return base
451 raise NoSourceFile(filename)
452
453
455 """remove submodules of `directories` from `sys.modules`"""
456 for modname, module in list(sys.modules.items()):
457 modfile = getattr(module, '__file__', None)
458 if modfile:
459 for directory in directories:
460 if modfile.startswith(directory):
461 del sys.modules[modname]
462 break
463
464
466 """
467 rtype: bool
468 return: True if the filename is a python source file
469 """
470 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
471
472
473
475 """try to guess if a module is a standard python module (by default,
476 see `std_path` parameter's description)
477
478 :type modname: str
479 :param modname: name of the module we are interested in
480
481 :type std_path: list(str) or tuple(str)
482 :param std_path: list of path considered has standard
483
484
485 :rtype: bool
486 :return:
487 true if the module:
488 - is located on the path listed in one of the directory in `std_path`
489 - is a built-in module
490 """
491 modname = modname.split('.')[0]
492 try:
493 filename = file_from_modpath([modname])
494 except ImportError as ex:
495
496
497 return 0
498
499
500 if filename is None:
501 return 1
502 filename = abspath(filename)
503 if filename.startswith(EXT_LIB_DIR):
504 return 0
505 for path in std_path:
506 if filename.startswith(abspath(path)):
507 return 1
508 return False
509
510
511
513 """return true if the given module name is relative to the given
514 file name
515
516 :type modname: str
517 :param modname: name of the module we are interested in
518
519 :type from_file: str
520 :param from_file:
521 path of the module from which modname has been imported
522
523 :rtype: bool
524 :return:
525 true if the module has been imported relatively to `from_file`
526 """
527 if not isdir(from_file):
528 from_file = dirname(from_file)
529 if from_file in sys.path:
530 return False
531 try:
532 find_module(modname.split('.')[0], [from_file])
533 return True
534 except ImportError:
535 return False
536
537
538
539
541 """given a mod path (i.e. splitted module / package name), return the
542 corresponding file
543
544 this function is used internally, see `file_from_modpath`'s
545 documentation for more information
546 """
547 assert len(modpath) > 0
548 if context is not None:
549 try:
550 mtype, mp_filename = _module_file(modpath, [context])
551 except ImportError:
552 mtype, mp_filename = _module_file(modpath, path)
553 else:
554 mtype, mp_filename = _module_file(modpath, path)
555 if mtype == PY_COMPILED:
556 try:
557 return get_source_file(mp_filename)
558 except NoSourceFile:
559 return mp_filename
560 elif mtype == C_BUILTIN:
561
562 return None
563 elif mtype == PKG_DIRECTORY:
564 mp_filename = _has_init(mp_filename)
565 return mp_filename
566
568 for filepath, importer in list(pic.items()):
569 if importer is not None:
570 if importer.find_module(modpath[0]):
571 if not importer.find_module('/'.join(modpath)):
572 raise ImportError('No module named %s in %s/%s' % (
573 '.'.join(modpath[1:]), file, modpath))
574 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
575 raise ImportError('No module named %s' % '.'.join(modpath))
576
578 """get a module type / file path
579
580 :type modpath: list or tuple
581 :param modpath:
582 splitted module's name (i.e name of a module or package splitted
583 on '.'), with leading empty strings for explicit relative import
584
585 :type path: list or None
586 :param path:
587 optional list of path where the module or package should be
588 searched (use sys.path if nothing or None is given)
589
590
591 :rtype: tuple(int, str)
592 :return: the module type flag and the file path for a module
593 """
594
595 try:
596 pic = sys.path_importer_cache
597 _path = (path is None and sys.path or path)
598 for __path in _path:
599 if not __path in pic:
600 try:
601 pic[__path] = zipimport.zipimporter(__path)
602 except zipimport.ZipImportError:
603 pic[__path] = None
604 checkeggs = True
605 except AttributeError:
606 checkeggs = False
607 imported = []
608 while modpath:
609 try:
610 _, mp_filename, mp_desc = find_module(modpath[0], path)
611 except ImportError:
612 if checkeggs:
613 return _search_zip(modpath, pic)[:2]
614 raise
615 else:
616 if checkeggs:
617 fullabspath = [abspath(x) for x in _path]
618 try:
619 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
620 emtype, emp_filename, zippath = _search_zip(modpath, pic)
621 if pathindex > _path.index(zippath):
622
623 return emtype, emp_filename
624 except ValueError:
625
626 pass
627 except ImportError:
628 pass
629 checkeggs = False
630 imported.append(modpath.pop(0))
631 mtype = mp_desc[2]
632 if modpath:
633 if mtype != PKG_DIRECTORY:
634 raise ImportError('No module %s in %s' % ('.'.join(modpath),
635 '.'.join(imported)))
636 path = [mp_filename]
637 return mtype, mp_filename
638
640 """return true if the given filename should be considered as a python file
641
642 .pyc and .pyo are ignored
643 """
644 for ext in ('.py', '.so', '.pyd', '.pyw'):
645 if filename.endswith(ext):
646 return True
647 return False
648
649
651 """if the given directory has a valid __init__ file, return its path,
652 else return None
653 """
654 mod_or_pack = join(directory, '__init__')
655 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
656 if exists(mod_or_pack + '.' + ext):
657 return mod_or_pack + '.' + ext
658 return None
659