1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21 :type PY_SOURCE_EXTS: tuple(str)
22 :var PY_SOURCE_EXTS: list of possible python source file extension
23
24 :type STD_LIB_DIR: str
25 :var STD_LIB_DIR: directory where standard modules are located
26
27 :type BUILTIN_MODULES: dict
28 :var BUILTIN_MODULES: dictionary with builtin module names as key
29 """
30
31 __docformat__ = "restructuredtext en"
32
33 import sys
34 import os
35 from os.path import (splitext, join, abspath, isdir, dirname, exists,
36 basename, expanduser, normcase, realpath)
37 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
38 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
39 from distutils.errors import DistutilsPlatformError
40
41 from six import PY3
42 from six.moves import map, range
43
44 try:
45 import zipimport
46 except ImportError:
47 zipimport = None
48
49 ZIPFILE = object()
50
51 from logilab.common import STD_BLACKLIST, _handle_blacklist
52 from logilab.common.deprecation import deprecated
53
54
55
56
57
58
59
60 if sys.platform.startswith('win'):
61 PY_SOURCE_EXTS = ('py', 'pyw')
62 PY_COMPILED_EXTS = ('dll', 'pyd')
63 else:
64 PY_SOURCE_EXTS = ('py',)
65 PY_COMPILED_EXTS = ('so',)
66
67 try:
68 STD_LIB_DIR = get_python_lib(standard_lib=True)
69
70
71 except DistutilsPlatformError:
72 STD_LIB_DIR = '//'
73
74 EXT_LIB_DIR = get_python_lib()
75
76 BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
80 """exception raised when we are not able to get a python
81 source file for a precompiled file
82 """
83
86 self.module = module
87 self.obj = obj
88 self._imported = None
89
91 if self._imported is None:
92 self._imported = getattr(load_module_from_name(self.module),
93 self.obj)
94 return self._imported
95
97 try:
98 return super(LazyObject, self).__getattribute__(attr)
99 except AttributeError as ex:
100 return getattr(self._getobj(), attr)
101
103 return self._getobj()(*args, **kwargs)
104
107 """Load a Python module from its name.
108
109 :type dotted_name: str
110 :param dotted_name: python name of a module or package
111
112 :type path: list or None
113 :param path:
114 optional list of path where the module or package should be
115 searched (use sys.path if nothing or None is given)
116
117 :type use_sys: bool
118 :param use_sys:
119 boolean indicating whether the sys.modules dictionary should be
120 used or not
121
122
123 :raise ImportError: if the module or package is not found
124
125 :rtype: module
126 :return: the loaded module
127 """
128 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
129
132 """Load a python module from its splitted name.
133
134 :type parts: list(str) or tuple(str)
135 :param parts:
136 python name of a module or package splitted on '.'
137
138 :type path: list or None
139 :param path:
140 optional list of path where the module or package should be
141 searched (use sys.path if nothing or None is given)
142
143 :type use_sys: bool
144 :param use_sys:
145 boolean indicating whether the sys.modules dictionary should be used or not
146
147 :raise ImportError: if the module or package is not found
148
149 :rtype: module
150 :return: the loaded module
151 """
152 if use_sys:
153 try:
154 return sys.modules['.'.join(parts)]
155 except KeyError:
156 pass
157 modpath = []
158 prevmodule = None
159 for part in parts:
160 modpath.append(part)
161 curname = '.'.join(modpath)
162 module = None
163 if len(modpath) != len(parts):
164
165 module = sys.modules.get(curname)
166 elif use_sys:
167
168 module = sys.modules.get(curname)
169 if module is None:
170 mp_file, mp_filename, mp_desc = find_module(part, path)
171 try:
172 module = load_module(curname, mp_file, mp_filename, mp_desc)
173 finally:
174 if mp_file is not None:
175 mp_file.close()
176 if prevmodule:
177 setattr(prevmodule, part, module)
178 _file = getattr(module, '__file__', '')
179 prevmodule = module
180 if not _file and _is_namespace(curname):
181 continue
182 if not _file and len(modpath) != len(parts):
183 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
184 path = [dirname( _file )]
185 return module
186
189 """Load a Python module from it's path.
190
191 :type filepath: str
192 :param filepath: path to the python module or package
193
194 :type path: list or None
195 :param path:
196 optional list of path where the module or package should be
197 searched (use sys.path if nothing or None is given)
198
199 :type use_sys: bool
200 :param use_sys:
201 boolean indicating whether the sys.modules dictionary should be
202 used or not
203
204
205 :raise ImportError: if the module or package is not found
206
207 :rtype: module
208 :return: the loaded module
209 """
210 modpath = modpath_from_file(filepath, extrapath)
211 return load_module_from_modpath(modpath, path, use_sys)
212
215 """check there are some __init__.py all along the way"""
216 modpath = []
217 for part in mod_path:
218 modpath.append(part)
219 path = join(path, part)
220 if not _is_namespace('.'.join(modpath)) and not _has_init(path):
221 return False
222 return True
223
226 return realpath(expanduser(path))
227
230 if PY3:
231 return filename
232 else:
233 if filename.endswith(".pyc"):
234 return filename[:-1]
235 return filename
236
237
238 @deprecated('you should avoid using modpath_from_file()')
239 -def modpath_from_file(filename, extrapath=None):
240 """DEPRECATED: doens't play well with symlinks and sys.meta_path
241
242 Given a file path return the corresponding splitted module's name
243 (i.e name of a module or package splitted on '.')
244
245 :type filename: str
246 :param filename: file's path for which we want the module's name
247
248 :type extrapath: dict
249 :param extrapath:
250 optional extra search path, with path as key and package name for the path
251 as value. This is usually useful to handle package splitted in multiple
252 directories using __path__ trick.
253
254
255 :raise ImportError:
256 if the corresponding module's name has not been found
257
258 :rtype: list(str)
259 :return: the corresponding splitted module's name
260 """
261 filename = _path_from_filename(filename)
262 filename = _canonicalize_path(filename)
263 base = os.path.splitext(filename)[0]
264
265 if extrapath is not None:
266 for path_ in map(_canonicalize_path, extrapath):
267 path = abspath(path_)
268 if path and normcase(base[:len(path)]) == normcase(path):
269 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
270 if pkg]
271 if _check_init(path, submodpath[:-1]):
272 return extrapath[path_].split('.') + submodpath
273
274 for path in map(_canonicalize_path, sys.path):
275 if path and normcase(base).startswith(path):
276 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
277 if _check_init(path, modpath[:-1]):
278 return modpath
279
280 raise ImportError('Unable to find module for %s in %s' % (
281 filename, ', \n'.join(sys.path)))
282
285 """given a mod path (i.e. splitted module / package name), return the
286 corresponding file, giving priority to source file over precompiled
287 file if it exists
288
289 :type modpath: list or tuple
290 :param modpath:
291 splitted module's name (i.e name of a module or package splitted
292 on '.')
293 (this means explicit relative imports that start with dots have
294 empty strings in this list!)
295
296 :type path: list or None
297 :param path:
298 optional list of path where the module or package should be
299 searched (use sys.path if nothing or None is given)
300
301 :type context_file: str or None
302 :param context_file:
303 context file to consider, necessary if the identifier has been
304 introduced using a relative import unresolvable in the actual
305 context (i.e. modutils)
306
307 :raise ImportError: if there is no such module in the directory
308
309 :rtype: str or None
310 :return:
311 the path to the module's file or None if it's an integrated
312 builtin module such as 'sys'
313 """
314 if context_file is not None:
315 context = dirname(context_file)
316 else:
317 context = context_file
318 if modpath[0] == 'xml':
319
320 try:
321 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
322 except ImportError:
323 return _file_from_modpath(modpath, path, context)
324 elif modpath == ['os', 'path']:
325
326 return os.path.__file__
327 return _file_from_modpath(modpath, path, context)
328
332 """given a dotted name return the module part of the name :
333
334 >>> get_module_part('logilab.common.modutils.get_module_part')
335 'logilab.common.modutils'
336
337 :type dotted_name: str
338 :param dotted_name: full name of the identifier we are interested in
339
340 :type context_file: str or None
341 :param context_file:
342 context file to consider, necessary if the identifier has been
343 introduced using a relative import unresolvable in the actual
344 context (i.e. modutils)
345
346
347 :raise ImportError: if there is no such module in the directory
348
349 :rtype: str or None
350 :return:
351 the module part of the name or None if we have not been able at
352 all to import the given name
353
354 XXX: deprecated, since it doesn't handle package precedence over module
355 (see #10066)
356 """
357
358 if dotted_name.startswith('os.path'):
359 return 'os.path'
360 parts = dotted_name.split('.')
361 if context_file is not None:
362
363
364 if parts[0] in BUILTIN_MODULES:
365 if len(parts) > 2:
366 raise ImportError(dotted_name)
367 return parts[0]
368
369 path = None
370 starti = 0
371 if parts[0] == '':
372 assert context_file is not None, \
373 'explicit relative import, but no context_file?'
374 path = []
375 starti = 1
376 while parts[starti] == '':
377 starti += 1
378 context_file = dirname(context_file)
379 for i in range(starti, len(parts)):
380 try:
381 file_from_modpath(parts[starti:i+1],
382 path=path, context_file=context_file)
383 except ImportError:
384 if not i >= max(1, len(parts) - 2):
385 raise
386 return '.'.join(parts[:i])
387 return dotted_name
388
391 """given a package directory return a list of all available python
392 modules in the package and its subpackages
393
394 :type package: str
395 :param package: the python name for the package
396
397 :type src_directory: str
398 :param src_directory:
399 path of the directory corresponding to the package
400
401 :type blacklist: list or tuple
402 :param blacklist:
403 optional list of files or directory to ignore, default to
404 the value of `logilab.common.STD_BLACKLIST`
405
406 :rtype: list
407 :return:
408 the list of all available python modules in the package and its
409 subpackages
410 """
411 modules = []
412 for directory, dirnames, filenames in os.walk(src_directory):
413 _handle_blacklist(blacklist, dirnames, filenames)
414
415 if not '__init__.py' in filenames:
416 dirnames[:] = ()
417 continue
418 if directory != src_directory:
419 dir_package = directory[len(src_directory):].replace(os.sep, '.')
420 modules.append(package + dir_package)
421 for filename in filenames:
422 if _is_python_file(filename) and filename != '__init__.py':
423 src = join(directory, filename)
424 module = package + src[len(src_directory):-3]
425 modules.append(module.replace(os.sep, '.'))
426 return modules
427
431 """given a package directory return a list of all available python
432 module's files in the package and its subpackages
433
434 :type src_directory: str
435 :param src_directory:
436 path of the directory corresponding to the package
437
438 :type blacklist: list or tuple
439 :param blacklist:
440 optional list of files or directory to ignore, default to the value of
441 `logilab.common.STD_BLACKLIST`
442
443 :rtype: list
444 :return:
445 the list of all available python module's files in the package and
446 its subpackages
447 """
448 files = []
449 for directory, dirnames, filenames in os.walk(src_directory):
450 _handle_blacklist(blacklist, dirnames, filenames)
451
452 if not '__init__.py' in filenames:
453 dirnames[:] = ()
454 continue
455 for filename in filenames:
456 if _is_python_file(filename):
457 src = join(directory, filename)
458 files.append(src)
459 return files
460
463 """given a python module's file name return the matching source file
464 name (the filename will be returned identically if it's a already an
465 absolute path to a python source file...)
466
467 :type filename: str
468 :param filename: python module's file name
469
470
471 :raise NoSourceFile: if no source file exists on the file system
472
473 :rtype: str
474 :return: the absolute path of the source file if it exists
475 """
476 base, orig_ext = splitext(abspath(filename))
477 for ext in PY_SOURCE_EXTS:
478 source_path = '%s.%s' % (base, ext)
479 if exists(source_path):
480 return source_path
481 if include_no_ext and not orig_ext and exists(base):
482 return base
483 raise NoSourceFile(filename)
484
487 """remove submodules of `directories` from `sys.modules`"""
488 cleaned = []
489 for modname, module in list(sys.modules.items()):
490 modfile = getattr(module, '__file__', None)
491 if modfile:
492 for directory in directories:
493 if modfile.startswith(directory):
494 cleaned.append(modname)
495 del sys.modules[modname]
496 break
497 return cleaned
498
501 """remove submodules starting with name from `names` from `sys.modules`"""
502 cleaned = set()
503 for modname in list(sys.modules):
504 for name in names:
505 if modname.startswith(name):
506 del sys.modules[modname]
507 cleaned.add(modname)
508 break
509 return cleaned
510
513 """
514 rtype: bool
515 return: True if the filename is a python source file
516 """
517 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
518
521 """try to guess if a module is a standard python module (by default,
522 see `std_path` parameter's description)
523
524 :type modname: str
525 :param modname: name of the module we are interested in
526
527 :type std_path: list(str) or tuple(str)
528 :param std_path: list of path considered as standard
529
530
531 :rtype: bool
532 :return:
533 true if the module:
534 - is located on the path listed in one of the directory in `std_path`
535 - is a built-in module
536
537 Note: this function is known to return wrong values when inside virtualenv.
538 See https://www.logilab.org/ticket/294756.
539 """
540 modname = modname.split('.')[0]
541 try:
542 filename = file_from_modpath([modname])
543 except ImportError as ex:
544
545
546 return False
547
548
549 if filename is None:
550
551 return not _is_namespace(modname)
552 filename = abspath(filename)
553 if filename.startswith(EXT_LIB_DIR):
554 return False
555 for path in std_path:
556 if filename.startswith(abspath(path)):
557 return True
558 return False
559
563 """return true if the given module name is relative to the given
564 file name
565
566 :type modname: str
567 :param modname: name of the module we are interested in
568
569 :type from_file: str
570 :param from_file:
571 path of the module from which modname has been imported
572
573 :rtype: bool
574 :return:
575 true if the module has been imported relatively to `from_file`
576 """
577 if not isdir(from_file):
578 from_file = dirname(from_file)
579 if from_file in sys.path:
580 return False
581 try:
582 find_module(modname.split('.')[0], [from_file])
583 return True
584 except ImportError:
585 return False
586
591 """given a mod path (i.e. splitted module / package name), return the
592 corresponding file
593
594 this function is used internally, see `file_from_modpath`'s
595 documentation for more information
596 """
597 assert len(modpath) > 0
598 if context is not None:
599 try:
600 mtype, mp_filename = _module_file(modpath, [context])
601 except ImportError:
602 mtype, mp_filename = _module_file(modpath, path)
603 else:
604 mtype, mp_filename = _module_file(modpath, path)
605 if mtype == PY_COMPILED:
606 try:
607 return get_source_file(mp_filename)
608 except NoSourceFile:
609 return mp_filename
610 elif mtype == C_BUILTIN:
611
612 return None
613 elif mtype == PKG_DIRECTORY:
614 mp_filename = _has_init(mp_filename)
615 return mp_filename
616
618 for filepath, importer in pic.items():
619 if importer is not None:
620 if importer.find_module(modpath[0]):
621 if not importer.find_module('/'.join(modpath)):
622 raise ImportError('No module named %s in %s/%s' % (
623 '.'.join(modpath[1:]), filepath, modpath))
624 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
625 raise ImportError('No module named %s' % '.'.join(modpath))
626
627 try:
628 import pkg_resources
629 except ImportError:
630 pkg_resources = None
636
639 """get a module type / file path
640
641 :type modpath: list or tuple
642 :param modpath:
643 splitted module's name (i.e name of a module or package splitted
644 on '.'), with leading empty strings for explicit relative import
645
646 :type path: list or None
647 :param path:
648 optional list of path where the module or package should be
649 searched (use sys.path if nothing or None is given)
650
651
652 :rtype: tuple(int, str)
653 :return: the module type flag and the file path for a module
654 """
655
656 try:
657 pic = sys.path_importer_cache
658 _path = (path is None and sys.path or path)
659 for __path in _path:
660 if not __path in pic:
661 try:
662 pic[__path] = zipimport.zipimporter(__path)
663 except zipimport.ZipImportError:
664 pic[__path] = None
665 checkeggs = True
666 except AttributeError:
667 checkeggs = False
668
669 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules):
670
671
672 module = sys.modules[modpath.pop(0)]
673
674
675 path = list(module.__path__)
676 if not modpath:
677 return C_BUILTIN, None
678 imported = []
679 while modpath:
680 modname = modpath[0]
681
682
683
684
685
686
687
688
689
690 try:
691 _, mp_filename, mp_desc = find_module(modname, path)
692 except ImportError:
693 if checkeggs:
694 return _search_zip(modpath, pic)[:2]
695 raise
696 else:
697 if checkeggs and mp_filename:
698 fullabspath = [abspath(x) for x in _path]
699 try:
700 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
701 emtype, emp_filename, zippath = _search_zip(modpath, pic)
702 if pathindex > _path.index(zippath):
703
704 return emtype, emp_filename
705 except ValueError:
706
707 pass
708 except ImportError:
709 pass
710 checkeggs = False
711 imported.append(modpath.pop(0))
712 mtype = mp_desc[2]
713 if modpath:
714 if mtype != PKG_DIRECTORY:
715 raise ImportError('No module %s in %s' % ('.'.join(modpath),
716 '.'.join(imported)))
717
718
719 try:
720 with open(join(mp_filename, '__init__.py')) as stream:
721 data = stream.read(4096)
722 except IOError:
723 path = [mp_filename]
724 else:
725 if 'pkgutil' in data and 'extend_path' in data:
726
727
728 path = [join(p, *imported) for p in sys.path
729 if isdir(join(p, *imported))]
730 else:
731 path = [mp_filename]
732 return mtype, mp_filename
733
735 """return true if the given filename should be considered as a python file
736
737 .pyc and .pyo are ignored
738 """
739 for ext in ('.py', '.so', '.pyd', '.pyw'):
740 if filename.endswith(ext):
741 return True
742 return False
743
746 """if the given directory has a valid __init__ file, return its path,
747 else return None
748 """
749 mod_or_pack = join(directory, '__init__')
750 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
751 if exists(mod_or_pack + '.' + ext):
752 return mod_or_pack + '.' + ext
753 return None
754