Package logilab :: Package common :: Module modutils
[frames] | no frames]

Source Code for Module logilab.common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names as key 
 29  """ 
 30   
 31  __docformat__ = "restructuredtext en" 
 32   
 33  import sys 
 34  import os 
 35  from os.path import (splitext, join, abspath, isdir, dirname, exists, 
 36                       basename, expanduser, normcase, realpath) 
 37  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 38  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 39  from distutils.errors import DistutilsPlatformError 
 40   
 41  from six import PY3 
 42  from six.moves import map, range 
 43   
 44  try: 
 45      import zipimport 
 46  except ImportError: 
 47      zipimport = None 
 48   
 49  ZIPFILE = object() 
 50   
 51  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 52  from logilab.common.deprecation import deprecated 
 53   
 54  # Notes about STD_LIB_DIR 
 55  # Consider arch-specific installation for STD_LIB_DIR definition 
 56  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 57  # 
 58  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 59  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 60  if sys.platform.startswith('win'): 
 61      PY_SOURCE_EXTS = ('py', 'pyw') 
 62      PY_COMPILED_EXTS = ('dll', 'pyd') 
 63  else: 
 64      PY_SOURCE_EXTS = ('py',) 
 65      PY_COMPILED_EXTS = ('so',) 
 66   
 67  try: 
 68      STD_LIB_DIR = get_python_lib(standard_lib=True) 
 69  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to 
 70  # non-valid path, see https://bugs.pypy.org/issue1164 
 71  except DistutilsPlatformError: 
 72      STD_LIB_DIR = '//' 
 73   
 74  EXT_LIB_DIR = get_python_lib() 
 75   
 76  BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) 
77 78 79 -class NoSourceFile(Exception):
80 """exception raised when we are not able to get a python 81 source file for a precompiled file 82 """
83
84 -class LazyObject(object):
85 - def __init__(self, module, obj):
86 self.module = module 87 self.obj = obj 88 self._imported = None
89
90 - def _getobj(self):
91 if self._imported is None: 92 self._imported = getattr(load_module_from_name(self.module), 93 self.obj) 94 return self._imported
95
96 - def __getattribute__(self, attr):
97 try: 98 return super(LazyObject, self).__getattribute__(attr) 99 except AttributeError as ex: 100 return getattr(self._getobj(), attr)
101
102 - def __call__(self, *args, **kwargs):
103 return self._getobj()(*args, **kwargs)
104
105 106 -def load_module_from_name(dotted_name, path=None, use_sys=True):
107 """Load a Python module from its name. 108 109 :type dotted_name: str 110 :param dotted_name: python name of a module or package 111 112 :type path: list or None 113 :param path: 114 optional list of path where the module or package should be 115 searched (use sys.path if nothing or None is given) 116 117 :type use_sys: bool 118 :param use_sys: 119 boolean indicating whether the sys.modules dictionary should be 120 used or not 121 122 123 :raise ImportError: if the module or package is not found 124 125 :rtype: module 126 :return: the loaded module 127 """ 128 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
129
130 131 -def load_module_from_modpath(parts, path=None, use_sys=True):
132 """Load a python module from its splitted name. 133 134 :type parts: list(str) or tuple(str) 135 :param parts: 136 python name of a module or package splitted on '.' 137 138 :type path: list or None 139 :param path: 140 optional list of path where the module or package should be 141 searched (use sys.path if nothing or None is given) 142 143 :type use_sys: bool 144 :param use_sys: 145 boolean indicating whether the sys.modules dictionary should be used or not 146 147 :raise ImportError: if the module or package is not found 148 149 :rtype: module 150 :return: the loaded module 151 """ 152 if use_sys: 153 try: 154 return sys.modules['.'.join(parts)] 155 except KeyError: 156 pass 157 modpath = [] 158 prevmodule = None 159 for part in parts: 160 modpath.append(part) 161 curname = '.'.join(modpath) 162 module = None 163 if len(modpath) != len(parts): 164 # even with use_sys=False, should try to get outer packages from sys.modules 165 module = sys.modules.get(curname) 166 elif use_sys: 167 # because it may have been indirectly loaded through a parent 168 module = sys.modules.get(curname) 169 if module is None: 170 mp_file, mp_filename, mp_desc = find_module(part, path) 171 try: 172 module = load_module(curname, mp_file, mp_filename, mp_desc) 173 finally: 174 if mp_file is not None: 175 mp_file.close() 176 if prevmodule: 177 setattr(prevmodule, part, module) 178 _file = getattr(module, '__file__', '') 179 prevmodule = module 180 if not _file and _is_namespace(curname): 181 continue 182 if not _file and len(modpath) != len(parts): 183 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 184 path = [dirname( _file )] 185 return module
186
187 188 -def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):
189 """Load a Python module from it's path. 190 191 :type filepath: str 192 :param filepath: path to the python module or package 193 194 :type path: list or None 195 :param path: 196 optional list of path where the module or package should be 197 searched (use sys.path if nothing or None is given) 198 199 :type use_sys: bool 200 :param use_sys: 201 boolean indicating whether the sys.modules dictionary should be 202 used or not 203 204 205 :raise ImportError: if the module or package is not found 206 207 :rtype: module 208 :return: the loaded module 209 """ 210 modpath = modpath_from_file(filepath, extrapath) 211 return load_module_from_modpath(modpath, path, use_sys)
212
213 214 -def _check_init(path, mod_path):
215 """check there are some __init__.py all along the way""" 216 modpath = [] 217 for part in mod_path: 218 modpath.append(part) 219 path = join(path, part) 220 if not _is_namespace('.'.join(modpath)) and not _has_init(path): 221 return False 222 return True
223
224 225 -def _canonicalize_path(path):
226 return realpath(expanduser(path))
227
228 229 -def _path_from_filename(filename):
230 if PY3: 231 return filename 232 else: 233 if filename.endswith(".pyc"): 234 return filename[:-1] 235 return filename
236
237 238 @deprecated('you should avoid using modpath_from_file()') 239 -def modpath_from_file(filename, extrapath=None):
240 """DEPRECATED: doens't play well with symlinks and sys.meta_path 241 242 Given a file path return the corresponding splitted module's name 243 (i.e name of a module or package splitted on '.') 244 245 :type filename: str 246 :param filename: file's path for which we want the module's name 247 248 :type extrapath: dict 249 :param extrapath: 250 optional extra search path, with path as key and package name for the path 251 as value. This is usually useful to handle package splitted in multiple 252 directories using __path__ trick. 253 254 255 :raise ImportError: 256 if the corresponding module's name has not been found 257 258 :rtype: list(str) 259 :return: the corresponding splitted module's name 260 """ 261 filename = _path_from_filename(filename) 262 filename = _canonicalize_path(filename) 263 base = os.path.splitext(filename)[0] 264 265 if extrapath is not None: 266 for path_ in map(_canonicalize_path, extrapath): 267 path = abspath(path_) 268 if path and normcase(base[:len(path)]) == normcase(path): 269 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 270 if pkg] 271 if _check_init(path, submodpath[:-1]): 272 return extrapath[path_].split('.') + submodpath 273 274 for path in map(_canonicalize_path, sys.path): 275 if path and normcase(base).startswith(path): 276 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 277 if _check_init(path, modpath[:-1]): 278 return modpath 279 280 raise ImportError('Unable to find module for %s in %s' % ( 281 filename, ', \n'.join(sys.path)))
282
283 284 -def file_from_modpath(modpath, path=None, context_file=None):
285 """given a mod path (i.e. splitted module / package name), return the 286 corresponding file, giving priority to source file over precompiled 287 file if it exists 288 289 :type modpath: list or tuple 290 :param modpath: 291 splitted module's name (i.e name of a module or package splitted 292 on '.') 293 (this means explicit relative imports that start with dots have 294 empty strings in this list!) 295 296 :type path: list or None 297 :param path: 298 optional list of path where the module or package should be 299 searched (use sys.path if nothing or None is given) 300 301 :type context_file: str or None 302 :param context_file: 303 context file to consider, necessary if the identifier has been 304 introduced using a relative import unresolvable in the actual 305 context (i.e. modutils) 306 307 :raise ImportError: if there is no such module in the directory 308 309 :rtype: str or None 310 :return: 311 the path to the module's file or None if it's an integrated 312 builtin module such as 'sys' 313 """ 314 if context_file is not None: 315 context = dirname(context_file) 316 else: 317 context = context_file 318 if modpath[0] == 'xml': 319 # handle _xmlplus 320 try: 321 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 322 except ImportError: 323 return _file_from_modpath(modpath, path, context) 324 elif modpath == ['os', 'path']: 325 # FIXME: currently ignoring search_path... 326 return os.path.__file__ 327 return _file_from_modpath(modpath, path, context)
328
329 330 331 -def get_module_part(dotted_name, context_file=None):
332 """given a dotted name return the module part of the name : 333 334 >>> get_module_part('logilab.common.modutils.get_module_part') 335 'logilab.common.modutils' 336 337 :type dotted_name: str 338 :param dotted_name: full name of the identifier we are interested in 339 340 :type context_file: str or None 341 :param context_file: 342 context file to consider, necessary if the identifier has been 343 introduced using a relative import unresolvable in the actual 344 context (i.e. modutils) 345 346 347 :raise ImportError: if there is no such module in the directory 348 349 :rtype: str or None 350 :return: 351 the module part of the name or None if we have not been able at 352 all to import the given name 353 354 XXX: deprecated, since it doesn't handle package precedence over module 355 (see #10066) 356 """ 357 # os.path trick 358 if dotted_name.startswith('os.path'): 359 return 'os.path' 360 parts = dotted_name.split('.') 361 if context_file is not None: 362 # first check for builtin module which won't be considered latter 363 # in that case (path != None) 364 if parts[0] in BUILTIN_MODULES: 365 if len(parts) > 2: 366 raise ImportError(dotted_name) 367 return parts[0] 368 # don't use += or insert, we want a new list to be created ! 369 path = None 370 starti = 0 371 if parts[0] == '': 372 assert context_file is not None, \ 373 'explicit relative import, but no context_file?' 374 path = [] # prevent resolving the import non-relatively 375 starti = 1 376 while parts[starti] == '': # for all further dots: change context 377 starti += 1 378 context_file = dirname(context_file) 379 for i in range(starti, len(parts)): 380 try: 381 file_from_modpath(parts[starti:i+1], 382 path=path, context_file=context_file) 383 except ImportError: 384 if not i >= max(1, len(parts) - 2): 385 raise 386 return '.'.join(parts[:i]) 387 return dotted_name
388
389 390 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
391 """given a package directory return a list of all available python 392 modules in the package and its subpackages 393 394 :type package: str 395 :param package: the python name for the package 396 397 :type src_directory: str 398 :param src_directory: 399 path of the directory corresponding to the package 400 401 :type blacklist: list or tuple 402 :param blacklist: 403 optional list of files or directory to ignore, default to 404 the value of `logilab.common.STD_BLACKLIST` 405 406 :rtype: list 407 :return: 408 the list of all available python modules in the package and its 409 subpackages 410 """ 411 modules = [] 412 for directory, dirnames, filenames in os.walk(src_directory): 413 _handle_blacklist(blacklist, dirnames, filenames) 414 # check for __init__.py 415 if not '__init__.py' in filenames: 416 dirnames[:] = () 417 continue 418 if directory != src_directory: 419 dir_package = directory[len(src_directory):].replace(os.sep, '.') 420 modules.append(package + dir_package) 421 for filename in filenames: 422 if _is_python_file(filename) and filename != '__init__.py': 423 src = join(directory, filename) 424 module = package + src[len(src_directory):-3] 425 modules.append(module.replace(os.sep, '.')) 426 return modules
427
428 429 430 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
431 """given a package directory return a list of all available python 432 module's files in the package and its subpackages 433 434 :type src_directory: str 435 :param src_directory: 436 path of the directory corresponding to the package 437 438 :type blacklist: list or tuple 439 :param blacklist: 440 optional list of files or directory to ignore, default to the value of 441 `logilab.common.STD_BLACKLIST` 442 443 :rtype: list 444 :return: 445 the list of all available python module's files in the package and 446 its subpackages 447 """ 448 files = [] 449 for directory, dirnames, filenames in os.walk(src_directory): 450 _handle_blacklist(blacklist, dirnames, filenames) 451 # check for __init__.py 452 if not '__init__.py' in filenames: 453 dirnames[:] = () 454 continue 455 for filename in filenames: 456 if _is_python_file(filename): 457 src = join(directory, filename) 458 files.append(src) 459 return files
460
461 462 -def get_source_file(filename, include_no_ext=False):
463 """given a python module's file name return the matching source file 464 name (the filename will be returned identically if it's a already an 465 absolute path to a python source file...) 466 467 :type filename: str 468 :param filename: python module's file name 469 470 471 :raise NoSourceFile: if no source file exists on the file system 472 473 :rtype: str 474 :return: the absolute path of the source file if it exists 475 """ 476 base, orig_ext = splitext(abspath(filename)) 477 for ext in PY_SOURCE_EXTS: 478 source_path = '%s.%s' % (base, ext) 479 if exists(source_path): 480 return source_path 481 if include_no_ext and not orig_ext and exists(base): 482 return base 483 raise NoSourceFile(filename)
484
485 486 -def cleanup_sys_modules(directories):
487 """remove submodules of `directories` from `sys.modules`""" 488 cleaned = [] 489 for modname, module in list(sys.modules.items()): 490 modfile = getattr(module, '__file__', None) 491 if modfile: 492 for directory in directories: 493 if modfile.startswith(directory): 494 cleaned.append(modname) 495 del sys.modules[modname] 496 break 497 return cleaned
498
499 500 -def clean_sys_modules(names):
501 """remove submodules starting with name from `names` from `sys.modules`""" 502 cleaned = set() 503 for modname in list(sys.modules): 504 for name in names: 505 if modname.startswith(name): 506 del sys.modules[modname] 507 cleaned.add(modname) 508 break 509 return cleaned
510
511 512 -def is_python_source(filename):
513 """ 514 rtype: bool 515 return: True if the filename is a python source file 516 """ 517 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
518
519 520 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
521 """try to guess if a module is a standard python module (by default, 522 see `std_path` parameter's description) 523 524 :type modname: str 525 :param modname: name of the module we are interested in 526 527 :type std_path: list(str) or tuple(str) 528 :param std_path: list of path considered as standard 529 530 531 :rtype: bool 532 :return: 533 true if the module: 534 - is located on the path listed in one of the directory in `std_path` 535 - is a built-in module 536 537 Note: this function is known to return wrong values when inside virtualenv. 538 See https://www.logilab.org/ticket/294756. 539 """ 540 modname = modname.split('.')[0] 541 try: 542 filename = file_from_modpath([modname]) 543 except ImportError as ex: 544 # import failed, i'm probably not so wrong by supposing it's 545 # not standard... 546 return False 547 # modules which are not living in a file are considered standard 548 # (sys and __builtin__ for instance) 549 if filename is None: 550 # we assume there are no namespaces in stdlib 551 return not _is_namespace(modname) 552 filename = abspath(filename) 553 if filename.startswith(EXT_LIB_DIR): 554 return False 555 for path in std_path: 556 if filename.startswith(abspath(path)): 557 return True 558 return False
559
560 561 562 -def is_relative(modname, from_file):
563 """return true if the given module name is relative to the given 564 file name 565 566 :type modname: str 567 :param modname: name of the module we are interested in 568 569 :type from_file: str 570 :param from_file: 571 path of the module from which modname has been imported 572 573 :rtype: bool 574 :return: 575 true if the module has been imported relatively to `from_file` 576 """ 577 if not isdir(from_file): 578 from_file = dirname(from_file) 579 if from_file in sys.path: 580 return False 581 try: 582 find_module(modname.split('.')[0], [from_file]) 583 return True 584 except ImportError: 585 return False
586
587 588 # internal only functions ##################################################### 589 590 -def _file_from_modpath(modpath, path=None, context=None):
591 """given a mod path (i.e. splitted module / package name), return the 592 corresponding file 593 594 this function is used internally, see `file_from_modpath`'s 595 documentation for more information 596 """ 597 assert len(modpath) > 0 598 if context is not None: 599 try: 600 mtype, mp_filename = _module_file(modpath, [context]) 601 except ImportError: 602 mtype, mp_filename = _module_file(modpath, path) 603 else: 604 mtype, mp_filename = _module_file(modpath, path) 605 if mtype == PY_COMPILED: 606 try: 607 return get_source_file(mp_filename) 608 except NoSourceFile: 609 return mp_filename 610 elif mtype == C_BUILTIN: 611 # integrated builtin module 612 return None 613 elif mtype == PKG_DIRECTORY: 614 mp_filename = _has_init(mp_filename) 615 return mp_filename
616
617 -def _search_zip(modpath, pic):
618 for filepath, importer in pic.items(): 619 if importer is not None: 620 if importer.find_module(modpath[0]): 621 if not importer.find_module('/'.join(modpath)): 622 raise ImportError('No module named %s in %s/%s' % ( 623 '.'.join(modpath[1:]), filepath, modpath)) 624 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 625 raise ImportError('No module named %s' % '.'.join(modpath))
626 627 try: 628 import pkg_resources 629 except ImportError: 630 pkg_resources = None
631 632 633 -def _is_namespace(modname):
634 return (pkg_resources is not None 635 and modname in pkg_resources._namespace_packages)
636
637 638 -def _module_file(modpath, path=None):
639 """get a module type / file path 640 641 :type modpath: list or tuple 642 :param modpath: 643 splitted module's name (i.e name of a module or package splitted 644 on '.'), with leading empty strings for explicit relative import 645 646 :type path: list or None 647 :param path: 648 optional list of path where the module or package should be 649 searched (use sys.path if nothing or None is given) 650 651 652 :rtype: tuple(int, str) 653 :return: the module type flag and the file path for a module 654 """ 655 # egg support compat 656 try: 657 pic = sys.path_importer_cache 658 _path = (path is None and sys.path or path) 659 for __path in _path: 660 if not __path in pic: 661 try: 662 pic[__path] = zipimport.zipimporter(__path) 663 except zipimport.ZipImportError: 664 pic[__path] = None 665 checkeggs = True 666 except AttributeError: 667 checkeggs = False 668 # pkg_resources support (aka setuptools namespace packages) 669 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules): 670 # setuptools has added into sys.modules a module object with proper 671 # __path__, get back information from there 672 module = sys.modules[modpath.pop(0)] 673 # use list() to protect against _NamespacePath instance we get with python 3, which 674 # find_module later doesn't like 675 path = list(module.__path__) 676 if not modpath: 677 return C_BUILTIN, None 678 imported = [] 679 while modpath: 680 modname = modpath[0] 681 # take care to changes in find_module implementation wrt builtin modules 682 # 683 # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23) 684 # >>> imp.find_module('posix') 685 # (None, 'posix', ('', '', 6)) 686 # 687 # Python 3.3.1 (default, Apr 26 2013, 12:08:46) 688 # >>> imp.find_module('posix') 689 # (None, None, ('', '', 6)) 690 try: 691 _, mp_filename, mp_desc = find_module(modname, path) 692 except ImportError: 693 if checkeggs: 694 return _search_zip(modpath, pic)[:2] 695 raise 696 else: 697 if checkeggs and mp_filename: 698 fullabspath = [abspath(x) for x in _path] 699 try: 700 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 701 emtype, emp_filename, zippath = _search_zip(modpath, pic) 702 if pathindex > _path.index(zippath): 703 # an egg takes priority 704 return emtype, emp_filename 705 except ValueError: 706 # XXX not in _path 707 pass 708 except ImportError: 709 pass 710 checkeggs = False 711 imported.append(modpath.pop(0)) 712 mtype = mp_desc[2] 713 if modpath: 714 if mtype != PKG_DIRECTORY: 715 raise ImportError('No module %s in %s' % ('.'.join(modpath), 716 '.'.join(imported))) 717 # XXX guess if package is using pkgutil.extend_path by looking for 718 # those keywords in the first four Kbytes 719 try: 720 with open(join(mp_filename, '__init__.py')) as stream: 721 data = stream.read(4096) 722 except IOError: 723 path = [mp_filename] 724 else: 725 if 'pkgutil' in data and 'extend_path' in data: 726 # extend_path is called, search sys.path for module/packages 727 # of this name see pkgutil.extend_path documentation 728 path = [join(p, *imported) for p in sys.path 729 if isdir(join(p, *imported))] 730 else: 731 path = [mp_filename] 732 return mtype, mp_filename
733
734 -def _is_python_file(filename):
735 """return true if the given filename should be considered as a python file 736 737 .pyc and .pyo are ignored 738 """ 739 for ext in ('.py', '.so', '.pyd', '.pyw'): 740 if filename.endswith(ext): 741 return True 742 return False
743
744 745 -def _has_init(directory):
746 """if the given directory has a valid __init__ file, return its path, 747 else return None 748 """ 749 mod_or_pack = join(directory, '__init__') 750 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 751 if exists(mod_or_pack + '.' + ext): 752 return mod_or_pack + '.' + ext 753 return None
754