123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647 |
- # -*- coding: utf-8 -*-
- # Copyright (c) 2014-2016 Claudiu Popa <pcmanticore@gmail.com>
- # Copyright (c) 2014 Google, Inc.
- # Copyright (c) 2015 Florian Bruhin <me@the-compiler.org>
- # Copyright (c) 2015 Radosław Ganczarek <radoslaw@ganczarek.in>
- # Copyright (c) 2016 Jakub Wilk <jwilk@jwilk.net>
-
- # Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
- # For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER
-
- """Python modules manipulation utility functions.
-
- :type PY_SOURCE_EXTS: tuple(str)
- :var PY_SOURCE_EXTS: list of possible python source file extension
-
- :type STD_LIB_DIRS: set of str
- :var STD_LIB_DIRS: directories where standard modules are located
-
- :type BUILTIN_MODULES: dict
- :var BUILTIN_MODULES: dictionary with builtin module names has key
- """
- import imp
- import os
- import platform
- import sys
- from distutils.sysconfig import get_python_lib # pylint: disable=import-error
- # pylint: disable=import-error, no-name-in-module
- from distutils.errors import DistutilsPlatformError
- # distutils is replaced by virtualenv with a module that does
- # weird path manipulations in order to get to the
- # real distutils module.
-
- import six
-
- from .interpreter._import import spec
- from .interpreter._import import util
-
- if sys.platform.startswith('win'):
- PY_SOURCE_EXTS = ('py', 'pyw')
- PY_COMPILED_EXTS = ('dll', 'pyd')
- else:
- PY_SOURCE_EXTS = ('py',)
- PY_COMPILED_EXTS = ('so',)
-
-
- try:
- # The explicit sys.prefix is to work around a patch in virtualenv that
- # replaces the 'real' sys.prefix (i.e. the location of the binary)
- # with the prefix from which the virtualenv was created. This throws
- # off the detection logic for standard library modules, thus the
- # workaround.
- STD_LIB_DIRS = set([
- get_python_lib(standard_lib=True, prefix=sys.prefix),
- # Take care of installations where exec_prefix != prefix.
- get_python_lib(standard_lib=True, prefix=sys.exec_prefix),
- get_python_lib(standard_lib=True)])
- # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to
- # non-valid path, see https://bugs.pypy.org/issue1164
- except DistutilsPlatformError:
- STD_LIB_DIRS = set()
-
- if os.name == 'nt':
- STD_LIB_DIRS.add(os.path.join(sys.prefix, 'dlls'))
- try:
- # real_prefix is defined when running inside virtual environments,
- # created with the **virtualenv** library.
- STD_LIB_DIRS.add(os.path.join(sys.real_prefix, 'dlls'))
- except AttributeError:
- # sys.base_exec_prefix is always defined, but in a virtual environment
- # created with the stdlib **venv** module, it points to the original
- # installation, if the virtual env is activated.
- try:
- STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, 'dlls'))
- except AttributeError:
- pass
-
- if platform.python_implementation() == 'PyPy':
- _root = os.path.join(sys.prefix, 'lib_pypy')
- STD_LIB_DIRS.add(_root)
- try:
- # real_prefix is defined when running inside virtualenv.
- STD_LIB_DIRS.add(os.path.join(sys.real_prefix, 'lib_pypy'))
- except AttributeError:
- pass
- del _root
- if os.name == 'posix':
- # Need the real prefix is we're under a virtualenv, otherwise
- # the usual one will do.
- try:
- prefix = sys.real_prefix
- except AttributeError:
- prefix = sys.prefix
-
- def _posix_path(path):
- base_python = 'python%d.%d' % sys.version_info[:2]
- return os.path.join(prefix, path, base_python)
-
- STD_LIB_DIRS.add(_posix_path('lib'))
- if sys.maxsize > 2**32:
- # This tries to fix a problem with /usr/lib64 builds,
- # where systems are running both 32-bit and 64-bit code
- # on the same machine, which reflects into the places where
- # standard library could be found. More details can be found
- # here http://bugs.python.org/issue1294959.
- # An easy reproducing case would be
- # https://github.com/PyCQA/pylint/issues/712#issuecomment-163178753
- STD_LIB_DIRS.add(_posix_path('lib64'))
-
- EXT_LIB_DIR = get_python_lib()
- IS_JYTHON = platform.python_implementation() == 'Jython'
- BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
-
-
- class NoSourceFile(Exception):
- """exception raised when we are not able to get a python
- source file for a precompiled file
- """
-
- def _normalize_path(path):
- return os.path.normcase(os.path.abspath(path))
-
-
- def _canonicalize_path(path):
- return os.path.realpath(os.path.expanduser(path))
-
-
- def _path_from_filename(filename, is_jython=IS_JYTHON):
- if not is_jython:
- if sys.version_info > (3, 0):
- return filename
- else:
- if filename.endswith(".pyc"):
- return filename[:-1]
- return filename
- head, has_pyclass, _ = filename.partition("$py.class")
- if has_pyclass:
- return head + ".py"
- return filename
-
-
- def _handle_blacklist(blacklist, dirnames, filenames):
- """remove files/directories in the black list
-
- dirnames/filenames are usually from os.walk
- """
- for norecurs in blacklist:
- if norecurs in dirnames:
- dirnames.remove(norecurs)
- elif norecurs in filenames:
- filenames.remove(norecurs)
-
-
- _NORM_PATH_CACHE = {}
-
- def _cache_normalize_path(path):
- """abspath with caching"""
- # _module_file calls abspath on every path in sys.path every time it's
- # called; on a larger codebase this easily adds up to half a second just
- # assembling path components. This cache alleviates that.
- try:
- return _NORM_PATH_CACHE[path]
- except KeyError:
- if not path: # don't cache result for ''
- return _normalize_path(path)
- result = _NORM_PATH_CACHE[path] = _normalize_path(path)
- return result
-
- def load_module_from_name(dotted_name, path=None, use_sys=True):
- """Load a Python module from its name.
-
- :type dotted_name: str
- :param dotted_name: python name of a module or package
-
- :type path: list or None
- :param path:
- optional list of path where the module or package should be
- searched (use sys.path if nothing or None is given)
-
- :type use_sys: bool
- :param use_sys:
- boolean indicating whether the sys.modules dictionary should be
- used or not
-
-
- :raise ImportError: if the module or package is not found
-
- :rtype: module
- :return: the loaded module
- """
- return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
-
-
- def load_module_from_modpath(parts, path=None, use_sys=1):
- """Load a python module from its split name.
-
- :type parts: list(str) or tuple(str)
- :param parts:
- python name of a module or package split on '.'
-
- :type path: list or None
- :param path:
- optional list of path where the module or package should be
- searched (use sys.path if nothing or None is given)
-
- :type use_sys: bool
- :param use_sys:
- boolean indicating whether the sys.modules dictionary should be used or not
-
- :raise ImportError: if the module or package is not found
-
- :rtype: module
- :return: the loaded module
- """
- if use_sys:
- try:
- return sys.modules['.'.join(parts)]
- except KeyError:
- pass
- modpath = []
- prevmodule = None
- for part in parts:
- modpath.append(part)
- curname = '.'.join(modpath)
- module = None
- if len(modpath) != len(parts):
- # even with use_sys=False, should try to get outer packages from sys.modules
- module = sys.modules.get(curname)
- elif use_sys:
- # because it may have been indirectly loaded through a parent
- module = sys.modules.get(curname)
- if module is None:
- mp_file, mp_filename, mp_desc = imp.find_module(part, path)
- module = imp.load_module(curname, mp_file, mp_filename, mp_desc)
- # mp_file still needs to be closed.
- if mp_file:
- mp_file.close()
- if prevmodule:
- setattr(prevmodule, part, module)
- _file = getattr(module, '__file__', '')
- prevmodule = module
- if not _file and util.is_namespace(curname):
- continue
- if not _file and len(modpath) != len(parts):
- raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]))
- path = [os.path.dirname(_file)]
- return module
-
-
- def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):
- """Load a Python module from it's path.
-
- :type filepath: str
- :param filepath: path to the python module or package
-
- :type path: list or None
- :param path:
- optional list of path where the module or package should be
- searched (use sys.path if nothing or None is given)
-
- :type use_sys: bool
- :param use_sys:
- boolean indicating whether the sys.modules dictionary should be
- used or not
-
-
- :raise ImportError: if the module or package is not found
-
- :rtype: module
- :return: the loaded module
- """
- modpath = modpath_from_file(filepath, extrapath)
- return load_module_from_modpath(modpath, path, use_sys)
-
-
- def check_modpath_has_init(path, mod_path):
- """check there are some __init__.py all along the way"""
- modpath = []
- for part in mod_path:
- modpath.append(part)
- path = os.path.join(path, part)
- if not _has_init(path):
- old_namespace = util.is_namespace('.'.join(modpath))
- if not old_namespace:
- return False
- return True
-
-
- def modpath_from_file_with_callback(filename, extrapath=None, is_package_cb=None):
- filename = _path_from_filename(filename)
- filename = os.path.realpath(os.path.expanduser(filename))
- base = os.path.splitext(filename)[0]
-
- if extrapath is not None:
- for path_ in six.moves.map(_canonicalize_path, extrapath):
- path = os.path.abspath(path_)
- if path and os.path.normcase(base[:len(path)]) == os.path.normcase(path):
- submodpath = [pkg for pkg in base[len(path):].split(os.sep)
- if pkg]
- if is_package_cb(path, submodpath[:-1]):
- return extrapath[path_].split('.') + submodpath
-
- for path in six.moves.map(_canonicalize_path, sys.path):
- path = _cache_normalize_path(path)
- if path and os.path.normcase(base).startswith(path):
- modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
- if is_package_cb(path, modpath[:-1]):
- return modpath
-
- raise ImportError('Unable to find module for %s in %s' % (
- filename, ', \n'.join(sys.path)))
-
-
-
- def modpath_from_file(filename, extrapath=None):
- """given a file path return the corresponding split module's name
- (i.e name of a module or package split on '.')
-
- :type filename: str
- :param filename: file's path for which we want the module's name
-
- :type extrapath: dict
- :param extrapath:
- optional extra search path, with path as key and package name for the path
- as value. This is usually useful to handle package split in multiple
- directories using __path__ trick.
-
-
- :raise ImportError:
- if the corresponding module's name has not been found
-
- :rtype: list(str)
- :return: the corresponding split module's name
- """
- return modpath_from_file_with_callback(filename, extrapath, check_modpath_has_init)
-
-
- def file_from_modpath(modpath, path=None, context_file=None):
- return file_info_from_modpath(modpath, path, context_file).location
-
- def file_info_from_modpath(modpath, path=None, context_file=None):
- """given a mod path (i.e. split module / package name), return the
- corresponding file, giving priority to source file over precompiled
- file if it exists
-
- :type modpath: list or tuple
- :param modpath:
- split module's name (i.e name of a module or package split
- on '.')
- (this means explicit relative imports that start with dots have
- empty strings in this list!)
-
- :type path: list or None
- :param path:
- optional list of path where the module or package should be
- searched (use sys.path if nothing or None is given)
-
- :type context_file: str or None
- :param context_file:
- context file to consider, necessary if the identifier has been
- introduced using a relative import unresolvable in the actual
- context (i.e. modutils)
-
- :raise ImportError: if there is no such module in the directory
-
- :rtype: (str or None, import type)
- :return:
- the path to the module's file or None if it's an integrated
- builtin module such as 'sys'
- """
- if context_file is not None:
- context = os.path.dirname(context_file)
- else:
- context = context_file
- if modpath[0] == 'xml':
- # handle _xmlplus
- try:
- return _spec_from_modpath(['_xmlplus'] + modpath[1:], path, context)
- except ImportError:
- return _spec_from_modpath(modpath, path, context)
- elif modpath == ['os', 'path']:
- # FIXME: currently ignoring search_path...
- return spec.ModuleSpec(name='os.path', location=os.path.__file__, module_type=imp.PY_SOURCE)
- return _spec_from_modpath(modpath, path, context)
-
-
- def get_module_part(dotted_name, context_file=None):
- """given a dotted name return the module part of the name :
-
- >>> get_module_part('astroid.as_string.dump')
- 'astroid.as_string'
-
- :type dotted_name: str
- :param dotted_name: full name of the identifier we are interested in
-
- :type context_file: str or None
- :param context_file:
- context file to consider, necessary if the identifier has been
- introduced using a relative import unresolvable in the actual
- context (i.e. modutils)
-
-
- :raise ImportError: if there is no such module in the directory
-
- :rtype: str or None
- :return:
- the module part of the name or None if we have not been able at
- all to import the given name
-
- XXX: deprecated, since it doesn't handle package precedence over module
- (see #10066)
- """
- # os.path trick
- if dotted_name.startswith('os.path'):
- return 'os.path'
- parts = dotted_name.split('.')
- if context_file is not None:
- # first check for builtin module which won't be considered latter
- # in that case (path != None)
- if parts[0] in BUILTIN_MODULES:
- if len(parts) > 2:
- raise ImportError(dotted_name)
- return parts[0]
- # don't use += or insert, we want a new list to be created !
- path = None
- starti = 0
- if parts[0] == '':
- assert context_file is not None, \
- 'explicit relative import, but no context_file?'
- path = [] # prevent resolving the import non-relatively
- starti = 1
- while parts[starti] == '': # for all further dots: change context
- starti += 1
- context_file = os.path.dirname(context_file)
- for i in range(starti, len(parts)):
- try:
- file_from_modpath(parts[starti:i+1], path=path,
- context_file=context_file)
- except ImportError:
- if i < max(1, len(parts) - 2):
- raise
- return '.'.join(parts[:i])
- return dotted_name
-
-
- def get_module_files(src_directory, blacklist, list_all=False):
- """given a package directory return a list of all available python
- module's files in the package and its subpackages
-
- :type src_directory: str
- :param src_directory:
- path of the directory corresponding to the package
-
- :type blacklist: list or tuple
- :param blacklist: iterable
- list of files or directories to ignore.
-
- :type list_all: bool
- :param list_all:
- get files from all paths, including ones without __init__.py
-
- :rtype: list
- :return:
- the list of all available python module's files in the package and
- its subpackages
- """
- files = []
- for directory, dirnames, filenames in os.walk(src_directory):
- if directory in blacklist:
- continue
- _handle_blacklist(blacklist, dirnames, filenames)
- # check for __init__.py
- if not list_all and '__init__.py' not in filenames:
- dirnames[:] = ()
- continue
- for filename in filenames:
- if _is_python_file(filename):
- src = os.path.join(directory, filename)
- files.append(src)
- return files
-
-
- def get_source_file(filename, include_no_ext=False):
- """given a python module's file name return the matching source file
- name (the filename will be returned identically if it's a already an
- absolute path to a python source file...)
-
- :type filename: str
- :param filename: python module's file name
-
-
- :raise NoSourceFile: if no source file exists on the file system
-
- :rtype: str
- :return: the absolute path of the source file if it exists
- """
- filename = os.path.abspath(_path_from_filename(filename))
- base, orig_ext = os.path.splitext(filename)
- for ext in PY_SOURCE_EXTS:
- source_path = '%s.%s' % (base, ext)
- if os.path.exists(source_path):
- return source_path
- if include_no_ext and not orig_ext and os.path.exists(base):
- return base
- raise NoSourceFile(filename)
-
-
- def is_python_source(filename):
- """
- rtype: bool
- return: True if the filename is a python source file
- """
- return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
-
-
- def is_standard_module(modname, std_path=None):
- """try to guess if a module is a standard python module (by default,
- see `std_path` parameter's description)
-
- :type modname: str
- :param modname: name of the module we are interested in
-
- :type std_path: list(str) or tuple(str)
- :param std_path: list of path considered has standard
-
-
- :rtype: bool
- :return:
- true if the module:
- - is located on the path listed in one of the directory in `std_path`
- - is a built-in module
- """
- modname = modname.split('.')[0]
- try:
- filename = file_from_modpath([modname])
- except ImportError:
- # import failed, i'm probably not so wrong by supposing it's
- # not standard...
- return False
- # modules which are not living in a file are considered standard
- # (sys and __builtin__ for instance)
- if filename is None:
- # we assume there are no namespaces in stdlib
- return not util.is_namespace(modname)
- filename = _normalize_path(filename)
- if filename.startswith(_cache_normalize_path(EXT_LIB_DIR)):
- return False
- if std_path is None:
- std_path = STD_LIB_DIRS
- for path in std_path:
- if filename.startswith(_cache_normalize_path(path)):
- return True
- return False
-
-
-
- def is_relative(modname, from_file):
- """return true if the given module name is relative to the given
- file name
-
- :type modname: str
- :param modname: name of the module we are interested in
-
- :type from_file: str
- :param from_file:
- path of the module from which modname has been imported
-
- :rtype: bool
- :return:
- true if the module has been imported relatively to `from_file`
- """
- if not os.path.isdir(from_file):
- from_file = os.path.dirname(from_file)
- if from_file in sys.path:
- return False
- try:
- stream, _, _ = imp.find_module(modname.split('.')[0], [from_file])
-
- # Close the stream to avoid ResourceWarnings.
- if stream:
- stream.close()
- return True
- except ImportError:
- return False
-
-
- # internal only functions #####################################################
-
- def _spec_from_modpath(modpath, path=None, context=None):
- """given a mod path (i.e. split module / package name), return the
- corresponding spec
-
- this function is used internally, see `file_from_modpath`'s
- documentation for more information
- """
- assert modpath
- location = None
- if context is not None:
- try:
- found_spec = spec.find_spec(modpath, [context])
- location = found_spec.location
- except ImportError:
- found_spec = spec.find_spec(modpath, path)
- location = found_spec.location
- else:
- found_spec = spec.find_spec(modpath, path)
- if found_spec.type == spec.ModuleType.PY_COMPILED:
- try:
- location = get_source_file(found_spec.location)
- return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
- except NoSourceFile:
- return found_spec._replace(location=location)
- elif found_spec.type == spec.ModuleType.C_BUILTIN:
- # integrated builtin module
- return found_spec._replace(location=None)
- elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:
- location = _has_init(found_spec.location)
- return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
- return found_spec
-
-
- def _is_python_file(filename):
- """return true if the given filename should be considered as a python file
-
- .pyc and .pyo are ignored
- """
- for ext in ('.py', '.so', '.pyd', '.pyw'):
- if filename.endswith(ext):
- return True
- return False
-
-
- def _has_init(directory):
- """if the given directory has a valid __init__ file, return its path,
- else return None
- """
- mod_or_pack = os.path.join(directory, '__init__')
- for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
- if os.path.exists(mod_or_pack + '.' + ext):
- return mod_or_pack + '.' + ext
- return None
-
-
- def is_namespace(specobj):
- return specobj.type == spec.ModuleType.PY_NAMESPACE
-
- def is_directory(specobj):
- return specobj.type == spec.ModuleType.PKG_DIRECTORY
|