|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781 |
- # -*- test-case-name: twisted.test.test_modules -*-
- # Copyright (c) Twisted Matrix Laboratories.
- # See LICENSE for details.
-
- """
- This module aims to provide a unified, object-oriented view of Python's
- runtime hierarchy.
-
- Python is a very dynamic language with wide variety of introspection utilities.
- However, these utilities can be hard to use, because there is no consistent
- API. The introspection API in python is made up of attributes (__name__,
- __module__, func_name, etc) on instances, modules, classes and functions which
- vary between those four types, utility modules such as 'inspect' which provide
- some functionality, the 'imp' module, the "compiler" module, the semantics of
- PEP 302 support, and setuptools, among other things.
-
- At the top, you have "PythonPath", an abstract representation of sys.path which
- includes methods to locate top-level modules, with or without loading them.
- The top-level exposed functions in this module for accessing the system path
- are "walkModules", "iterModules", and "getModule".
-
- From most to least specific, here are the objects provided::
-
- PythonPath # sys.path
- |
- v
- PathEntry # one entry on sys.path: an importer
- |
- v
- PythonModule # a module or package that can be loaded
- |
- v
- PythonAttribute # an attribute of a module (function or class)
- |
- v
- PythonAttribute # an attribute of a function or class
- |
- v
- ...
-
- Here's an example of idiomatic usage: this is what you would do to list all of
- the modules outside the standard library's python-files directory::
-
- import os
- stdlibdir = os.path.dirname(os.__file__)
-
- from twisted.python.modules import iterModules
-
- for modinfo in iterModules():
- if (modinfo.pathEntry.filePath.path != stdlibdir
- and not modinfo.isPackage()):
- print('unpackaged: %s: %s' % (
- modinfo.name, modinfo.filePath.path))
-
- @var theSystemPath: The very top of the Python object space.
- @type theSystemPath: L{PythonPath}
- """
-
-
- import inspect
- import sys
- import warnings
- import zipimport
-
- # let's try to keep path imports to a minimum...
- from os.path import dirname, split as splitpath
-
- from zope.interface import Interface, implementer
-
- from twisted.python.compat import nativeString
- from twisted.python.components import registerAdapter
- from twisted.python.filepath import FilePath, UnlistableError
- from twisted.python.reflect import namedAny
- from twisted.python.zippath import ZipArchive
-
- _nothing = object()
-
- PYTHON_EXTENSIONS = [".py"]
- OPTIMIZED_MODE = __doc__ is None
- if OPTIMIZED_MODE:
- PYTHON_EXTENSIONS.append(".pyo")
- else:
- PYTHON_EXTENSIONS.append(".pyc")
-
-
- def _isPythonIdentifier(string):
- """
- cheezy fake test for proper identifier-ness.
-
- @param string: a L{str} which might or might not be a valid python
- identifier.
- @return: True or False
- """
- textString = nativeString(string)
- return " " not in textString and "." not in textString and "-" not in textString
-
-
- def _isPackagePath(fpath):
- # Determine if a FilePath-like object is a Python package. TODO: deal with
- # __init__module.(so|dll|pyd)?
- extless = fpath.splitext()[0]
- basend = splitpath(extless)[1]
- return basend == "__init__"
-
-
- class _ModuleIteratorHelper:
- """
- This mixin provides common behavior between python module and path entries,
- since the mechanism for searching sys.path and __path__ attributes is
- remarkably similar.
- """
-
- def iterModules(self):
- """
- Loop over the modules present below this entry or package on PYTHONPATH.
-
- For modules which are not packages, this will yield nothing.
-
- For packages and path entries, this will only yield modules one level
- down; i.e. if there is a package a.b.c, iterModules on a will only
- return a.b. If you want to descend deeply, use walkModules.
-
- @return: a generator which yields PythonModule instances that describe
- modules which can be, or have been, imported.
- """
- yielded = {}
- if not self.filePath.exists():
- return
-
- for placeToLook in self._packagePaths():
- try:
- children = sorted(placeToLook.children())
- except UnlistableError:
- continue
-
- for potentialTopLevel in children:
- ext = potentialTopLevel.splitext()[1]
- potentialBasename = potentialTopLevel.basename()[: -len(ext)]
- if ext in PYTHON_EXTENSIONS:
- # TODO: this should be a little choosier about which path entry
- # it selects first, and it should do all the .so checking and
- # crud
- if not _isPythonIdentifier(potentialBasename):
- continue
- modname = self._subModuleName(potentialBasename)
- if modname.split(".")[-1] == "__init__":
- # This marks the directory as a package so it can't be
- # a module.
- continue
- if modname not in yielded:
- yielded[modname] = True
- pm = PythonModule(modname, potentialTopLevel, self._getEntry())
- assert pm != self
- yield pm
- else:
- if (
- ext
- or not _isPythonIdentifier(potentialBasename)
- or not potentialTopLevel.isdir()
- ):
- continue
- modname = self._subModuleName(potentialTopLevel.basename())
- for ext in PYTHON_EXTENSIONS:
- initpy = potentialTopLevel.child("__init__" + ext)
- if initpy.exists() and modname not in yielded:
- yielded[modname] = True
- pm = PythonModule(modname, initpy, self._getEntry())
- assert pm != self
- yield pm
- break
-
- def walkModules(self, importPackages=False):
- """
- Similar to L{iterModules}, this yields self, and then every module in my
- package or entry, and every submodule in each package or entry.
-
- In other words, this is deep, and L{iterModules} is shallow.
- """
- yield self
- for package in self.iterModules():
- yield from package.walkModules(importPackages=importPackages)
-
- def _subModuleName(self, mn):
- """
- This is a hook to provide packages with the ability to specify their names
- as a prefix to submodules here.
- """
- return mn
-
- def _packagePaths(self):
- """
- Implement in subclasses to specify where to look for modules.
-
- @return: iterable of FilePath-like objects.
- """
- raise NotImplementedError()
-
- def _getEntry(self):
- """
- Implement in subclasses to specify what path entry submodules will come
- from.
-
- @return: a PathEntry instance.
- """
- raise NotImplementedError()
-
- def __getitem__(self, modname):
- """
- Retrieve a module from below this path or package.
-
- @param modname: a str naming a module to be loaded. For entries, this
- is a top-level, undotted package name, and for packages it is the name
- of the module without the package prefix. For example, if you have a
- PythonModule representing the 'twisted' package, you could use::
-
- twistedPackageObj['python']['modules']
-
- to retrieve this module.
-
- @raise KeyError: if the module is not found.
-
- @return: a PythonModule.
- """
- for module in self.iterModules():
- if module.name == self._subModuleName(modname):
- return module
- raise KeyError(modname)
-
- def __iter__(self):
- """
- Implemented to raise NotImplementedError for clarity, so that attempting to
- loop over this object won't call __getitem__.
-
- Note: in the future there might be some sensible default for iteration,
- like 'walkEverything', so this is deliberately untested and undefined
- behavior.
- """
- raise NotImplementedError()
-
-
- class PythonAttribute:
- """
- I represent a function, class, or other object that is present.
-
- @ivar name: the fully-qualified python name of this attribute.
-
- @ivar onObject: a reference to a PythonModule or other PythonAttribute that
- is this attribute's logical parent.
-
- @ivar name: the fully qualified python name of the attribute represented by
- this class.
- """
-
- def __init__(self, name, onObject, loaded, pythonValue):
- """
- Create a PythonAttribute. This is a private constructor. Do not construct
- me directly, use PythonModule.iterAttributes.
-
- @param name: the FQPN
- @param onObject: see ivar
- @param loaded: always True, for now
- @param pythonValue: the value of the attribute we're pointing to.
- """
- self.name: str = name
- self.onObject = onObject
- self._loaded = loaded
- self.pythonValue = pythonValue
-
- def __repr__(self) -> str:
- return f"PythonAttribute<{self.name!r}>"
-
- def isLoaded(self):
- """
- Return a boolean describing whether the attribute this describes has
- actually been loaded into memory by importing its module.
-
- Note: this currently always returns true; there is no Python parser
- support in this module yet.
- """
- return self._loaded
-
- def load(self, default=_nothing):
- """
- Load the value associated with this attribute.
-
- @return: an arbitrary Python object, or 'default' if there is an error
- loading it.
- """
- return self.pythonValue
-
- def iterAttributes(self):
- for name, val in inspect.getmembers(self.load()):
- yield PythonAttribute(self.name + "." + name, self, True, val)
-
-
- class PythonModule(_ModuleIteratorHelper):
- """
- Representation of a module which could be imported from sys.path.
-
- @ivar name: the fully qualified python name of this module.
-
- @ivar filePath: a FilePath-like object which points to the location of this
- module.
-
- @ivar pathEntry: a L{PathEntry} instance which this module was located
- from.
- """
-
- def __init__(self, name, filePath, pathEntry):
- """
- Create a PythonModule. Do not construct this directly, instead inspect a
- PythonPath or other PythonModule instances.
-
- @param name: see ivar
- @param filePath: see ivar
- @param pathEntry: see ivar
- """
- _name = nativeString(name)
- assert not _name.endswith(".__init__")
- self.name: str = _name
- self.filePath = filePath
- self.parentPath = filePath.parent()
- self.pathEntry = pathEntry
-
- def _getEntry(self):
- return self.pathEntry
-
- def __repr__(self) -> str:
- """
- Return a string representation including the module name.
- """
- return f"PythonModule<{self.name!r}>"
-
- def isLoaded(self):
- """
- Determine if the module is loaded into sys.modules.
-
- @return: a boolean: true if loaded, false if not.
- """
- return self.pathEntry.pythonPath.moduleDict.get(self.name) is not None
-
- def iterAttributes(self):
- """
- List all the attributes defined in this module.
-
- Note: Future work is planned here to make it possible to list python
- attributes on a module without loading the module by inspecting ASTs or
- bytecode, but currently any iteration of PythonModule objects insists
- they must be loaded, and will use inspect.getmodule.
-
- @raise NotImplementedError: if this module is not loaded.
-
- @return: a generator yielding PythonAttribute instances describing the
- attributes of this module.
- """
- if not self.isLoaded():
- raise NotImplementedError(
- "You can't load attributes from non-loaded modules yet."
- )
- for name, val in inspect.getmembers(self.load()):
- yield PythonAttribute(self.name + "." + name, self, True, val)
-
- def isPackage(self):
- """
- Returns true if this module is also a package, and might yield something
- from iterModules.
- """
- return _isPackagePath(self.filePath)
-
- def load(self, default=_nothing):
- """
- Load this module.
-
- @param default: if specified, the value to return in case of an error.
-
- @return: a genuine python module.
-
- @raise Exception: Importing modules is a risky business;
- the erorrs of any code run at module scope may be raised from here, as
- well as ImportError if something bizarre happened to the system path
- between the discovery of this PythonModule object and the attempt to
- import it. If you specify a default, the error will be swallowed
- entirely, and not logged.
-
- @rtype: types.ModuleType.
- """
- try:
- return self.pathEntry.pythonPath.moduleLoader(self.name)
- except BaseException: # this needs more thought...
- if default is not _nothing:
- return default
- raise
-
- def __eq__(self, other: object) -> bool:
- """
- PythonModules with the same name are equal.
- """
- if isinstance(other, PythonModule):
- return other.name == self.name
- return NotImplemented
-
- def walkModules(self, importPackages=False):
- if importPackages and self.isPackage():
- self.load()
- return super().walkModules(importPackages=importPackages)
-
- def _subModuleName(self, mn):
- """
- submodules of this module are prefixed with our name.
- """
- return self.name + "." + mn
-
- def _packagePaths(self):
- """
- Yield a sequence of FilePath-like objects which represent path segments.
- """
- if not self.isPackage():
- return
- if self.isLoaded():
- load = self.load()
- if hasattr(load, "__path__"):
- for fn in load.__path__:
- if fn == self.parentPath.path:
- # this should _really_ exist.
- assert self.parentPath.exists()
- yield self.parentPath
- else:
- smp = self.pathEntry.pythonPath._smartPath(fn)
- if smp.exists():
- yield smp
- else:
- yield self.parentPath
-
-
- class PathEntry(_ModuleIteratorHelper):
- """
- I am a proxy for a single entry on sys.path.
-
- @ivar filePath: a FilePath-like object pointing at the filesystem location
- or archive file where this path entry is stored.
-
- @ivar pythonPath: a PythonPath instance.
- """
-
- def __init__(self, filePath, pythonPath):
- """
- Create a PathEntry. This is a private constructor.
- """
- self.filePath = filePath
- self.pythonPath = pythonPath
-
- def _getEntry(self):
- return self
-
- def __repr__(self) -> str:
- return f"PathEntry<{self.filePath!r}>"
-
- def _packagePaths(self):
- yield self.filePath
-
-
- class IPathImportMapper(Interface):
- """
- This is an internal interface, used to map importers to factories for
- FilePath-like objects.
- """
-
- def mapPath(pathLikeString):
- """
- Return a FilePath-like object.
-
- @param pathLikeString: a path-like string, like one that might be
- passed to an import hook.
-
- @return: a L{FilePath}, or something like it (currently only a
- L{ZipPath}, but more might be added later).
- """
-
-
- @implementer(IPathImportMapper)
- class _DefaultMapImpl:
- """Wrapper for the default importer, i.e. None."""
-
- def mapPath(self, fsPathString):
- return FilePath(fsPathString)
-
-
- _theDefaultMapper = _DefaultMapImpl()
-
-
- @implementer(IPathImportMapper)
- class _ZipMapImpl:
- """IPathImportMapper implementation for zipimport.ZipImporter."""
-
- def __init__(self, importer):
- self.importer = importer
-
- def mapPath(self, fsPathString):
- """
- Map the given FS path to a ZipPath, by looking at the ZipImporter's
- "archive" attribute and using it as our ZipArchive root, then walking
- down into the archive from there.
-
- @return: a L{zippath.ZipPath} or L{zippath.ZipArchive} instance.
- """
- za = ZipArchive(self.importer.archive)
- myPath = FilePath(self.importer.archive)
- itsPath = FilePath(fsPathString)
- if myPath == itsPath:
- return za
- # This is NOT a general-purpose rule for sys.path or __file__:
- # zipimport specifically uses regular OS path syntax in its
- # pathnames, even though zip files specify that slashes are always
- # the separator, regardless of platform.
- segs = itsPath.segmentsFrom(myPath)
- zp = za
- for seg in segs:
- zp = zp.child(seg)
- return zp
-
-
- registerAdapter(_ZipMapImpl, zipimport.zipimporter, IPathImportMapper)
-
-
- def _defaultSysPathFactory():
- """
- Provide the default behavior of PythonPath's sys.path factory, which is to
- return the current value of sys.path.
-
- @return: L{sys.path}
- """
- return sys.path
-
-
- class PythonPath:
- """
- I represent the very top of the Python object-space, the module list in
- C{sys.path} and the modules list in C{sys.modules}.
-
- @ivar _sysPath: A sequence of strings like C{sys.path}. This attribute is
- read-only.
-
- @ivar sysPath: The current value of the module search path list.
- @type sysPath: C{list}
-
- @ivar moduleDict: A dictionary mapping string module names to module
- objects, like C{sys.modules}.
-
- @ivar sysPathHooks: A list of PEP-302 path hooks, like C{sys.path_hooks}.
-
- @ivar moduleLoader: A function that takes a fully-qualified python name and
- returns a module, like L{twisted.python.reflect.namedAny}.
- """
-
- def __init__(
- self,
- sysPath=None,
- moduleDict=sys.modules,
- sysPathHooks=sys.path_hooks,
- importerCache=sys.path_importer_cache,
- moduleLoader=namedAny,
- sysPathFactory=None,
- ):
- """
- Create a PythonPath. You almost certainly want to use
- modules.theSystemPath, or its aliased methods, rather than creating a
- new instance yourself, though.
-
- All parameters are optional, and if unspecified, will use 'system'
- equivalents that makes this PythonPath like the global L{theSystemPath}
- instance.
-
- @param sysPath: a sys.path-like list to use for this PythonPath, to
- specify where to load modules from.
-
- @param moduleDict: a sys.modules-like dictionary to use for keeping
- track of what modules this PythonPath has loaded.
-
- @param sysPathHooks: sys.path_hooks-like list of PEP-302 path hooks to
- be used for this PythonPath, to determie which importers should be
- used.
-
- @param importerCache: a sys.path_importer_cache-like list of PEP-302
- importers. This will be used in conjunction with the given
- sysPathHooks.
-
- @param moduleLoader: a module loader function which takes a string and
- returns a module. That is to say, it is like L{namedAny} - *not* like
- L{__import__}.
-
- @param sysPathFactory: a 0-argument callable which returns the current
- value of a sys.path-like list of strings. Specify either this, or
- sysPath, not both. This alternative interface is provided because the
- way the Python import mechanism works, you can re-bind the 'sys.path'
- name and that is what is used for current imports, so it must be a
- factory rather than a value to deal with modification by rebinding
- rather than modification by mutation. Note: it is not recommended to
- rebind sys.path. Although this mechanism can deal with that, it is a
- subtle point which some tools that it is easy for tools which interact
- with sys.path to miss.
- """
- if sysPath is not None:
- sysPathFactory = lambda: sysPath
- elif sysPathFactory is None:
- sysPathFactory = _defaultSysPathFactory
- self._sysPathFactory = sysPathFactory
- self._sysPath = sysPath
- self.moduleDict = moduleDict
- self.sysPathHooks = sysPathHooks
- self.importerCache = importerCache
- self.moduleLoader = moduleLoader
-
- @property
- def sysPath(self):
- """
- Retrieve the current value of the module search path list.
- """
- return self._sysPathFactory()
-
- def _findEntryPathString(self, modobj):
- """
- Determine where a given Python module object came from by looking at path
- entries.
- """
- topPackageObj = modobj
- while "." in topPackageObj.__name__:
- topPackageObj = self.moduleDict[
- ".".join(topPackageObj.__name__.split(".")[:-1])
- ]
- if _isPackagePath(FilePath(topPackageObj.__file__)):
- # if package 'foo' is on sys.path at /a/b/foo, package 'foo's
- # __file__ will be /a/b/foo/__init__.py, and we are looking for
- # /a/b here, the path-entry; so go up two steps.
- rval = dirname(dirname(topPackageObj.__file__))
- else:
- # the module is completely top-level, not within any packages. The
- # path entry it's on is just its dirname.
- rval = dirname(topPackageObj.__file__)
-
- # There are probably some awful tricks that an importer could pull
- # which would break this, so let's just make sure... it's a loaded
- # module after all, which means that its path MUST be in
- # path_importer_cache according to PEP 302 -glyph
- if rval not in self.importerCache:
- warnings.warn(
- "%s (for module %s) not in path importer cache "
- "(PEP 302 violation - check your local configuration)."
- % (rval, modobj.__name__),
- stacklevel=3,
- )
-
- return rval
-
- def _smartPath(self, pathName):
- """
- Given a path entry from sys.path which may refer to an importer,
- return the appropriate FilePath-like instance.
-
- @param pathName: a str describing the path.
-
- @return: a FilePath-like object.
- """
- importr = self.importerCache.get(pathName, _nothing)
- if importr is _nothing:
- for hook in self.sysPathHooks:
- try:
- importr = hook(pathName)
- except ImportError:
- pass
- if importr is _nothing: # still
- importr = None
- return IPathImportMapper(importr, _theDefaultMapper).mapPath(pathName)
-
- def iterEntries(self):
- """
- Iterate the entries on my sysPath.
-
- @return: a generator yielding PathEntry objects
- """
- for pathName in self.sysPath:
- fp = self._smartPath(pathName)
- yield PathEntry(fp, self)
-
- def __getitem__(self, modname):
- """
- Get a python module by its given fully-qualified name.
-
- @param modname: The fully-qualified Python module name to load.
-
- @type modname: C{str}
-
- @return: an object representing the module identified by C{modname}
-
- @rtype: L{PythonModule}
-
- @raise KeyError: if the module name is not a valid module name, or no
- such module can be identified as loadable.
- """
- # See if the module is already somewhere in Python-land.
- moduleObject = self.moduleDict.get(modname)
- if moduleObject is not None:
- # we need 2 paths; one of the path entry and one for the module.
- pe = PathEntry(
- self._smartPath(self._findEntryPathString(moduleObject)), self
- )
- mp = self._smartPath(moduleObject.__file__)
- return PythonModule(modname, mp, pe)
-
- # Recurse if we're trying to get a submodule.
- if "." in modname:
- pkg = self
- for name in modname.split("."):
- pkg = pkg[name]
- return pkg
-
- # Finally do the slowest possible thing and iterate
- for module in self.iterModules():
- if module.name == modname:
- return module
- raise KeyError(modname)
-
- def __contains__(self, module):
- """
- Check to see whether or not a module exists on my import path.
-
- @param module: The name of the module to look for on my import path.
- @type module: C{str}
- """
- try:
- self.__getitem__(module)
- return True
- except KeyError:
- return False
-
- def __repr__(self) -> str:
- """
- Display my sysPath and moduleDict in a string representation.
- """
- return f"PythonPath({self.sysPath!r},{self.moduleDict!r})"
-
- def iterModules(self):
- """
- Yield all top-level modules on my sysPath.
- """
- for entry in self.iterEntries():
- yield from entry.iterModules()
-
- def walkModules(self, importPackages=False):
- """
- Similar to L{iterModules}, this yields every module on the path, then every
- submodule in each package or entry.
- """
- for package in self.iterModules():
- yield from package.walkModules(importPackages=False)
-
-
- theSystemPath = PythonPath()
-
-
- def walkModules(importPackages=False):
- """
- Deeply iterate all modules on the global python path.
-
- @param importPackages: Import packages as they are seen.
- """
- return theSystemPath.walkModules(importPackages=importPackages)
-
-
- def iterModules():
- """
- Iterate all modules and top-level packages on the global Python path, but
- do not descend into packages.
- """
- return theSystemPath.iterModules()
-
-
- def getModule(moduleName):
- """
- Retrieve a module from the system path.
- """
- return theSystemPath[moduleName]
|