Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework reparenting #737

Closed
wants to merge 4 commits into from
Closed

Rework reparenting #737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
190b5a7
- Parse module imports and store them into Module.imports attribute.
tristanlatr Sep 29, 2023
cd76414
Move the re exporting to post processing
tristanlatr Sep 30, 2023
a3ad4eb
Fix mypy and simplify
tristanlatr Sep 30, 2023
416d1d0
Fix space
tristanlatr Sep 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 155 additions & 79 deletions pydoctor/astbuilder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,130 @@ def extract_final_subscript(annotation: ast.Subscript) -> ast.expr:
assert isinstance(ann_slice, ast.expr)
return ann_slice

def getModuleExports(mod: model.Module) -> Collection[str]:
# Fetch names to export.
exports = mod.all
if exports is None:
exports = []
return exports

def getPublicNames(mod: model.Module) -> Collection[str]:
"""
Get all names to import when wildcardm importing the given module:
use __all__ if available, otherwise take all names that are not private.
"""
names = mod.all
if names is None:
names = [
name
for name in chain(mod.contents.keys(),
mod._localNameToFullName_map.keys())
if not name.startswith('_')
]
return names

# post-processes

def _handleReExport(new_parent: 'model.Module',
origin_name: str,
as_name: str,
origin_module: model.Module,
linenumber: int) -> None:
"""
Move re-exported objects into module C{new_parent}.
"""
modname = origin_module.fullName()

# In case of duplicates names, we can't rely on resolveName,
# So we use content.get first to resolve non-alias names.
ob = origin_module.contents.get(origin_name) or origin_module.resolveName(origin_name)
if ob is None:
new_parent.report("cannot resolve re-exported name: "
f'\'{modname}.{origin_name}\'',
lineno_offset=linenumber)
else:
if origin_module.all is None or origin_name not in origin_module.all:
if as_name != ob.name:
new_parent.system.msg(
"astbuilder",
f"moving {ob.fullName()!r} into {new_parent.fullName()!r} as {as_name!r}")
else:
new_parent.system.msg(
"astbuilder",
f"moving {ob.fullName()!r} into {new_parent.fullName()!r}")
ob.reparent(new_parent, as_name)
else:
new_parent.system.msg(
"astbuilder",
f"not moving {ob.fullName()} into {new_parent.fullName()}, "
f"because {origin_name!r} is already exported in {modname}.__all__")

def processReExports(system: model.System) -> None:
for mod in tuple(system.objectsOfType(model.Module)):
exports = getModuleExports(mod)
for imported_name in mod.imports:
local_name = imported_name.name
orgname = imported_name.orgname
orgmodule = imported_name.orgmodule
if local_name != '*' and (not orgname or local_name not in exports):
continue

origin = system.modules.get(orgmodule)
if origin is None:
if orgmodule.split('.', 1)[0] in system.root_names:
msg = f"cannot resolve origin module of re-exported name: {orgname or local_name!r}"
if orgname and local_name!=orgname:
msg += f" as {local_name!r}"
msg += f" from origin module {imported_name.orgmodule!r}"
mod.report(msg, lineno_offset=imported_name.linenumber)
elif local_name != '*':
if orgname:
# only 'import from' statements can be used in re-exporting currently.
_handleReExport(mod, orgname, local_name, origin,
linenumber=imported_name.linenumber)
else:
for n in getPublicNames(origin):
if n in exports:
_handleReExport(mod, n, n, origin,
linenumber=imported_name.linenumber)


def postProcessClasses(system: model.System) -> None:
for cls in system.objectsOfType(model.Class):
# Initiate the MROs
cls._init_mro()
# Lookup of constructors
cls._init_constructors()

# Compute subclasses
for b in cls.baseobjects:
if b is not None:
b.subclasses.append(cls)

# Checking whether the class is an exception
if model.is_exception(cls):
cls.kind = model.DocumentableKind.EXCEPTION

def postProcessAttributes(system:model.System) -> None:
for attrib in system.objectsOfType(model.Attribute):
_inherits_instance_variable_kind(attrib)

def _inherits_instance_variable_kind(attr: model.Attribute) -> None:
"""
If any of the inherited members of a class variable is an instance variable,
then the subclass' class variable become an instance variable as well.
"""
if attr.kind is not model.DocumentableKind.CLASS_VARIABLE:
return
docsources = attr.docsources()
next(docsources)
for inherited in docsources:
if inherited.kind is model.DocumentableKind.INSTANCE_VARIABLE:
attr.kind = model.DocumentableKind.INSTANCE_VARIABLE
break

# main ast visitor

class ModuleVistor(NodeVisitor):

def __init__(self, builder: 'ASTBuilder', module: model.Module):
Expand Down Expand Up @@ -325,115 +449,58 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
assert modname is not None

if node.names[0].name == '*':
self._importAll(modname)
self._importAll(modname, linenumber=node.lineno)
else:
self._importNames(modname, node.names)
self._importNames(modname, node.names, linenumber=node.lineno)

def _importAll(self, modname: str) -> None:
def _importAll(self, modname: str, linenumber:int) -> None:
"""Handle a C{from <modname> import *} statement."""

ctx = self.builder.current
if isinstance(ctx, model.Module):
ctx.imports.append(model.Import('*', modname,
linenumber=linenumber, orgname='*'))

mod = self.system.getProcessedModule(modname)
if mod is None:
# We don't have any information about the module, so we don't know
# what names to import.
self.builder.current.report(f"import * from unknown {modname}", thresh=1)
ctx.report(f"import * from unknown {modname}", thresh=1)
return

self.builder.current.report(f"import * from {modname}", thresh=1)

# Get names to import: use __all__ if available, otherwise take all
# names that are not private.
names = mod.all
if names is None:
names = [
name
for name in chain(mod.contents.keys(),
mod._localNameToFullName_map.keys())
if not name.startswith('_')
]

# Fetch names to export.
exports = self._getCurrentModuleExports()
ctx.report(f"import * from {modname}", thresh=1)

# Add imported names to our module namespace.
assert isinstance(self.builder.current, model.CanContainImportsDocumentable)
_localNameToFullName = self.builder.current._localNameToFullName_map
expandName = mod.expandName
for name in names:

if self._handleReExport(exports, name, name, mod) is True:
continue

for name in getPublicNames(mod):
_localNameToFullName[name] = expandName(name)

def _getCurrentModuleExports(self) -> Collection[str]:
# Fetch names to export.
current = self.builder.current
if isinstance(current, model.Module):
exports = current.all
if exports is None:
exports = []
else:
# Don't export names imported inside classes or functions.
exports = []
return exports

def _handleReExport(self, curr_mod_exports:Collection[str],
origin_name:str, as_name:str,
origin_module:model.Module) -> bool:
"""
Move re-exported objects into current module.

@returns: True if the imported name has been sucessfully re-exported.
"""
# Move re-exported objects into current module.
current = self.builder.current
modname = origin_module.fullName()
if as_name in curr_mod_exports:
# In case of duplicates names, we can't rely on resolveName,
# So we use content.get first to resolve non-alias names.
ob = origin_module.contents.get(origin_name) or origin_module.resolveName(origin_name)
if ob is None:
current.report("cannot resolve re-exported name :"
f'{modname}.{origin_name}', thresh=1)
else:
if origin_module.all is None or origin_name not in origin_module.all:
self.system.msg(
"astbuilder",
"moving %r into %r" % (ob.fullName(), current.fullName())
)
# Must be a Module since the exports is set to an empty list if it's not.
assert isinstance(current, model.Module)
ob.reparent(current, as_name)
return True
return False

def _importNames(self, modname: str, names: Iterable[ast.alias]) -> None:
def _importNames(self, modname: str, names: Iterable[ast.alias], linenumber:int) -> None:
"""Handle a C{from <modname> import <names>} statement."""

# Process the module we're importing from.
mod = self.system.getProcessedModule(modname)

# Fetch names to export.
exports = self._getCurrentModuleExports()

current = self.builder.current
assert isinstance(current, model.CanContainImportsDocumentable)
_localNameToFullName = current._localNameToFullName_map
is_module = isinstance(current, model.Module)

for al in names:
orgname, asname = al.name, al.asname
if asname is None:
asname = orgname

if mod is not None and self._handleReExport(exports, orgname, asname, mod) is True:
continue

# If we're importing from a package, make sure imported modules
# are processed (getProcessedModule() ignores non-modules).
if isinstance(mod, model.Package):
self.system.getProcessedModule(f'{modname}.{orgname}')

_localNameToFullName[asname] = f'{modname}.{orgname}'
if is_module:
cast(model.Module,
current).imports.append(model.Import(asname, modname,
orgname=orgname, linenumber=linenumber))

def visit_Import(self, node: ast.Import) -> None:
"""Process an import statement.
Expand All @@ -448,16 +515,23 @@ def visit_Import(self, node: ast.Import) -> None:
(dotted_name, as_name) where as_name is None if there was no 'as foo'
part of the statement.
"""
if not isinstance(self.builder.current, model.CanContainImportsDocumentable):
ctx = self.builder.current
if not isinstance(ctx, model.CanContainImportsDocumentable):
# processing import statement in odd context
return
_localNameToFullName = self.builder.current._localNameToFullName_map
_localNameToFullName = ctx._localNameToFullName_map
is_module = isinstance(ctx, model.Module)

for al in node.names:
targetname, asname = al.name, al.asname
if asname is None:
# we're keeping track of all defined names
asname = targetname = targetname.split('.')[0]
_localNameToFullName[asname] = targetname
if is_module:
cast(model.Module,
ctx).imports.append(model.Import(asname, targetname,
linenumber=node.lineno))

def _handleOldSchoolMethodDecoration(self, target: str, expr: Optional[ast.expr]) -> bool:
if not isinstance(expr, ast.Call):
Expand Down Expand Up @@ -1292,4 +1366,6 @@ def parseDocformat(node: ast.Assign, mod: model.Module) -> None:

def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None:
r.register_astbuilder_visitor(TypeAliasVisitorExt)
r.register_post_processor(model.defaultPostProcess, priority=200)
r.register_post_processor(processReExports, priority=250)
r.register_post_processor(postProcessClasses, priority=200)
r.register_post_processor(postProcessAttributes, priority=200)
Loading