diff --git a/pydoctor_primer/globals.py b/pydoctor_primer/globals.py index 01c7755..b6411d7 100644 --- a/pydoctor_primer/globals.py +++ b/pydoctor_primer/globals.py @@ -15,12 +15,6 @@ class _Args: new: str | None old: str | None repo: str | None - type_checker: str - mypyc_compile_level: int | None - - custom_typeshed_repo: str - new_typeshed: str | None - old_typeshed: str | None additional_flags: list[str] @@ -38,7 +32,6 @@ class _Args: old_success: bool # modes group - coverage: bool bisect: bool bisect_output: str | None validate_expected_success: bool @@ -59,58 +52,32 @@ class _Args: def parse_options(argv: list[str]) -> _Args: parser = argparse.ArgumentParser() - type_checker_group = parser.add_argument_group("type checker") + type_checker_group = parser.add_argument_group("pydoctor") type_checker_group.add_argument( "--new", help=( - "new type checker version, defaults to HEAD " + "new pydoctor version, defaults to HEAD " "(pypi version, anything commit-ish, or isoformatted date)" ), ) type_checker_group.add_argument( "--old", help=( - "old type checker version, defaults to latest tag " + "old pydoctor version, defaults to latest tag " "(pypi version, anything commit-ish, or isoformatted date)" ), ) - type_checker_group.add_argument( - "--type-checker", - default="mypy", - choices=["mypy", "pyright"], - help="type checker to use", - ) type_checker_group.add_argument( "--repo", help=( - "type checker repo to use (passed to git clone. if unspecified, we first try pypi, " + "pydoctor repo to use (passed to git clone. if unspecified, we first try pypi, " "then fall back to github)" ), ) - type_checker_group.add_argument( - "--mypyc-compile-level", - default=None, - type=int, - help="Compile mypy with the given mypyc optimisation level", - ) - - type_checker_group.add_argument( - "--custom-typeshed-repo", - default="https://github.com/python/typeshed", - help="typeshed repo to use (passed to git clone)", - ) - type_checker_group.add_argument( - "--new-typeshed", - help="new typeshed version, defaults to vendored (commit-ish or isoformatted date)", - ) - type_checker_group.add_argument( - "--old-typeshed", - help="old typeshed version, defaults to vendored (commit-ish, or isoformatted date)", - ) type_checker_group.add_argument( "--additional-flags", - help="additional flags to pass to mypy", + help="additional flags to pass to pydoctor", nargs="*", default=[], ) @@ -123,16 +90,15 @@ def parse_options(argv: list[str]) -> _Args: "-p", "--local-project", help=( - "run only on the given file or directory. if a single file, supports a " - "'# flags: ...' comment, like mypy unit tests" + "run pydoctor only on the given file or directory" ), ) proj_group.add_argument( "--expected-success", action="store_true", help=( - "filter to hardcoded subset of projects where some recent mypy version succeeded " - "aka are committed to the mypy way of life. also look at: --old-success" + "filter to hardcoded subset of projects where some recent pydoctor " + "version succeeded. also look at: --old-success" ), ) proj_group.add_argument( @@ -160,18 +126,15 @@ def parse_options(argv: list[str]) -> _Args: output_group.add_argument( "--old-success", action="store_true", - help="only output a result for a project if the old mypy run was successful", + help="only output a result for a project if the old pydoctor run was successful", ) modes_group = parser.add_argument_group("modes") modes_group.add_argument( - "--coverage", action="store_true", help="count files and lines covered" - ) - modes_group.add_argument( - "--bisect", action="store_true", help="find first mypy revision to introduce a difference" + "--bisect", action="store_true", help="find first pydoctor revision to introduce a difference" ) modes_group.add_argument( - "--bisect-output", help="find first mypy revision with output matching given regex" + "--bisect-output", help="find first pydoctor revision with output matching given regex" ) modes_group.add_argument( "--validate-expected-success", action="store_true", help=argparse.SUPPRESS @@ -191,7 +154,7 @@ def parse_options(argv: list[str]) -> _Args: primer_group.add_argument("--debug", action="store_true", help="print commands as they run") primer_group.add_argument( "--base-dir", - default=Path(pydoctor_primer.utils.TEMP_DIR) / "mypy_primer", + default=Path(pydoctor_primer.utils.TEMP_DIR) / "pydoctor_primer", type=Path, help="dir to store repos and venvs", ) diff --git a/pydoctor_primer/main.py b/pydoctor_primer/main.py index 6c01cc1..542ef9c 100644 --- a/pydoctor_primer/main.py +++ b/pydoctor_primer/main.py @@ -18,89 +18,37 @@ from pydoctor_primer.globals import ctx, parse_options_and_set_ctx from pydoctor_primer.model import Project, TypeCheckResult from pydoctor_primer.projects import get_projects -from pydoctor_primer.type_checker import setup_mypy, setup_pyright, setup_typeshed +from pydoctor_primer.type_checker import setup_pydoctor from pydoctor_primer.utils import Style, debug_print, line_count, run, strip_colour_code T = TypeVar("T") -async def setup_new_and_old_mypy( - new_mypy_revision: RevisionLike, old_mypy_revision: RevisionLike +async def setup_new_and_old_pydoctor( + new_revision: RevisionLike, old_revision: RevisionLike ) -> tuple[Path, Path]: - new_mypy, old_mypy = await asyncio.gather( - setup_mypy( - ctx.get().base_dir / "new_mypy", - new_mypy_revision, + new_exe, old_exe = await asyncio.gather( + setup_pydoctor( + ctx.get().base_dir / "new_pydoctor", + new_revision, repo=ctx.get().repo, - mypyc_compile_level=ctx.get().mypyc_compile_level, ), - setup_mypy( - ctx.get().base_dir / "old_mypy", - old_mypy_revision, + setup_pydoctor( + ctx.get().base_dir / "old_pydoctor", + old_revision, repo=ctx.get().repo, - mypyc_compile_level=ctx.get().mypyc_compile_level, ), ) if ctx.get().debug: (new_version, _), (old_version, _) = await asyncio.gather( - run([str(new_mypy), "--version"], output=True), - run([str(old_mypy), "--version"], output=True), + run([str(new_exe), "--version"], output=True), + run([str(old_exe), "--version"], output=True), ) - debug_print(f"{Style.BLUE}new mypy version: {new_version.stdout.strip()}{Style.RESET}") - debug_print(f"{Style.BLUE}old mypy version: {old_version.stdout.strip()}{Style.RESET}") - - return new_mypy, old_mypy - - -async def setup_new_and_old_pyright( - new_pyright_revision: RevisionLike, old_pyright_revision: RevisionLike -) -> tuple[Path, Path]: - new_pyright, old_pyright = await asyncio.gather( - setup_pyright( - ctx.get().base_dir / "new_pyright", - new_pyright_revision, - repo=ctx.get().repo, - ), - setup_pyright( - ctx.get().base_dir / "old_pyright", - old_pyright_revision, - repo=ctx.get().repo, - ), - ) - - if ctx.get().debug: - (new_version, _), (old_version, _) = await asyncio.gather( - run([str(new_pyright), "--version"], output=True), - run([str(old_pyright), "--version"], output=True), - ) - debug_print(f"{Style.BLUE}new pyright version: {new_version.stdout.strip()}{Style.RESET}") - debug_print(f"{Style.BLUE}old pyright version: {old_version.stdout.strip()}{Style.RESET}") - - return new_pyright, old_pyright - - -async def setup_new_and_old_typeshed( - new_typeshed_revision: RevisionLike, old_typeshed_revision: RevisionLike -) -> tuple[Path | None, Path | None]: - typeshed_repo = ctx.get().custom_typeshed_repo - - new_typeshed_dir = None - old_typeshed_dir = None - if ctx.get().new_typeshed: - new_typeshed_dir = await setup_typeshed( - ctx.get().base_dir / "new_typeshed", - repo=typeshed_repo, - revision_like=new_typeshed_revision, - ) - if ctx.get().old_typeshed: - old_typeshed_dir = await setup_typeshed( - ctx.get().base_dir / "old_typeshed", - repo=typeshed_repo, - revision_like=old_typeshed_revision, - ) - return new_typeshed_dir, old_typeshed_dir + debug_print(f"{Style.BLUE}new: {new_version.stdout.strip()}{Style.RESET}") + debug_print(f"{Style.BLUE}old: {old_version.stdout.strip()}{Style.RESET}") + return new_exe, old_exe # ============================== # project utils @@ -117,14 +65,12 @@ def select_projects() -> list[Project]: for p in get_projects() if not (p.min_python_version and sys.version_info < p.min_python_version) ) - if ARGS.type_checker == "pyright": - project_iter = iter(p for p in project_iter if p.pyright_cmd is not None) if ARGS.project_selector: project_iter = iter( p for p in project_iter if re.search(ARGS.project_selector, p.location, flags=re.I) ) if ARGS.expected_success: - project_iter = (p for p in project_iter if p.expected_success(ARGS.type_checker)) + project_iter = (p for p in project_iter if p.expected_success) if ARGS.project_date: project_iter = (replace(p, revision=ARGS.project_date) for p in project_iter) @@ -136,9 +82,9 @@ def select_projects() -> list[Project]: assert ARGS.shard_index is not None shard_costs = [0] * ARGS.num_shards shard_projects: list[list[Project]] = [[] for _ in range(ARGS.num_shards)] - for p in sorted(projects, key=lambda p: (p.mypy_cost, p.location), reverse=True): + for p in sorted(projects, key=lambda p: (1, p.location), reverse=True): min_shard = min(range(ARGS.num_shards), key=lambda i: shard_costs[i]) - shard_costs[min_shard] += p.mypy_cost + shard_costs[min_shard] += 1 shard_projects[min_shard].append(p) return shard_projects[ARGS.shard_index] return projects @@ -148,45 +94,42 @@ def select_projects() -> list[Project]: # hidden entrypoint logic # ============================== -RECENT_MYPYS = ["0.991", "0.982", "0.971", "0.961"] +RECENT_PYDOCTOR = ["23.9.0", "23.4.1", "22.9.1", "22.7.0"] async def validate_expected_success() -> None: """Check correctness of hardcoded Project.expected_success""" ARGS = ctx.get() - assert ARGS.type_checker == "mypy" - - recent_mypy_exes = await asyncio.gather( + recent_pydoctor_exes = await asyncio.gather( *[ - setup_mypy( - ARGS.base_dir / ("mypy_" + recent_mypy), - recent_mypy, + setup_pydoctor( + ARGS.base_dir / ("pydoctor_" + pydoctor_dir), + pydoctor_dir, repo=ARGS.repo, - mypyc_compile_level=ARGS.mypyc_compile_level, ) - for recent_mypy in RECENT_MYPYS + for pydoctor_dir in RECENT_PYDOCTOR ] ) async def inner(project: Project) -> str | None: await project.setup() success = None - for mypy_exe in recent_mypy_exes: - mypy_result = await project.run_mypy(mypy_exe, typeshed_dir=None) + for exe in recent_pydoctor_exes: + result = await project.run_pydoctor(exe) if ARGS.debug: debug_print(format(Style.BLUE)) - debug_print(mypy_result) + debug_print(result) debug_print(format(Style.RESET)) - if mypy_result.success: - success = mypy_exe + if result.success: + success = exe break - if bool(success) and not project.expected_mypy_success: + if bool(success) and not project.expected_success: return ( f"Project {project.location} succeeded with {success}, " "but is not marked as expecting success" ) - if not bool(success) and project.expected_mypy_success: + if not bool(success) and project.expected_success: return f"Project {project.location} did not succeed, but is marked as expecting success" return None @@ -197,20 +140,18 @@ async def inner(project: Project) -> str | None: async def measure_project_runtimes() -> None: - """Check mypy's runtime over each project.""" + """Check pydoctor's runtime over each project.""" ARGS = ctx.get() - assert ARGS.type_checker == "mypy" - - mypy_exe = await setup_mypy( - ARGS.base_dir / "timer_mypy", - ARGS.new or RECENT_MYPYS[0], + + exe = await setup_pydoctor( + ARGS.base_dir / "runtimes", + ARGS.new or RECENT_PYDOCTOR[0], repo=ARGS.repo, - mypyc_compile_level=ARGS.mypyc_compile_level, ) async def inner(project: Project) -> tuple[float, Project]: await project.setup() - result = await project.run_mypy(mypy_exe, typeshed_dir=None) + result = await project.run_pydoctor(exe) return (result.runtime, project) results = sorted( @@ -229,25 +170,20 @@ async def inner(project: Project) -> tuple[float, Project]: async def bisect() -> None: ARGS = ctx.get() - assert ARGS.type_checker == "mypy" - assert not ARGS.new_typeshed - assert not ARGS.old_typeshed - - mypy_exe = await setup_mypy( - ARGS.base_dir / "bisect_mypy", + exe = await setup_pydoctor( + ARGS.base_dir / "bisect", revision_or_recent_tag_fn(ARGS.old), repo=ARGS.repo, - mypyc_compile_level=ARGS.mypyc_compile_level, editable=True, ) - repo_dir = ARGS.base_dir / "bisect_mypy" / "mypy" + repo_dir = ARGS.base_dir / "bisect" / "pydoctor" assert repo_dir.is_dir() projects = select_projects() await asyncio.wait([project.setup() for project in projects]) async def run_wrapper(project: Project) -> tuple[str, TypeCheckResult]: - return project.name, (await project.run_mypy(str(mypy_exe), typeshed_dir=None)) + return project.name, (await project.run_pydoctor(str(exe))) results_fut = await asyncio.gather(*(run_wrapper(project) for project in projects)) old_results: dict[str, TypeCheckResult] = dict(results_fut) @@ -291,66 +227,19 @@ def are_results_good(results: dict[str, TypeCheckResult]) -> bool: debug_print(format(Style.RESET)) -async def coverage() -> None: - ARGS = ctx.get() - assert ARGS.type_checker == "mypy" - mypy_exe = await setup_mypy( - ARGS.base_dir / "new_mypy", - revision_like=ARGS.new, - repo=ARGS.repo, - mypyc_compile_level=ARGS.mypyc_compile_level, - ) - - projects = select_projects() - mypy_python = mypy_exe.parent / "python" - assert mypy_python.exists() - - all_paths = await asyncio.gather( - *[project.mypy_source_paths(str(mypy_python)) for project in projects] - ) - - project_to_paths: dict[str, int] = {} - project_to_lines: dict[str, int] = {} - for project, paths in zip(projects, all_paths): - project_to_paths[project.location] = len(paths) - project_to_lines[project.location] = sum(map(line_count, paths)) - - for project in sorted(projects, key=lambda p: project_to_lines[p.location], reverse=True): - p = project.location - print(p, project_to_lines[p], project_to_paths[p]) - - print(f"Checking {len(projects)} projects...") - print(f"Containing {sum(project_to_paths.values())} files...") - print(f"Totalling to {sum(project_to_lines.values())} lines...") - - async def primer() -> int: projects = select_projects() ARGS = ctx.get() - if ARGS.type_checker == "mypy": - new_type_checker, old_type_checker = await setup_new_and_old_mypy( - new_mypy_revision=ARGS.new, - old_mypy_revision=revision_or_recent_tag_fn(ARGS.old), - ) - elif ARGS.type_checker == "pyright": - new_type_checker, old_type_checker = await setup_new_and_old_pyright( - new_pyright_revision=ARGS.new, - old_pyright_revision=revision_or_recent_tag_fn(ARGS.old), - ) - else: - raise ValueError(f"Unknown type checker {ARGS.type_checker}") - - new_typeshed_dir, old_typeshed_dir = await setup_new_and_old_typeshed( - ARGS.new_typeshed, ARGS.old_typeshed + new_type_checker, old_type_checker = await setup_new_and_old_pydoctor( + ARGS.new, + revision_or_recent_tag_fn(ARGS.old), ) results = [ project.primer_result( new_type_checker=str(new_type_checker), old_type_checker=str(old_type_checker), - new_typeshed=new_typeshed_dir, - old_typeshed=old_typeshed_dir, ) for project in projects ] @@ -364,8 +253,6 @@ async def primer() -> int: elif ARGS.output == "diff": print(result.format_diff_only()) elif ARGS.output == "concise": - # using ARGS.output == "concise" also causes us to: - # - always pass in --no-pretty and --no-error-summary concise = result.format_concise() if concise: print(concise) @@ -391,9 +278,7 @@ def inner() -> int | None: ARGS.projects_dir.mkdir(exist_ok=True) coro: Awaitable[int | None] - if ARGS.coverage: - coro = coverage() - elif ARGS.bisect or ARGS.bisect_output: + if ARGS.bisect or ARGS.bisect_output: coro = bisect() elif ARGS.validate_expected_success: coro = validate_expected_success() diff --git a/pydoctor_primer/model.py b/pydoctor_primer/model.py index bbb55bd..5e9e526 100644 --- a/pydoctor_primer/model.py +++ b/pydoctor_primer/model.py @@ -25,33 +25,21 @@ @dataclass(frozen=True, **extra_dataclass_args) class Project: location: str - mypy_cmd: str + pydoctor_cmd: str revision: str | None = None min_python_version: tuple[int, int] | None = None pip_cmd: str | None = None - # if expected_success, there is a recent version of mypy which passes cleanly - expected_mypy_success: bool = False - # mypy_cost is vaguely proportional to mypy's type check time - mypy_cost: int = 3 - - pyright_cmd: str | None = None - expected_pyright_success: bool = False - + # if expected_success, there is a recent version of pydoctor which passes cleanly + expected_success: bool = False name_override: str | None = None # custom __repr__ that omits defaults. def __repr__(self) -> str: - result = f"Project(location={self.location!r}, mypy_cmd={self.mypy_cmd!r}" - if self.pyright_cmd: - result += f", pyright_cmd={self.pyright_cmd!r}" + result = f"Project(location={self.location!r}, pydoctor_cmd={self.pydoctor_cmd!r}" if self.pip_cmd: result += f", pip_cmd={self.pip_cmd!r}" - if self.expected_mypy_success: - result += f", expected_mypy_success={self.expected_mypy_success!r}" - if self.expected_pyright_success: - result += f", expected_pyright_success={self.expected_pyright_success!r}" - if self.mypy_cost != 3: - result += f", mypy_cost={self.mypy_cost!r}" + if self.expected_success: + result += f", expected_success={self.expected_success!r}" if self.revision: result += f", revision={self.revision!r}" if self.min_python_version: @@ -71,14 +59,6 @@ def name(self) -> str: def venv_dir(self) -> Path: return ctx.get().projects_dir / f"_{self.name}_venv" - def expected_success(self, type_checker: str) -> bool: - if type_checker == "mypy": - return self.expected_mypy_success - elif type_checker == "pyright": - return self.expected_pyright_success - else: - raise ValueError(f"unknown type checker {type_checker}") - async def setup(self) -> None: if Path(self.location).exists(): repo_dir = ctx.get().projects_dir / self.name @@ -117,42 +97,24 @@ async def setup(self) -> None: print(e.stderr) raise RuntimeError(f"pip install failed for {self.name}") from e - def get_mypy_cmd(self, mypy: str | Path, additional_flags: Sequence[str] = ()) -> str: - mypy_cmd = self.mypy_cmd - assert "{mypy}" in self.mypy_cmd - mypy_cmd = mypy_cmd.format(mypy=mypy) + def get_pydoctor_cmd(self, pydoctor_path: str | Path, additional_flags: Sequence[str] = ()) -> str: + cmd = self.pydoctor_cmd + assert "{pydoctor}" in self.pydoctor_cmd + cmd = cmd.format(pydoctor=pydoctor_path) - if self.pip_cmd: - python_exe = self.venv_dir / BIN_DIR / "python" - mypy_cmd += f" --python-executable={quote_path(python_exe)}" if additional_flags: - mypy_cmd += " " + " ".join(additional_flags) - if ctx.get().output == "concise": - mypy_cmd += " --no-pretty --no-error-summary" + cmd += " " + " ".join(additional_flags) - mypy_cmd += " --warn-unused-ignores --warn-redundant-casts" - mypy_cmd += ( - f" --no-incremental --cache-dir={os.devnull} --show-traceback --soft-error-limit=-1" - ) - return mypy_cmd + cmd += " --make-html --quiet" + return cmd - async def run_mypy(self, mypy: str | Path, typeshed_dir: Path | None) -> TypeCheckResult: + async def run_pydoctor(self, pydoctor_path: str | Path) -> TypeCheckResult: additional_flags = ctx.get().additional_flags.copy() env = os.environ.copy() - env["MYPY_FORCE_COLOR"] = "1" - - mypy_path = [] # TODO: this used to be exposed, could be useful to expose it again - if typeshed_dir is not None: - additional_flags.append(f"--custom-typeshed-dir={quote_path(typeshed_dir)}") - mypy_path += list(map(str, typeshed_dir.glob("stubs/*"))) - if "MYPYPATH" in env: - mypy_path = env["MYPYPATH"].split(os.pathsep) + mypy_path - env["MYPYPATH"] = os.pathsep.join(mypy_path) - - mypy_cmd = self.get_mypy_cmd(mypy, additional_flags) + cmd = self.get_pydoctor_cmd(pydoctor_path, additional_flags) proc, runtime = await run( - mypy_cmd, + cmd, shell=True, output=True, check=False, @@ -160,127 +122,42 @@ async def run_mypy(self, mypy: str | Path, typeshed_dir: Path | None) -> TypeChe env=env, ) if ctx.get().debug: - debug_print(f"{Style.BLUE}{mypy} on {self.name} took {runtime:.2f}s{Style.RESET}") + debug_print(f"{Style.BLUE}{pydoctor_path} on {self.name} took {runtime:.2f}s{Style.RESET}") output = proc.stderr + proc.stdout - # Various logic to reduce noise in the diff - if typeshed_dir is not None: - # Differing line numbers and typeshed paths create noisy diffs. - # Not a problem for stdlib because mypy silences errors from --custom-typeshed-dir. - output = "".join( - line - for line in output.splitlines(keepends=True) - if not line.startswith(str(typeshed_dir / "stubs")) - ) - - # Redact "note" lines which contain base_dir - # Avoids noisy diffs when e.g., mypy points to a stub definition + # Redact lines which contain base_dir + # Avoids noisy diffs base_dir_re = ( f"({re.escape(str(ctx.get().base_dir))}" f"|{re.escape(str(ctx.get().base_dir.resolve()))})" - ".*: note:" ) - output = re.sub(base_dir_re, "note:", output) - - # Avoids some noise in tracebacks - if "error: INTERNAL ERROR" in output: - output = re.sub('File ".*/mypy', 'File "', output) + output = re.sub(base_dir_re, "", output) return TypeCheckResult( - mypy_cmd, output, not bool(proc.returncode), self.expected_mypy_success, runtime - ) - - def get_pyright_cmd(self, pyright: str | Path, additional_flags: Sequence[str] = ()) -> str: - pyright_cmd = self.pyright_cmd or "{pyright}" - assert "{pyright}" in pyright_cmd - if additional_flags: - pyright_cmd += " " + " ".join(additional_flags) - pyright_cmd = pyright_cmd.format(pyright=pyright) - return pyright_cmd - - async def run_pyright(self, pyright: str | Path, typeshed_dir: Path | None) -> TypeCheckResult: - additional_flags: list[str] = [] - if typeshed_dir is not None: - additional_flags.append(f"--typeshedpath {quote_path(typeshed_dir)}") - pyright_cmd = self.get_pyright_cmd(pyright, additional_flags) - if self.pip_cmd: - activate = ( - f"source {shlex.quote(str(self.venv_dir / BIN_DIR / 'activate'))}" - if sys.platform != "win32" - else str(self.venv_dir / BIN_DIR / "activate.bat") - ) - pyright_cmd = f"{activate}; {pyright_cmd}" - proc, runtime = await run( - pyright_cmd, - shell=True, - output=True, - check=False, - cwd=ctx.get().projects_dir / self.name, - ) - if ctx.get().debug: - debug_print(f"{Style.BLUE}{pyright} on {self.name} took {runtime:.2f}s{Style.RESET}") - - output = proc.stderr + proc.stdout - return TypeCheckResult( - pyright_cmd, output, not bool(proc.returncode), self.expected_pyright_success, runtime + cmd, output, not bool(proc.returncode), self.expected_success, runtime ) async def run_typechecker( - self, type_checker: str | Path, typeshed_dir: Path | None + self, type_checker: str | Path, ) -> TypeCheckResult: - if ctx.get().type_checker == "mypy": - return await self.run_mypy(type_checker, typeshed_dir) - elif ctx.get().type_checker == "pyright": - return await self.run_pyright(type_checker, typeshed_dir) - else: - raise ValueError(f"Unknown type checker: {ctx.get().type_checker}") + return await self.run_pydoctor(type_checker) async def primer_result( self, new_type_checker: str, old_type_checker: str, - new_typeshed: Path | None, - old_typeshed: Path | None, ) -> PrimerResult: await self.setup() new_result, old_result = await asyncio.gather( - self.run_typechecker(new_type_checker, new_typeshed), - self.run_typechecker(old_type_checker, old_typeshed), + self.run_typechecker(new_type_checker), + self.run_typechecker(old_type_checker), ) return PrimerResult(self, new_result, old_result) - async def mypy_source_paths(self, mypy_python: str) -> list[Path]: - await self.setup() - mypy_cmd = self.get_mypy_cmd(mypy="mypyprimersentinel") - mypy_cmd = mypy_cmd.split("mypyprimersentinel", maxsplit=1)[1] - program = """ -import io, sys, mypy.fscache, mypy.main -args = sys.argv[1:] -fscache = mypy.fscache.FileSystemCache() -sources, _ = mypy.main.process_options(args, io.StringIO(), io.StringIO(), fscache=fscache) -for source in sources: - if source.path is not None: # can happen for modules... - print(source.path) -""" - # the extra shell stuff here makes sure we expand globs in mypy_cmd - proc, _ = await run( - f"{mypy_python} -c {shlex.quote(program)} {mypy_cmd}", - output=True, - cwd=ctx.get().projects_dir / self.name, - shell=True, - ) - return [ctx.get().projects_dir / self.name / p for p in proc.stdout.splitlines()] - @classmethod def from_location(cls, location: str) -> Project: - additional_flags = "" - if Path(location).is_file(): - with open(location, encoding="UTF-8") as f: - header = f.readline().strip() - if header.startswith("# flags:"): - additional_flags = header[len("# flags:") :] - return Project(location=location, mypy_cmd=f"{{mypy}} {location} {additional_flags}") + return Project(location=location, pydoctor_cmd=f"{{pydoctor}} {location}") @dataclass(frozen=True) diff --git a/pydoctor_primer/projects.py b/pydoctor_primer/projects.py index a6a1d92..775d738 100644 --- a/pydoctor_primer/projects.py +++ b/pydoctor_primer/projects.py @@ -48,823 +48,14 @@ def update_projects(projects: list[Project], check: bool = False) -> None: def get_projects() -> list[Project]: projects = [ Project( - location="https://github.com/python/mypy", - mypy_cmd="{mypy} --config-file mypy_self_check.ini -p mypy -p mypyc", - pip_cmd="{pip} install pytest types-psutil types-setuptools filelock tomli", - expected_mypy_success=True, - mypy_cost=20, + location="https://github.com/twisted/pydoctor", + pydoctor_cmd="{pydoctor} ./pydoctor --privacy='HIDDEN:pydoctor.test'", + expected_success=True ), Project( - location="https://github.com/hauntsaninja/mypy_primer", - mypy_cmd="{mypy} -p mypy_primer --strict", - expected_mypy_success=True, - ), - Project( - location="https://github.com/psf/black", - mypy_cmd="{mypy} src", - pip_cmd="{pip} install aiohttp click pathspec tomli platformdirs", - expected_mypy_success=True, - ), - Project( - location="https://github.com/hauntsaninja/pyp", - mypy_cmd="{mypy} --strict -m pyp", - pyright_cmd="{pyright}", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pytest-dev/pytest", - mypy_cmd="{mypy} src testing", - pip_cmd="{pip} install attrs py types-setuptools", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pandas-dev/pandas", - mypy_cmd="{mypy} pandas", - pyright_cmd="{pyright} pandas", - pip_cmd=( - "{pip} install numpy types-python-dateutil types-pytz types-PyMySQL" - " types-setuptools pytest" - ), - expected_mypy_success=True, - mypy_cost=120, - ), - Project( - location="https://github.com/pycqa/pylint", - mypy_cmd="{mypy} pylint/checkers --ignore-missing-imports", - pip_cmd="{pip} install types-toml", - expected_mypy_success=True, - ), - Project( - location="https://github.com/aio-libs/aiohttp", - mypy_cmd="{mypy} aiohttp", - pip_cmd="AIOHTTP_NO_EXTENSIONS=1 {pip} install -e . pytest", - expected_mypy_success=True, - ), - Project( - location="https://github.com/python-attrs/attrs", - mypy_cmd=( - "{mypy} src/attr/__init__.pyi src/attr/_version_info.pyi src/attr/converters.pyi" - " src/attr/exceptions.pyi src/attr/filters.pyi src/attr/setters.pyi" - " src/attr/validators.pyi tests/typing_example.py" - ), - expected_mypy_success=True, - ), - Project( - location="https://github.com/sphinx-doc/sphinx", - mypy_cmd="{mypy} sphinx", - pip_cmd="{pip} install babel docutils-stubs types-requests packaging", - expected_mypy_success=True, - ), - Project( - location="https://github.com/scikit-learn/scikit-learn", - mypy_cmd="{mypy} sklearn", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pypa/bandersnatch", - mypy_cmd="{mypy} src", - pip_cmd="{pip} install types-filelock types-freezegun types-setuptools", - expected_mypy_success=True, - ), - Project( - location="https://github.com/hauntsaninja/boostedblob", - mypy_cmd="{mypy} boostedblob", - pip_cmd="{pip} install aiohttp uvloop pycryptodome", - expected_mypy_success=True, - ), - Project( - location="https://github.com/quora/asynq", - mypy_cmd="{mypy} asynq", - pip_cmd="{pip} install qcore", - expected_mypy_success=True, - ), - Project( - location="https://github.com/scrapy/scrapy", - mypy_cmd="{mypy} scrapy tests", - pip_cmd="{pip} install attrs types-pyOpenSSL types-setuptools", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pypa/twine", - mypy_cmd="{mypy} twine", - pip_cmd="{pip} install keyring types-requests", - expected_mypy_success=True, - ), - Project( - location="https://github.com/more-itertools/more-itertools", - mypy_cmd="{mypy} more_itertools", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pydata/xarray", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install types-PyYAML types-python-dateutil types-pytz", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pallets/werkzeug", - mypy_cmd="{mypy} src/werkzeug tests", - pip_cmd="{pip} install types-setuptools pytest markupsafe", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pallets/jinja", - mypy_cmd="{mypy}", - pip_cmd="{pip} install markupsafe", - expected_mypy_success=True, - ), - Project( - location="https://github.com/mystor/git-revise", - mypy_cmd="{mypy} gitrevise", - expected_mypy_success=True, - ), - Project( - location="https://github.com/PyGithub/PyGithub", - mypy_cmd="{mypy} github tests", - pip_cmd="{pip} install types-requests pyjwt", - expected_mypy_success=True, - ), - Project( - location="https://github.com/we-like-parsers/pegen", - mypy_cmd="{mypy} src/pegen", - expected_mypy_success=True, - ), - Project( - location="https://github.com/zulip/zulip", - mypy_cmd=( - "{mypy} zerver zilencer zproject tools analytics corporate scripts --platform=linux" - ), - pip_cmd=( - "{pip} install types-PyYAML types-polib types-redis types-Markdown types-decorator" - " types-pytz types-requests types-python-dateutil types-orjson cryptography" - ), - expected_mypy_success=True, - ), - Project( - location="https://github.com/dropbox/stone", - mypy_cmd="{mypy} stone test", - pip_cmd="{pip} install types-six", - expected_mypy_success=True, - ), - Project( - location="https://github.com/yelp/paasta", - mypy_cmd="{mypy} paasta_tools", - pip_cmd=( - "{pip} install types-retry types-tzlocal types-ujson types-python-dateutil" - " types-pytz types-PyYAML types-requests" - ), - expected_mypy_success=True, - ), - Project( - location="https://github.com/PrefectHQ/prefect", - mypy_cmd="{mypy} src", - pip_cmd=( - "{pip} install types-python-dateutil types-requests types-simplejson types-toml" - " types-croniter types-PyYAML types-python-slugify types-pytz cryptography" - " SQLAlchemy" - ), - expected_mypy_success=True, - ), - Project( - location="https://github.com/pallets/itsdangerous", - mypy_cmd="{mypy}", - pip_cmd="{pip} install pytest", - expected_mypy_success=True, - ), - Project( - location="https://github.com/jab/bidict", - mypy_cmd="{mypy} bidict", - pyright_cmd="{pyright}", - expected_mypy_success=True, - ), - Project( - location="https://github.com/jaraco/zipp", - mypy_cmd="{mypy} .", - expected_mypy_success=True, - ), - Project( - location="https://github.com/aaugustin/websockets", - mypy_cmd="{mypy} --strict src", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pycqa/isort", - mypy_cmd="{mypy} --ignore-missing-imports isort", - pip_cmd="{pip} install types-setuptools", - expected_mypy_success=True, - ), - Project( - location="https://github.com/aio-libs/aioredis", - mypy_cmd="{mypy} aioredis --ignore-missing-imports", - expected_mypy_success=True, - ), - Project( - location="https://github.com/agronholm/anyio", - mypy_cmd="{mypy} src", - expected_mypy_success=True, - ), - Project( - location="https://github.com/aio-libs/yarl", - mypy_cmd="{mypy} --show-error-codes yarl tests", - pip_cmd="{pip} install multidict", - expected_mypy_success=True, - ), - Project( - location="https://github.com/freqtrade/freqtrade", - mypy_cmd="{mypy} freqtrade scripts", - pyright_cmd="{pyright}", - pip_cmd=( - "{pip} install types-cachetools types-requests types-python-dateutil types-tabulate" - " types-filelock" - ), - expected_mypy_success=True, - ), - Project( - location="https://github.com/google/jax", - mypy_cmd="{mypy} jax", - pip_cmd="{pip} install types-requests", - expected_mypy_success=True, - ), - Project( - location="https://github.com/dulwich/dulwich", - mypy_cmd="{mypy} dulwich", - pip_cmd="{pip} install types-certifi types-paramiko", - expected_mypy_success=True, - ), - Project( - location="https://github.com/optuna/optuna", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install types-PyYAML types-redis types-setuptools SQLAlchemy", - expected_mypy_success=True, - ), - Project( - location="https://github.com/trailofbits/manticore", - mypy_cmd="{mypy}", - pip_cmd="{pip} install types-protobuf types-PyYAML types-redis types-setuptools", - expected_mypy_success=True, - ), - Project( - location="https://github.com/aiortc/aiortc", - mypy_cmd="{mypy} src", - pip_cmd="{pip} install cryptography", - expected_mypy_success=True, - ), - Project( - location="https://github.com/Textualize/rich", - mypy_cmd="{mypy} -p rich --ignore-missing-imports --warn-unreachable", - pip_cmd="{pip} install attrs", - expected_mypy_success=True, - ), - Project( - location="https://github.com/dedupeio/dedupe", - mypy_cmd="{mypy} --ignore-missing-imports dedupe", - expected_mypy_success=True, - ), - Project( - location="https://github.com/schemathesis/schemathesis", - mypy_cmd="{mypy} src/schemathesis", - pip_cmd="{pip} install attrs types-requests types-PyYAML", - expected_mypy_success=True, - ), - Project( - location="https://github.com/graphql-python/graphql-core", - mypy_cmd="{mypy} src tests", - expected_mypy_success=True, - mypy_cost=70, - ), - Project( - location="https://github.com/Legrandin/pycryptodome", - mypy_cmd="{mypy} lib", - expected_mypy_success=True, - ), - Project( - location="https://github.com/niklasf/python-chess", - mypy_cmd="{mypy} --strict chess", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pytorch/ignite", - mypy_cmd="{mypy}", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pypa/packaging", - mypy_cmd="{mypy} packaging", - expected_mypy_success=True, - ), - Project( - location="https://github.com/samuelcolvin/pydantic", - mypy_cmd="{mypy} pydantic", - pyright_cmd="{pyright}", - pip_cmd="{pip} install types-toml", - expected_mypy_success=True, - ), - Project( - location="https://github.com/encode/starlette", - mypy_cmd="{mypy} starlette tests", - pip_cmd="{pip} install types-requests types-PyYAML", - expected_mypy_success=True, - ), - Project( - location="https://github.com/aio-libs/janus", - mypy_cmd="{mypy} janus --disallow-untyped-calls --disallow-incomplete-defs --strict", - expected_mypy_success=True, - ), - Project( - location="https://github.com/alerta/alerta", - mypy_cmd="{mypy} alerta tests", - pip_cmd="{pip} install types-PyYAML types-setuptools types-requests types-pytz", - expected_mypy_success=True, - ), - Project( - location="https://github.com/nolar/kopf", - mypy_cmd="{mypy} kopf", - pip_cmd="{pip} install types-setuptools types-PyYAML", - expected_mypy_success=True, - ), - Project( - location="https://github.com/davidhalter/parso", - mypy_cmd="{mypy} parso", - expected_mypy_success=True, - ), - Project( - location="https://github.com/konradhalas/dacite", - mypy_cmd="{mypy} dacite", - expected_mypy_success=True, - ), - Project( - location="https://github.com/ilevkivskyi/com2ann", - mypy_cmd="{mypy} --python-version=3.8 src/com2ann.py src/test_com2ann.py", - expected_mypy_success=True, - ), - Project( - location="https://github.com/srittau/python-htmlgen", - mypy_cmd="{mypy} htmlgen test_htmlgen", - pip_cmd="{pip} install asserts", - expected_mypy_success=True, - ), - Project( - location="https://github.com/mitmproxy/mitmproxy", - mypy_cmd="{mypy} .", - expected_mypy_success=True, - ), - Project( - location="https://github.com/jpadilla/pyjwt", - mypy_cmd="{mypy} jwt", - pip_cmd="{pip} install cryptography", - expected_mypy_success=True, - ), - Project( - location="https://github.com/apache/spark", - mypy_cmd="{mypy} --config python/mypy.ini python/pyspark", - pip_cmd="{pip} install numpy", - expected_mypy_success=True, - mypy_cost=20, - ), - Project( - location="https://github.com/laowantong/paroxython", - mypy_cmd="{mypy} paroxython", - expected_mypy_success=True, - ), - Project( - location="https://github.com/Akuli/porcupine", - mypy_cmd="{mypy} --config-file= porcupine", - expected_mypy_success=True, - ), - Project( - location="https://github.com/dropbox/mypy-protobuf", - mypy_cmd="{mypy} mypy_protobuf/", - pyright_cmd="{pyright}", - pip_cmd="{pip} install types-protobuf", - expected_mypy_success=True, - ), - Project( - location="https://github.com/spack/spack", - mypy_cmd="{mypy} -p spack -p llnl", - pyright_cmd="{pyright}", - expected_mypy_success=True, - ), - Project( - location="https://github.com/johtso/httpx-caching", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install types-freezegun types-mock", - expected_mypy_success=True, - ), - Project( - location="https://github.com/python-poetry/poetry", - mypy_cmd="{mypy}", - pip_cmd="{pip} install types-requests", - expected_mypy_success=True, - ), - Project( - location="https://github.com/awslabs/sockeye", - mypy_cmd=( - "{mypy} --ignore-missing-imports --follow-imports=silent @typechecked-files" - " --no-strict-optional" - ), - pip_cmd="{pip} install types-PyYAML", - expected_mypy_success=True, - ), - Project( - location="https://github.com/wntrblm/nox", - mypy_cmd="{mypy} nox", - pip_cmd="{pip} install jinja2 packaging importlib_metadata", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pandera-dev/pandera", - mypy_cmd="{mypy} pandera tests", - pyright_cmd="{pyright}", - pip_cmd="{pip} install types-click types-PyYAML types-setuptools types-requests", - expected_mypy_success=True, - ), - Project( - location="https://gitlab.com/cki-project/cki-lib", - mypy_cmd="{mypy} --strict", - pip_cmd="{pip} install types-PyYAML types-requests", - expected_mypy_success=True, - ), - Project( - location="https://github.com/python-jsonschema/check-jsonschema", - mypy_cmd="{mypy} src", - pip_cmd="{pip} install types-jsonschema types-requests", - expected_mypy_success=True, - ), - Project( - location="https://github.com/pybind/pybind11", - mypy_cmd="{mypy} --exclude '^(tests|docs)/' .", - pip_cmd="{pip} install nox rich", - expected_mypy_success=True, - ), - Project( - location="https://github.com/rpdelaney/downforeveryone", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install types-requests types-requests", - expected_mypy_success=True, - ), - Project( - location="https://github.com/DataDog/dd-trace-py", - mypy_cmd="{mypy}", - pyright_cmd="{pyright}", - pip_cmd=( - "{pip} install attrs types-six types-setuptools types-docutils types-PyYAML" - " types-protobuf" - ), - expected_mypy_success=True, - ), - Project( - location="https://github.com/systemd/mkosi", - mypy_cmd="{mypy} mkosi", - pyright_cmd="{pyright}", - pip_cmd="{pip} install cryptography", - expected_mypy_success=True, - ), - Project( - location="https://github.com/sympy/sympy", - mypy_cmd="{mypy} sympy", - expected_mypy_success=True, - mypy_cost=70, - ), - Project( - location="https://github.com/nion-software/nionutils", - mypy_cmd="{mypy} --namespace-packages --strict -p nion.utils", - expected_mypy_success=True, - ), - Project( - location="https://github.com/PyCQA/flake8-pyi", - mypy_cmd="{mypy} pyi.py", - pip_cmd="{pip} install types-pyflakes", - expected_mypy_success=True, - ), - Project( - location="https://github.com/internetarchive/openlibrary", - mypy_cmd="{mypy} openlibrary", - pip_cmd=( - "{pip} install types-PyYAML types-python-dateutil types-requests types-simplejson" - " types-Deprecated" - ), - expected_mypy_success=True, - ), - Project( - location="https://github.com/JohannesBuchner/imagehash", - mypy_cmd="{mypy} imagehash", - pip_cmd="{pip} install numpy types-Pillow", - expected_mypy_success=True, - ), - Project( - location="https://github.com/Kalmat/PyWinCtl", - mypy_cmd="{mypy} src/pywinctl", - pyright_cmd="{pyright}", - pip_cmd="{pip} install types-setuptools types-pywin32 types-python-xlib", - expected_mypy_success=True, - ), - Project( - location="https://github.com/mesonbuild/meson", - mypy_cmd="./run_mypy.py --mypy {mypy}", - pip_cmd="{pip} install types-PyYAML", - expected_mypy_success=True, - ), - Project( - location="https://github.com/aio-libs/aiohttp-devtools", - mypy_cmd="{mypy}", - pip_cmd="{pip} install aiohttp watchfiles types-pygments", - expected_mypy_success=True, - ), - Project( - location="https://github.com/ZettaAI/zetta_utils", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install types-Pillow types-cachetools types-requests attrs", - expected_mypy_success=True, - ), - Project( - location="https://github.com/sco1/pylox", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install attrs", - expected_mypy_success=True, - min_python_version=(3, 10), - ), - Project( - location="https://github.com/ppb/ppb-vector", - mypy_cmd="{mypy} ppb_vector tests", - pip_cmd="{pip} install hypothesis", - expected_mypy_success=True, - min_python_version=(3, 10), - ), - Project( - location="https://github.com/pyppeteer/pyppeteer", - mypy_cmd="{mypy} pyppeteer --config-file tox.ini", - pip_cmd="{pip} install .", - ), - Project(location="https://github.com/pypa/pip", mypy_cmd="{mypy} src"), - Project(location="https://github.com/pytorch/vision", mypy_cmd="{mypy}"), - Project( - location="https://github.com/tornadoweb/tornado", - mypy_cmd="{mypy} tornado", - pip_cmd="{pip} install types-contextvars types-pycurl", - ), - Project( - location="https://github.com/scipy/scipy", - mypy_cmd="{mypy} scipy", - pip_cmd="{pip} install numpy", - ), - Project( - location="https://github.com/pycqa/flake8", - mypy_cmd="{mypy} src tests", - pip_cmd="{pip} install pytest", - ), - Project( - location="https://github.com/home-assistant/core", - mypy_cmd="{mypy} homeassistant", - pip_cmd=( - "{pip} install attrs pydantic types-setuptools types-atomicwrites types-certifi" - " types-croniter types-PyYAML types-requests types-python-slugify types-backports" - ), - mypy_cost=70, - ), - Project(location="https://github.com/kornia/kornia", mypy_cmd="{mypy} kornia"), - Project( - location="https://github.com/ibis-project/ibis", - mypy_cmd="{mypy} --ignore-missing-imports ibis", - pip_cmd=( - "{pip} install types-setuptools types-requests types-python-dateutil types-pytz" - " SQLAlchemy" - ), - ), - Project( - location="https://github.com/streamlit/streamlit", - mypy_cmd="{mypy} --config-file=lib/mypy.ini lib scripts", - pip_cmd=( - "{pip} install attrs tornado packaging types-toml types-python-dateutil" - " types-setuptools types-protobuf types-pytz types-requests types-cffi click pytest" - ), - ), - Project( - location="https://github.com/dragonchain/dragonchain", - mypy_cmd="{mypy} dragonchain --error-summary", - pip_cmd="{pip} install types-redis types-requests", - ), - Project( - location="https://github.com/mikeshardmind/SinbadCogs", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install attrs types-pytz types-python-dateutil types-PyYAML", - ), - Project( - location="https://github.com/rotki/rotki", - mypy_cmd="{mypy} rotkehlchen/ tools/data_faker", - pip_cmd="{pip} install eth-typing types-requests types-setuptools", - ), - Project( - location="https://github.com/arviz-devs/arviz", - mypy_cmd="{mypy} .", - pip_cmd="{pip} install pytest types-setuptools types-ujson numpy xarray", - mypy_cost=20, - ), - Project( - location="https://github.com/urllib3/urllib3", - mypy_cmd="{mypy} . --exclude setup.py", - pip_cmd=( - "{pip} install idna>=2.0.0 cryptography>=1.3.4 tornado>=6.1 pytest trustme==0.9.0" - " types-backports types-requests" - ), - ), - Project( - location="https://github.com/common-workflow-language/schema_salad", - mypy_cmd="MYPYPATH=$MYPYPATH:mypy-stubs {mypy} schema_salad", - pip_cmd=( - "{pip} install types-pkg_resources types-requests types-dataclasses" - " types-setuptools black pytest ruamel.yaml" - ), - ), - Project( - location="https://github.com/common-workflow-language/cwltool", - mypy_cmd="MYPYPATH=$MYPYPATH:mypy-stubs {mypy} cwltool/*.py tests/*.py", - pip_cmd=( - "{pip} install types-requests types-setuptools types-psutil types-mock cwl-utils" - " schema-salad ruamel-yaml pytest pytest-httpserver" - ), - mypy_cost=20, - ), - Project( - location="https://github.com/FasterSpeeding/Tanjun", - mypy_cmd="{mypy} tanjun", - pyright_cmd="{pyright}", - pip_cmd="{pip} install hikari alluka", - ), - Project( - location="https://github.com/joerick/pyinstrument", - mypy_cmd="{mypy} pyinstrument", - pyright_cmd="{pyright}", - ), - Project( - location="https://github.com/Gobot1234/steam.py", - mypy_cmd="{mypy} steam", - pyright_cmd="{pyright}", - pip_cmd="{pip} install cryptography", - ), - Project( - location="https://github.com/cpitclaudel/alectryon", - mypy_cmd="{mypy} alectryon.py", - pyright_cmd="{pyright}", - ), - Project( - location="https://github.com/yurijmikhalevich/rclip", - mypy_cmd="{mypy} rclip", - pyright_cmd="{pyright}", - ), - Project( - location="https://github.com/psycopg/psycopg", - mypy_cmd="{mypy}", - pip_cmd="{pip} install pytest pproxy", - ), - Project(location="https://gitlab.com/dkg/python-sop", mypy_cmd="{mypy} --strict sop"), - Project( - location="https://github.com/Rapptz/discord.py", - mypy_cmd="{mypy} discord", - pyright_cmd="{pyright}", - pip_cmd="{pip} install types-requests types-setuptools aiohttp", - mypy_cost=20, - ), - Project( - location="https://github.com/canonical/cloud-init", - mypy_cmd="{mypy} cloudinit/ tests/ tools/", - pip_cmd=( - "{pip} install jinja2 pytest types-jsonschema types-oauthlib types-pyyaml" - " types-requests types-setuptools" - ), - mypy_cost=20, - ), - Project( - location="https://github.com/mongodb/mongo-python-driver", - mypy_cmd="{mypy} bson gridfs tools pymongo", - pip_cmd="{pip} install types-requests types-pyOpenSSL cryptography certifi", - ), - Project( - location="https://github.com/artigraph/artigraph", - mypy_cmd="{mypy}", - pip_cmd="{pip} install pydantic numpy pytest", - ), - Project( - location="https://github.com/MaterializeInc/materialize", - mypy_cmd="MYPYPATH=$MYPYPATH:misc/python {mypy} ci misc/python", - pyright_cmd="{pyright}", - pip_cmd="{pip} install -r ci/builder/requirements.txt", - mypy_cost=30, - ), - Project( - location="https://github.com/canonical/operator", - mypy_cmd="{mypy} ops", - pyright_cmd="{pyright}", - pip_cmd="{pip} install types-PyYAML", - ), - Project( - location="https://github.com/astropenguin/xarray-dataclasses", - mypy_cmd="{mypy} xarray_dataclasses", - pyright_cmd="{pyright}", - pip_cmd="{pip} install numpy xarray", - ), - Project( - location="https://github.com/caronc/apprise", - mypy_cmd="{mypy} .", - pip_cmd=( - "{pip} install types-six types-mock cryptography types-requests types-Markdown" - " pytest certifi babel" - ), - ), - Project( - location="https://github.com/daveleroy/sublime_debugger", - mypy_cmd="{mypy} modules --namespace-packages", - pip_cmd="{pip} install certifi", - ), - Project( - location="https://github.com/Finistere/antidote", - mypy_cmd="{mypy} .", - pyright_cmd="{pyright}", - pip_cmd="{pip} install pytest", - ), - Project( - location="https://github.com/cognitedata/Expression", - mypy_cmd="{mypy} .", - pyright_cmd="{pyright}", - pip_cmd="{pip} install pytest", - ), - Project( - location="https://github.com/pyodide/pyodide", - mypy_cmd="{mypy} src pyodide-build --exclude 'setup.py|^src/tests|conftest.py'", - pip_cmd="{pip} install packaging types-docutils types-pyyaml types-setuptools numpy", - ), - Project( - location="https://github.com/bokeh/bokeh", - mypy_cmd="{mypy} src release", - pip_cmd="{pip} install types-boto tornado numpy jinja2 selenium", - ), - Project( - location="https://github.com/pandas-dev/pandas-stubs", - mypy_cmd="{mypy} pandas-stubs tests", - pyright_cmd="{pyright}", - pip_cmd=( - "{pip} install numpy types-pytz matplotlib xarray pyarrow jinja2 pytest SQLAlchemy" - ), - expected_pyright_success=True, - mypy_cost=20, - ), - Project( - location="https://github.com/enthought/comtypes", - mypy_cmd="{mypy} comtypes --platform win32", - pip_cmd="{pip} install numpy", - ), - Project( - location="https://github.com/mit-ll-responsible-ai/hydra-zen", - mypy_cmd="{mypy} src tests/annotations/mypy_checks.py", - pyright_cmd="{pyright} tests/annotations src", - pip_cmd="{pip} install pydantic beartype hydra-core", - mypy_cost=30, - ), - Project( - location="https://github.com/Toufool/AutoSplit", - mypy_cmd="{mypy} src", - pyright_cmd="{pyright} src", - pip_cmd=( - "{pip} install certifi ImageHash numpy packaging PyWinCtl PySide6-Essentials" - " types-D3DShot types-keyboard types-Pillow types-psutil types-PyAutoGUI" - " types-pyinstaller types-pywin32 types-requests types-toml" - ), - ), - Project( - location="https://github.com/Avasam/speedrun.com_global_scoreboard_webapp", - mypy_cmd="{mypy} backend", - pyright_cmd="{pyright}", - pip_cmd=( - "{pip} install Flask PyJWT requests-cache types-Flask-SQLAlchemy types-httplib2" - " types-requests" - ), - mypy_cost=30, - ), - Project( - location="https://github.com/pwndbg/pwndbg", - mypy_cmd="{mypy} pwndbg", - pip_cmd="{pip} install types-gdb", - ), - Project( - location="https://github.com/keithasaurus/koda-validate", - mypy_cmd="{mypy} koda_validate --strict", - pyright_cmd="{pyright}", - pip_cmd="{pip} install koda", - ), - Project( - location="https://github.com/python/cpython", - mypy_cmd="{mypy} --config-file Tools/clinic/mypy.ini", - name_override="CPython (Argument Clinic)", - ), - Project( - location="https://github.com/python/cpython", - mypy_cmd="{mypy} --config-file Tools/cases_generator/mypy.ini", - name_override="CPython (cases_generator)", - ), - Project( - location="https://github.com/python/cpython", - mypy_cmd="{mypy} --config-file Tools/peg_generator/mypy.ini", - pip_cmd="{pip} install types-setuptools types-psutil", - name_override="CPython (peg_generator)", + location="https://github.com/twisted/twisted", + pydoctor_cmd="{pydoctor} ./src/twisted --docformat=plaintext", + expected_success=True ), ] assert len(projects) == len({p.name for p in projects}) diff --git a/pydoctor_primer/type_checker.py b/pydoctor_primer/type_checker.py index 7420ff7..9204538 100644 --- a/pydoctor_primer/type_checker.py +++ b/pydoctor_primer/type_checker.py @@ -1,92 +1,44 @@ from __future__ import annotations -import os -import shutil import subprocess -import sys import venv from pathlib import Path from pydoctor_primer.git_utils import RevisionLike, ensure_repo_at_revision -from pydoctor_primer.utils import BIN_DIR, MYPY_EXE_NAME, run +from pydoctor_primer.utils import BIN_DIR, PYDOCTOR_EXE_NAME, run -async def setup_mypy( - mypy_dir: Path, +async def setup_pydoctor( + pydoctor_dir: Path, revision_like: RevisionLike, *, repo: str | None, - mypyc_compile_level: int | None, editable: bool = False, ) -> Path: - mypy_dir.mkdir(exist_ok=True) - venv_dir = mypy_dir / "venv" + pydoctor_dir.mkdir(exist_ok=True) + venv_dir = pydoctor_dir / "venv" venv.create(venv_dir, with_pip=True, clear=True) pip_exe = str(venv_dir / BIN_DIR / "pip") - if mypyc_compile_level is not None: - editable = True - install_from_repo = True if isinstance(revision_like, str) and not editable and repo is None: - # optimistically attempt to install the revision of mypy we want from pypi + # optimistically attempt to install the revision of pydoctor we want from pypi try: - await run([pip_exe, "install", f"mypy=={revision_like}"]) + await run([pip_exe, "install", f"pydoctor=={revision_like}"]) install_from_repo = False except subprocess.CalledProcessError: install_from_repo = True - if install_from_repo: if repo is None: - repo = "https://github.com/python/mypy" - repo_dir = await ensure_repo_at_revision(repo, mypy_dir, revision_like) - if mypyc_compile_level is not None: - env = os.environ.copy() - env["MYPYC_OPT_LEVEL"] = str(mypyc_compile_level) - python_exe = str(venv_dir / BIN_DIR / "python") - await run([pip_exe, "install", "typing_extensions", "mypy_extensions"]) - await run( - [python_exe, "setup.py", "--use-mypyc", "build_ext", "--inplace"], - cwd=repo_dir, - env=env, - ) + repo = "https://github.com/twisted/pydoctor" + repo_dir = await ensure_repo_at_revision(repo, pydoctor_dir, revision_like) + install_cmd = [pip_exe, "install"] if editable: install_cmd.append("--editable") install_cmd.append(str(repo_dir)) - install_cmd.append("tomli") await run(install_cmd) - mypy_exe = venv_dir / BIN_DIR / MYPY_EXE_NAME - if sys.platform == "darwin": - # warm up mypy on macos to avoid the first run being slow - await run([str(mypy_exe), "--version"]) - assert mypy_exe.exists() - return mypy_exe - - -async def setup_pyright( - pyright_dir: Path, - revision_like: RevisionLike, - *, - repo: str | None, -) -> Path: - pyright_dir.mkdir(exist_ok=True) - - if repo is None: - repo = "https://github.com/microsoft/pyright" - repo_dir = await ensure_repo_at_revision(repo, pyright_dir, revision_like) - - await run(["npm", "run", "install:all"], cwd=repo_dir) - await run(["npm", "run", "build"], cwd=repo_dir / "packages" / "pyright") - - pyright_exe = repo_dir / "packages" / "pyright" / "index.js" - assert pyright_exe.exists() - return pyright_exe - - -async def setup_typeshed(parent_dir: Path, *, repo: str, revision_like: RevisionLike) -> Path: - if parent_dir.exists(): - shutil.rmtree(parent_dir) - parent_dir.mkdir(exist_ok=True) - return await ensure_repo_at_revision(repo, parent_dir, revision_like) + pydoctor_exe = venv_dir / BIN_DIR / PYDOCTOR_EXE_NAME + assert pydoctor_exe.exists() + return pydoctor_exe diff --git a/pydoctor_primer/utils.py b/pydoctor_primer/utils.py index eddd5a1..b87fec1 100644 --- a/pydoctor_primer/utils.py +++ b/pydoctor_primer/utils.py @@ -16,11 +16,11 @@ import tempfile BIN_DIR = "scripts" - MYPY_EXE_NAME = "mypy.exe" + PYDOCTOR_EXE_NAME = "pydoctor.exe" TEMP_DIR = tempfile.gettempdir() else: BIN_DIR = "bin" - MYPY_EXE_NAME = "mypy" + PYDOCTOR_EXE_NAME = "pydoctor" TEMP_DIR = "/tmp"