From b6af73b8675cf5fc9b2593940cf509f14e425380 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 25 Oct 2024 15:00:54 -0700 Subject: [PATCH] Enable cross-PR testing --- .github/workflows/pull_request.yml | 21 +- .../Rules/UseShorthandTypeNames.swift | 10 +- cross-pr-checkout.py | 33 +++ cross-pr-checkout.swift | 217 ++++++++++++++++ cross-pr-checkout/.gitignore | 8 + cross-pr-checkout/Package.swift | 10 + .../Sources/cross-pr-checkout/main.swift | 234 ++++++++++++++++++ 7 files changed, 522 insertions(+), 11 deletions(-) create mode 100644 cross-pr-checkout.py create mode 100644 cross-pr-checkout.swift create mode 100644 cross-pr-checkout/.gitignore create mode 100644 cross-pr-checkout/Package.swift create mode 100644 cross-pr-checkout/Sources/cross-pr-checkout/main.swift diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d49113a8..e15c2ac4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -7,10 +7,19 @@ on: jobs: tests: name: Test - uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main - soundness: - name: Soundness - uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main + uses: ahoppen/github-workflows/.github/workflows/swift_package_test.yml@windows-error-propagation with: - license_header_check_enabled: false - license_header_check_project_name: "Swift.org" + linux_pre_build_command: | + cd cross-pr-checkout/Sources/cross-pr-checkout + swift cross-pr-checkout.swift "${{ github.repository }}" "${{ github.event.number }}" + windows_pre_build_command: | + mkdir $env:TEMP\cross-pr-checkout + cp cross-pr-checkout\Sources\cross-pr-checkout\main.swift $env:TEMP\cross-pr-checkout + swiftc -sdk $env:SDKROOT $env:TEMP\cross-pr-checkout\main.swift -o $env:TEMP\cross-pr-checkout\main.exe + $env:TEMP\cross-pr-checkout\main.exe "${{ github.repository }}" "${{ github.event.number }}" + # soundness: + # name: Soundness + # uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main + # with: + # license_header_check_enabled: false + # license_header_check_project_name: "Swift.org" diff --git a/Sources/SwiftFormat/Rules/UseShorthandTypeNames.swift b/Sources/SwiftFormat/Rules/UseShorthandTypeNames.swift index 67a8d8aa..a0c32241 100644 --- a/Sources/SwiftFormat/Rules/UseShorthandTypeNames.swift +++ b/Sources/SwiftFormat/Rules/UseShorthandTypeNames.swift @@ -48,7 +48,7 @@ public final class UseShorthandTypeNames: SyntaxFormatRule { switch node.name.text { case "Array": guard let argument = genericArgumentList.firstAndOnly, - case .type(let typeArgument) = argument else { + case .type(let typeArgument) = argument.argument else { newNode = nil break } @@ -62,7 +62,7 @@ public final class UseShorthandTypeNames: SyntaxFormatRule { case "Dictionary": guard let arguments = exactlyTwoChildren(of: genericArgumentList), case .type(let type0Argument) = arguments.0.argument, - caes .type(let type1Argument) = arguments.1.argument else { + case .type(let type1Argument) = arguments.1.argument else { newNode = nil break } @@ -79,7 +79,7 @@ public final class UseShorthandTypeNames: SyntaxFormatRule { break } guard let argument = genericArgumentList.firstAndOnly, - case .type(let typeArgument) = argument else { + case .type(let typeArgument) = argument.argument else { newNode = nil break } @@ -143,7 +143,7 @@ public final class UseShorthandTypeNames: SyntaxFormatRule { switch expression.baseName.text { case "Array": guard let argument = genericArgumentList.firstAndOnly, - case .type(let typeArgument) = argument else { + case .type(let typeArgument) = argument.argument else { newNode = nil break } @@ -172,7 +172,7 @@ public final class UseShorthandTypeNames: SyntaxFormatRule { case "Optional": guard let argument = genericArgumentList.firstAndOnly, - case .type(let typeArgument) = argument else { + case .type(let typeArgument) = argument.argument else { newNode = nil break } diff --git a/cross-pr-checkout.py b/cross-pr-checkout.py new file mode 100644 index 00000000..ebbb07a5 --- /dev/null +++ b/cross-pr-checkout.py @@ -0,0 +1,33 @@ +import subprocess +import pathlib +import requests + +class CrossRepoPR: + org: str + repo: str + pr_num: str + + def __init__(self, org: str, repo: str, pr_num: str) -> None: + self.org = org + self.repo = repo + self.pr_num = pr_num + +def cross_repo_prs() -> list[CrossRepoPR]: + return [ + CrossRepoPR("swiftlang", "swift-syntax", "2859") + ] + +def run(cmd: list[str], cwd: str|None = None): + print(" ".join(cmd)) + subprocess.check_call(cmd, cwd=cwd) + +def main(): + for cross_repo_pr in cross_repo_prs(): + run(["git", "clone", f"https://github.com/{cross_repo_pr.org}/{cross_repo_pr.repo}.git", f"{cross_repo_pr.repo}"], cwd="..") + run(["git", "fetch", "origin", f"pull/{cross_repo_pr.pr_num}/merge:pr_merge"], cwd="../swift-syntax") + run(["git", "checkout", "main"], cwd="../swift-syntax") + run(["git", "reset", "--hard", "pr_merge"], cwd="../swift-syntax") + run(["swift", "package", "config", "set-mirror", "--package-url", "https://github.com/swiftlang/swift-syntax.git", "--mirror-url", str(pathlib.Path("../swift-syntax").resolve())]) + +if __name__ == "__main__": + main() diff --git a/cross-pr-checkout.swift b/cross-pr-checkout.swift new file mode 100644 index 00000000..f90f162f --- /dev/null +++ b/cross-pr-checkout.swift @@ -0,0 +1,217 @@ +import Foundation + +#if canImport(FoundationNetworking) +import FoundationNetworking +#endif + +#if canImport(WinSDK) +import WinSDK +#endif + +struct GenericError: Error, CustomStringConvertible { + var description: String + + init(_ description: String) { + self.description = description + } +} + +/// Escape the given command to be printed for log output. +func escapeCommand(_ executable: URL, _ arguments: [String]) -> String { + return ([executable.path] + arguments).map { + if $0.contains(" ") { + return "'\($0)'" + } + return $0 + }.joined(separator: " ") +} + +/// Launch a subprocess with the given command and wait for it to finish +func run(_ executable: URL, _ arguments: String..., workingDirectory: URL? = nil) throws { + print("Running \(escapeCommand(executable, arguments))") + let process = Process() + process.executableURL = executable + process.arguments = arguments + if let workingDirectory { + process.currentDirectoryURL = workingDirectory + } + + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { + throw GenericError( + "\(escapeCommand(executable, arguments)) failed with non-zero exit code: \(process.terminationStatus)" + ) + } +} + +/// Find the executable with the given name +public func lookup(executable: String) throws -> URL { + // Compute search paths from PATH variable. + #if os(Windows) + let pathVariable = "Path" + let pathSeparator: Character = ";" + #else + let pathVariable = "PATH" + let pathSeparator: Character = ":" + #endif + guard let pathString = ProcessInfo.processInfo.environment[pathVariable] else { + throw GenericError("Failed to read path environment variable") + } + for searchPath in pathString.split(separator: pathSeparator) { + let candidateUrl = URL(fileURLWithPath: String(searchPath)).appendingPathComponent(executable) + if FileManager.default.isExecutableFile(atPath: candidateUrl.path) { + return candidateUrl + } + } + throw GenericError("Did not find \(executable)") +} + +/// The JSON fields of the `https://api.github.com/repos/\(repository)/pulls/\(prNumber)` endpoint that we care about. +struct PRInfo: Codable { + struct Base: Codable { + /// The name of the PR's base branch. + let ref: String + } + /// The base branch of the PR + let base: Base + + /// The PR's description. + let body: String? +} + +/// - Parameters: +/// - repository: The repository's name, eg. `swiftlang/swift-syntax` +func getPRInfo(repository: String, prNumber: String) throws -> PRInfo { + guard let prInfoUrl = URL(string: "https://api.github.com/repos/\(repository)/pulls/\(prNumber)") else { + throw GenericError("Failed to form URL for GitHub API") + } + + do { + let data = try Data(contentsOf: prInfoUrl) + return try JSONDecoder().decode(PRInfo.self, from: data) + } catch { + throw GenericError("Failed to load PR info from \(prInfoUrl): \(error)") + } +} + +/// Information about a PR that should be tested with this PR. +struct CrossRepoPR { + /// The owner of the repository, eg. `swiftlang` + let repositoryOwner: String + + /// The name of the repository, eg. `swift-syntax` + let repositoryName: String + + /// The PR number that's referenced. + let prNumber: String +} + +/// Retrieve all PRs that are referenced from the PR with the given number in `repository`. +/// `repository` is the owner and repo name joined by `/`, eg. `swiftlang/swift-syntax`. +func getCrossRepoPrs(repository: String, prNumber: String) throws -> [CrossRepoPR] { + var result: [CrossRepoPR] = [] + let prInfo = try getPRInfo(repository: repository, prNumber: prNumber) + for line in prInfo.body?.split(separator: "\n") ?? [] { + guard line.lowercased().starts(with: "linked pr:") else { + continue + } + // We can't use Swift's Regex here because this script needs to run on Windows with Swift 5.9, which doesn't support + // Swift Regex. + var remainder = line[...] + guard let ownerRange = remainder.firstRange(of: "swiftlang/") ?? remainder.firstRange(of: "apple/") else { + continue + } + let repositoryOwner = remainder[ownerRange].dropLast() + remainder = remainder[ownerRange.upperBound...] + let repositoryName = remainder.prefix { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" } + if repositoryName.isEmpty { + continue + } + remainder = remainder.dropFirst(repositoryName.count) + if remainder.starts(with: "/pull/") { + remainder = remainder.dropFirst(6) + } else if remainder.starts(with: "#") { + remainder = remainder.dropFirst() + } else { + continue + } + let pullRequestNum = remainder.prefix { $0.isNumber } + if pullRequestNum.isEmpty { + continue + } + result.append( + CrossRepoPR( + repositoryOwner: String(repositoryOwner), + repositoryName: String(repositoryName), + prNumber: String(pullRequestNum) + ) + ) + } + return result +} + +func main() throws { + guard ProcessInfo.processInfo.arguments.count >= 3 else { + throw GenericError( + """ + Expected two arguments: + - Repository name, eg. `swiftlang/swift-syntax + - PR number + """ + ) + } + let repository = ProcessInfo.processInfo.arguments[1] + let prNumber = ProcessInfo.processInfo.arguments[2] + + let crossRepoPrs = try getCrossRepoPrs(repository: repository, prNumber: prNumber) + if !crossRepoPrs.isEmpty { + print("Detected cross-repo PRs") + for crossRepoPr in crossRepoPrs { + print(" - \(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName)#\(crossRepoPr.prNumber)") + } + } + + for crossRepoPr in crossRepoPrs { + let git = try lookup(executable: "git") + let swift = try lookup(executable: "swift") + let baseBranch = try getPRInfo( + repository: "\(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName)", + prNumber: crossRepoPr.prNumber + ).base.ref + + let workspaceDir = URL(fileURLWithPath: "..") + let repoDir = workspaceDir.appendingPathComponent(crossRepoPr.repositoryName) + try run( + git, + "clone", + "https://github.com/\(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName).git", + "\(crossRepoPr.repositoryName)", + workingDirectory: workspaceDir + ) + try run(git, "fetch", "origin", "pull/\(crossRepoPr.prNumber)/merge:pr_merge", workingDirectory: repoDir) + try run(git, "checkout", baseBranch, workingDirectory: repoDir) + try run(git, "reset", "--hard", "pr_merge", workingDirectory: repoDir) + try run( + swift, + "package", + "config", + "set-mirror", + "--package-url", + "https://github.com/\(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName).git", + "--mirror-url", + repoDir.resolvingSymlinksInPath().path + ) + } +} + +do { + try main() +} catch { + print(error) + #if os(Windows) + _Exit(1) + #else + exit(1) + #endif +} diff --git a/cross-pr-checkout/.gitignore b/cross-pr-checkout/.gitignore new file mode 100644 index 00000000..0023a534 --- /dev/null +++ b/cross-pr-checkout/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/cross-pr-checkout/Package.swift b/cross-pr-checkout/Package.swift new file mode 100644 index 00000000..969d982c --- /dev/null +++ b/cross-pr-checkout/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 5.8 +import PackageDescription + +let package = Package( + name: "cross-pr-checkout", + platforms: [.macOS(.v13)], + targets: [ + .executableTarget(name: "cross-pr-checkout") + ] +) diff --git a/cross-pr-checkout/Sources/cross-pr-checkout/main.swift b/cross-pr-checkout/Sources/cross-pr-checkout/main.swift new file mode 100644 index 00000000..c73dbede --- /dev/null +++ b/cross-pr-checkout/Sources/cross-pr-checkout/main.swift @@ -0,0 +1,234 @@ +import Foundation + +#if canImport(FoundationNetworking) +import FoundationNetworking +#endif + +#if canImport(WinSDK) +import WinSDK +#endif + +struct GenericError: Error, CustomStringConvertible { + var description: String + + init(_ description: String) { + self.description = description + } +} + +/// Escape the given command to be printed for log output. +func escapeCommand(_ executable: URL, _ arguments: [String]) -> String { + return ([executable.path] + arguments).map { + if $0.contains(" ") { + return "'\($0)'" + } + return $0 + }.joined(separator: " ") +} + +/// Launch a subprocess with the given command and wait for it to finish +func run(_ executable: URL, _ arguments: String..., workingDirectory: URL? = nil) throws { + print("Running \(escapeCommand(executable, arguments)) (working directory: \(workingDirectory?.path ?? ""))") + let process = Process() + process.executableURL = executable + process.arguments = arguments + if let workingDirectory { + process.currentDirectoryURL = workingDirectory + } + + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { + throw GenericError( + "\(escapeCommand(executable, arguments)) failed with non-zero exit code: \(process.terminationStatus)" + ) + } +} + +/// Find the executable with the given name +public func lookup(executable: String) throws -> URL { + // Compute search paths from PATH variable. + #if os(Windows) + let pathSeparator: Character = ";" + let executable = executable + ".exe" + #else + let pathSeparator: Character = ":" + #endif + for pathVariable in ["PATH", "Path"] { + guard let pathString = ProcessInfo.processInfo.environment[pathVariable] else { + continue + } + for searchPath in pathString.split(separator: pathSeparator) { + let candidateUrl = URL(fileURLWithPath: String(searchPath)).appendingPathComponent(executable) + if FileManager.default.isExecutableFile(atPath: candidateUrl.path) { + return candidateUrl + } + } + } + throw GenericError("Did not find \(executable)") +} + +/// The JSON fields of the `https://api.github.com/repos/\(repository)/pulls/\(prNumber)` endpoint that we care about. +struct PRInfo: Codable { + struct Base: Codable { + /// The name of the PR's base branch. + let ref: String + } + /// The base branch of the PR + let base: Base + + /// The PR's description. + let body: String? +} + +func downloadData(from url: URL) async throws -> Data { + return try await withCheckedThrowingContinuation { continuation in + URLSession.shared.dataTask(with: url) { data, _, error in + if let error { + continuation.resume(throwing: error) + } + guard let data else { + continuation.resume(throwing: GenericError("Received no data for \(url)")) + return + } + continuation.resume(returning: data) + } + .resume() + } +} + +/// - Parameters: +/// - repository: The repository's name, eg. `swiftlang/swift-syntax` +func getPRInfo(repository: String, prNumber: String) async throws -> PRInfo { + guard let prInfoUrl = URL(string: "https://api.github.com/repos/\(repository)/pulls/\(prNumber)") else { + throw GenericError("Failed to form URL for GitHub API") + } + + do { + let data = try await downloadData(from: prInfoUrl) + return try JSONDecoder().decode(PRInfo.self, from: data) + } catch { + throw GenericError("Failed to load PR info from \(prInfoUrl): \(error)") + } +} + +/// Information about a PR that should be tested with this PR. +struct CrossRepoPR { + /// The owner of the repository, eg. `swiftlang` + let repositoryOwner: String + + /// The name of the repository, eg. `swift-syntax` + let repositoryName: String + + /// The PR number that's referenced. + let prNumber: String +} + +/// Retrieve all PRs that are referenced from the PR with the given number in `repository`. +/// `repository` is the owner and repo name joined by `/`, eg. `swiftlang/swift-syntax`. +func getCrossRepoPrs(repository: String, prNumber: String) async throws -> [CrossRepoPR] { + var result: [CrossRepoPR] = [] + let prInfo = try await getPRInfo(repository: repository, prNumber: prNumber) + for line in prInfo.body?.split(separator: "\n") ?? [] { + guard line.lowercased().starts(with: "linked pr:") else { + continue + } + // We can't use Swift's Regex here because this script needs to run on Windows with Swift 5.9, which doesn't support + // Swift Regex. + var remainder = line[...] + guard let ownerRange = remainder.firstRange(of: "swiftlang/") ?? remainder.firstRange(of: "apple/") else { + continue + } + let repositoryOwner = remainder[ownerRange].dropLast() + remainder = remainder[ownerRange.upperBound...] + let repositoryName = remainder.prefix { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" } + if repositoryName.isEmpty { + continue + } + remainder = remainder.dropFirst(repositoryName.count) + if remainder.starts(with: "/pull/") { + remainder = remainder.dropFirst(6) + } else if remainder.starts(with: "#") { + remainder = remainder.dropFirst() + } else { + continue + } + let pullRequestNum = remainder.prefix { $0.isNumber } + if pullRequestNum.isEmpty { + continue + } + result.append( + CrossRepoPR( + repositoryOwner: String(repositoryOwner), + repositoryName: String(repositoryName), + prNumber: String(pullRequestNum) + ) + ) + } + return result +} + +func main() async throws { + guard ProcessInfo.processInfo.arguments.count >= 3 else { + throw GenericError( + """ + Expected two arguments: + - Repository name, eg. `swiftlang/swift-syntax + - PR number + """ + ) + } + let repository = ProcessInfo.processInfo.arguments[1] + let prNumber = ProcessInfo.processInfo.arguments[2] + + let crossRepoPrs = try await getCrossRepoPrs(repository: repository, prNumber: prNumber) + if !crossRepoPrs.isEmpty { + print("Detected cross-repo PRs") + for crossRepoPr in crossRepoPrs { + print(" - \(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName)#\(crossRepoPr.prNumber)") + } + } + + for crossRepoPr in crossRepoPrs { + let git = try lookup(executable: "git") + let swift = try lookup(executable: "swift") + let baseBranch = try await getPRInfo( + repository: "\(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName)", + prNumber: crossRepoPr.prNumber + ).base.ref + + let workspaceDir = URL(fileURLWithPath: "..").resolvingSymlinksInPath() + let repoDir = workspaceDir.appendingPathComponent(crossRepoPr.repositoryName) + try run( + git, + "clone", + "https://github.com/\(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName).git", + "\(crossRepoPr.repositoryName)", + workingDirectory: workspaceDir + ) + try run(git, "fetch", "origin", "pull/\(crossRepoPr.prNumber)/merge:pr_merge", workingDirectory: repoDir) + try run(git, "checkout", baseBranch, workingDirectory: repoDir) + try run(git, "reset", "--hard", "pr_merge", workingDirectory: repoDir) + try run( + swift, + "package", + "config", + "set-mirror", + "--package-url", + "https://github.com/\(crossRepoPr.repositoryOwner)/\(crossRepoPr.repositoryName).git", + "--mirror-url", + repoDir.path + ) + } +} + +do { + try await main() +} catch { + print(error) + #if os(Windows) + _Exit(1) + #else + exit(1) + #endif +}