diff --git a/CMakeLists.txt b/CMakeLists.txt index e3ba564143c..1411371089d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,7 @@ if(FIND_PM_DEPS) find_package(SwiftCertificates CONFIG REQUIRED) find_package(SwiftCrypto CONFIG REQUIRED) find_package(SwiftBuild CONFIG REQUIRED) + find_package(SwiftSubprocess CONFIG REQUIRED) endif() find_package(dispatch QUIET) diff --git a/Package.swift b/Package.swift index aa32ba1e384..21ce8d114e4 100644 --- a/Package.swift +++ b/Package.swift @@ -239,6 +239,7 @@ let package = Package( dependencies: [ "_AsyncFileSystem", .target(name: "SPMSQLite3", condition: .when(platforms: [.macOS, .iOS, .tvOS, .watchOS, .visionOS, .macCatalyst, .linux, .openbsd, .custom("freebsd")])), + .product(name: "Subprocess", package: "swift-subprocess"), .product(name: "SwiftToolchainCSQLite", package: "swift-toolchain-sqlite", condition: .when(platforms: [.windows, .android])), .product(name: "DequeModule", package: "swift-collections"), .product(name: "OrderedCollections", package: "swift-collections"), @@ -575,6 +576,7 @@ let package = Package( dependencies: [ .product(name: "ArgumentParser", package: "swift-argument-parser"), .product(name: "OrderedCollections", package: "swift-collections"), + .product(name: "Subprocess", package: "swift-subprocess"), "Basics", "BinarySymbols", "Build", @@ -1113,6 +1115,7 @@ if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { .package(url: "https://github.com/apple/swift-system.git", revision: "1.5.0"), .package(url: "https://github.com/apple/swift-collections.git", revision: "1.1.6"), .package(url: "https://github.com/apple/swift-certificates.git", revision: "1.10.1"), + .package(url: "https://github.com/swiftlang/swift-subprocess.git", .upToNextMinor(from: "0.2.0")), .package(url: "https://github.com/swiftlang/swift-toolchain-sqlite.git", revision: "1.0.7"), // Not in toolchain, used for use in previewing documentation .package(url: "https://github.com/swiftlang/swift-docc-plugin", from: "1.1.0"), @@ -1131,6 +1134,7 @@ if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { .package(path: "../swift-system"), .package(path: "../swift-collections"), .package(path: "../swift-certificates"), + .package(path: "../swift-subprocess"), .package(path: "../swift-toolchain-sqlite"), ] if !swiftDriverDeps.isEmpty { diff --git a/Sources/Basics/CMakeLists.txt b/Sources/Basics/CMakeLists.txt index e2910d09b81..01f6994859c 100644 --- a/Sources/Basics/CMakeLists.txt +++ b/Sources/Basics/CMakeLists.txt @@ -73,6 +73,7 @@ add_library(Basics Serialization/SerializedJSON.swift SwiftVersion.swift SQLiteBackedCache.swift + Subprocess+Extensions.swift TestingLibrary.swift Triple+Basics.swift URL.swift @@ -83,6 +84,7 @@ add_library(Basics target_link_libraries(Basics PUBLIC _AsyncFileSystem SwiftCollections::OrderedCollections + SwiftSubprocess::Subprocess TSCBasic TSCUtility) target_link_libraries(Basics PRIVATE diff --git a/Sources/Basics/Cancellator.swift b/Sources/Basics/Cancellator.swift index 9cb30617d57..094864c6b52 100644 --- a/Sources/Basics/Cancellator.swift +++ b/Sources/Basics/Cancellator.swift @@ -190,6 +190,17 @@ public final class Cancellator: Cancellable, Sendable { } } +extension Cancellator { + public func run(name: String, _ block: @escaping @Sendable () async throws -> T) async throws -> T { + let task = Task { try await block() } + let token = register(name: name, handler: { _ in task.cancel() }) + defer { + token.map { deregister($0) } + } + return try await task.value + } +} + public protocol Cancellable { func cancel(deadline: DispatchTime) throws -> Void } diff --git a/Sources/Basics/Subprocess+Extensions.swift b/Sources/Basics/Subprocess+Extensions.swift new file mode 100644 index 00000000000..83b9a1d1885 --- /dev/null +++ b/Sources/Basics/Subprocess+Extensions.swift @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Subprocess +import TSCBasic + +#if canImport(System) +import System +#else +import SystemPackage +#endif + +extension Subprocess.Environment { + public init(_ env: Basics.Environment) { + var newEnv: [Subprocess.Environment.Key: String] = [:] + for (key, value) in env { + newEnv[.init(rawValue: key.rawValue)!] = value + } + self = Subprocess.Environment.custom(newEnv) + } +} + +extension Subprocess.Configuration { + public init(commandLine: [String], environment: Subprocess.Environment) throws { + guard let arg0 = commandLine.first else { + throw StringError("command line was unexpectedly empty") + } + self.init(.path(FilePath(arg0)), arguments: .init(Array(commandLine.dropFirst())), environment: environment) + } +} diff --git a/Sources/Commands/CMakeLists.txt b/Sources/Commands/CMakeLists.txt index 303944aebcc..a9d15ba329a 100644 --- a/Sources/Commands/CMakeLists.txt +++ b/Sources/Commands/CMakeLists.txt @@ -61,6 +61,7 @@ add_library(Commands target_link_libraries(Commands PUBLIC SwiftCollections::OrderedCollections SwiftSyntax::SwiftRefactor + SwiftSubprocess::Subprocess ArgumentParser Basics BinarySymbols diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index d2eefa5fd97..274a2de4d6f 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -11,6 +11,12 @@ //===----------------------------------------------------------------------===// import ArgumentParser +import Subprocess +#if canImport(System) +import System +#else +import SystemPackage +#endif @_spi(SwiftPMInternal) import Basics @@ -342,7 +348,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { observabilityScope: swiftCommandState.observabilityScope ) - testResults = try runner.run(tests) + testResults = try await runner.run(tests) result = runner.ranSuccessfully ? .success : .failure } @@ -538,7 +544,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) // Finally, run the tests. - return runner.test(outputHandler: { + return await runner.test(outputHandler: { // command's result output goes on stdout // ie "swift test" should output to stdout print($0, terminator: "") @@ -837,7 +843,7 @@ extension SwiftTestCommand { ) // Finally, run the tests. - let result = runner.test(outputHandler: { + let result = await runner.test(outputHandler: { // command's result output goes on stdout // ie "swift test" should output to stdout print($0, terminator: "") @@ -911,7 +917,7 @@ final class TestRunner { // The toolchain to use. private let toolchain: UserToolchain - private let testEnv: Environment + private let testEnv: Basics.Environment /// ObservabilityScope to emit diagnostics. private let observabilityScope: ObservabilityScope @@ -945,7 +951,7 @@ final class TestRunner { additionalArguments: [String], cancellator: Cancellator, toolchain: UserToolchain, - testEnv: Environment, + testEnv: Basics.Environment, observabilityScope: ObservabilityScope, library: TestingLibrary ) { @@ -974,10 +980,10 @@ final class TestRunner { /// Executes and returns execution status. Prints test output on standard streams if requested /// - Returns: Result of spawning and running the test process, and the output stream result - func test(outputHandler: @escaping (String) -> Void) -> Result { + func test(outputHandler: @escaping (String) -> Void) async -> Result { var results = [Result]() for path in self.bundlePaths { - let testSuccess = self.test(at: path, outputHandler: outputHandler) + let testSuccess = await self.test(at: path, outputHandler: outputHandler) results.append(testSuccess) } return results.reduce() @@ -1021,33 +1027,27 @@ final class TestRunner { return args } - private func test(at path: AbsolutePath, outputHandler: @escaping (String) -> Void) -> Result { + private func test(at path: AbsolutePath, outputHandler: @escaping (String) -> Void) async -> Result { let testObservabilityScope = self.observabilityScope.makeChildScope(description: "running test at \(path)") do { - let outputHandler = { (bytes: [UInt8]) in - if let output = String(bytes: bytes, encoding: .utf8) { - outputHandler(output) - } - } - let outputRedirection = AsyncProcess.OutputRedirection.stream( - stdout: outputHandler, - stderr: outputHandler - ) - let process = AsyncProcess(arguments: try args(forTestAt: path), environment: self.testEnv, outputRedirection: outputRedirection) - guard let terminationKey = self.cancellator.register(process) else { - return .failure // terminating + let args = try args(forTestAt: path) + let processConfig = try Subprocess.Configuration(commandLine: args, environment: .init(self.testEnv)) + let status = try await cancellator.run(name: "Test Execution") { + try await Subprocess.run(processConfig, input: .none, error: .combineWithOutput) { execution, outputSequence in + for try await line in outputSequence.lines() { + outputHandler(line) + } + }.terminationStatus } - defer { self.cancellator.deregister(terminationKey) } - try process.launch() - let result = try process.waitUntilExit() - switch result.exitStatus { - case .terminated(code: 0): + + switch status { + case .exited(code: 0): return .success - case .terminated(code: EXIT_NO_TESTS_FOUND) where library == .swiftTesting: + case .exited(code: numericCast(EXIT_NO_TESTS_FOUND)) where library == .swiftTesting: return .noMatchingTests #if !os(Windows) - case .signalled(let signal) where ![SIGINT, SIGKILL, SIGTERM].contains(signal): + case .unhandledException(let signal) where ![SIGINT, SIGKILL, SIGTERM].contains(signal): testObservabilityScope.emit(error: "Exited with unexpected signal code \(signal)") return .failure #endif @@ -1087,21 +1087,9 @@ final class ParallelTestRunner { /// Path to XCTest binaries. private let bundlePaths: [AbsolutePath] - /// The queue containing list of tests to run (producer). - private let pendingTests = SynchronizedQueue() - - /// The queue containing tests which are finished running. - private let finishedTests = SynchronizedQueue() - /// Instance of a terminal progress animation. private let progressAnimation: ProgressAnimationProtocol - /// Number of tests that will be executed. - private var numTests = 0 - - /// Number of the current tests that has been executed. - private var numCurrentTest = 0 - /// True if all tests executed successfully. private(set) var ranSuccessfully = true @@ -1160,27 +1148,8 @@ final class ParallelTestRunner { assert(numJobs > 0, "num jobs should be > 0") } - /// Updates the progress bar status. - private func updateProgress(for test: UnitTest) { - numCurrentTest += 1 - progressAnimation.update(step: numCurrentTest, total: numTests, text: "Testing \(test.specifier)") - } - - private func enqueueTests(_ tests: [UnitTest]) throws { - // Enqueue all the tests. - for test in tests { - pendingTests.enqueue(test) - } - self.numTests = tests.count - self.numCurrentTest = 0 - // Enqueue the sentinels, we stop a thread when it encounters a sentinel in the queue. - for _ in 0.. [TestResult] { + func run(_ tests: [UnitTest]) async throws -> [TestResult] { assert(!tests.isEmpty, "There should be at least one test to execute.") let testEnv = try TestingSupport.constructTestEnvironment( @@ -1190,69 +1159,66 @@ final class ParallelTestRunner { library: .xctest // swift-testing does not use ParallelTestRunner ) - // Enqueue all the tests. - try enqueueTests(tests) - - // Create the worker threads. - let workers: [Thread] = (0.. TestResult { + observabilityScope.emit(error: "enqueuing \(test.specifier)") + let additionalArguments = TestRunner.xctestArguments(forTestSpecifiers: CollectionOfOne(test.specifier)) + let testRunner = TestRunner( + bundlePaths: [test.productPath], + additionalArguments: additionalArguments, + cancellator: self.cancellator, + toolchain: self.toolchain, + testEnv: testEnv, + observabilityScope: self.observabilityScope, + library: .xctest // swift-testing does not use ParallelTestRunner + ) + var output = "" + let start = DispatchTime.now() + let result = await testRunner.test(outputHandler: { _output in output += _output }) + let duration = start.distance(to: .now()) + if result == .failure { + self.ranSuccessfully = false + } + return TestResult( + unitTest: test, + output: output, + success: result != .failure, + duration: duration + ) + } + for _ in 0..() - - // Report (consume) the tests which have finished running. - while let result = finishedTests.dequeue() { - updateProgress(for: result.unitTest) - - // Store the result. - processedTests.append(result) + var completedTests = 0 + while let result = await group.next() { + completedTests += 1 + progressAnimation.update(step: completedTests, total: tests.count, text: "Testing \(result.unitTest.specifier)") + + if !pendingTests.isEmpty { + let test = pendingTests.removeLast() + group.addTask(operation: { + await runTest(test) + }) + } - // We can't enqueue a sentinel into finished tests queue because we won't know - // which test is last one so exit this when all the tests have finished running. - if numCurrentTest == numTests { - break + // Store the result. + processedTests.append(result) } } - // Wait till all threads finish execution. - workers.forEach { $0.join() } - // Report the completion. - progressAnimation.complete(success: processedTests.get().contains(where: { !$0.success })) + progressAnimation.complete(success: processedTests.contains(where: { !$0.success })) // Print test results. - for test in processedTests.get() { + for test in processedTests { if (!test.success || shouldOutputSuccess) && !productsBuildParameters.testingParameters.experimentalTestOutput { // command's result output goes on stdout // ie "swift test" should output to stdout @@ -1260,7 +1226,7 @@ final class ParallelTestRunner { } } - return processedTests.get() + return processedTests } } diff --git a/Sources/Commands/Utilities/PluginDelegate.swift b/Sources/Commands/Utilities/PluginDelegate.swift index e5e82f10126..2ebe17dabab 100644 --- a/Sources/Commands/Utilities/PluginDelegate.swift +++ b/Sources/Commands/Utilities/PluginDelegate.swift @@ -288,7 +288,7 @@ final class PluginDelegate: PluginInvocationDelegate { // Run the test — for now we run the sequentially so we can capture accurate timing results. let startTime = DispatchTime.now() - let result = testRunner.test(outputHandler: { _ in }) // this drops the tests output + let result = await testRunner.test(outputHandler: { _ in }) // this drops the tests output let duration = Double(startTime.distance(to: .now()).milliseconds() ?? 0) / 1000.0 numFailedTests += (result != .failure) ? 0 : 1 testResults.append( diff --git a/Utilities/bootstrap b/Utilities/bootstrap index 5072f22eb9d..d1ecc6f2c62 100755 --- a/Utilities/bootstrap +++ b/Utilities/bootstrap @@ -248,6 +248,7 @@ def parse_global_args(args): args.source_dirs["swift-asn1"] = os.path.join(args.project_root, "..", "swift-asn1") args.source_dirs["swift-syntax"] = os.path.join(args.project_root, "..", "swift-syntax") args.source_dirs["swift-build"] = os.path.join(args.project_root, "..", "swift-build") + args.source_dirs["swift-subprocess"] = os.path.join(args.project_root, "..", "swift-subprocess") args.source_root = os.path.join(args.project_root, "Sources") if platform.system() == 'Darwin': @@ -447,6 +448,8 @@ def build(args): build_dependency(args, "swift-certificates", ["-DSwiftASN1_DIR=" + os.path.join(args.build_dirs["swift-asn1"], "cmake/modules"), "-DSwiftCrypto_DIR=" + os.path.join(args.build_dirs["swift-crypto"], "cmake/modules")]) + build_dependency(args, "swift-subprocess", + ["-DSwiftSystem_DIR=" + os.path.join(args.build_dirs["swift-system"], "cmake/modules")]) swift_build_cmake_flags = [ get_llbuild_cmake_arg(args), "-DSwiftSystem_DIR=" + os.path.join(args.build_dirs["swift-system"], "cmake/modules"), @@ -732,6 +735,7 @@ def build_swiftpm_with_cmake(args): cmake_flags = [ get_llbuild_cmake_arg(args), + "-DSwiftSubprocess_DIR=" + os.path.join(args.build_dirs["swift-subprocess"], "cmake/modules"), "-DTSC_DIR=" + os.path.join(args.build_dirs["tsc"], "cmake/modules"), "-DArgumentParser_DIR=" + os.path.join(args.build_dirs["swift-argument-parser"], "cmake/modules"), "-DSwiftDriver_DIR=" + os.path.join(args.build_dirs["swift-driver"], "cmake/modules"), @@ -765,6 +769,7 @@ def build_swiftpm_with_cmake(args): add_rpath_for_cmake_build(args, args.build_dirs["llbuild"]) if platform.system() == "Darwin": + add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-subprocess"], "lib")) add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-argument-parser"], "lib")) add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-crypto"], "lib")) add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-driver"], "lib")) @@ -772,7 +777,7 @@ def build_swiftpm_with_cmake(args): add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-collections"], "lib")) add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-asn1"], "lib")) add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-certificates"], "lib")) - add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-build"], "lib")) + add_rpath_for_cmake_build(args, os.path.join(args.build_dirs["swift-build"], "lib")) # rpaths for compatibility libraries for lib_path in get_swift_backdeploy_library_paths(args): @@ -901,6 +906,7 @@ def get_swiftpm_env_cmd(args): if args.bootstrap: libs = [ os.path.join(args.bootstrap_dir, "lib"), + os.path.join(args.build_dirs["swift-subprocess"], "lib"), os.path.join(args.build_dirs["tsc"], "lib"), os.path.join(args.build_dirs["llbuild"], "lib"), os.path.join(args.build_dirs["swift-argument-parser"], "lib"),