From 2c407571f5693f45e239c8de4b093f24ad625e64 Mon Sep 17 00:00:00 2001 From: "Lokesh.T.R" Date: Fri, 24 May 2024 16:40:25 +0000 Subject: [PATCH 01/69] Change static method `DocumentURI.for(_:testName:)` to an initializer `DocumentURI(for:testName:)` --- Sources/SKTestSupport/SkipUnless.swift | 8 ++-- Sources/SKTestSupport/Utils.swift | 7 +-- .../SourceKitDTests/CrashRecoveryTests.swift | 6 +-- Tests/SourceKitDTests/SourceKitDTests.swift | 2 +- .../SourceKitLSPTests/BuildSystemTests.swift | 8 ++-- Tests/SourceKitLSPTests/CodeActionTests.swift | 20 ++++----- Tests/SourceKitLSPTests/DefinitionTests.swift | 6 +-- .../DocumentColorTests.swift | 4 +- .../DocumentSymbolTests.swift | 2 +- .../DocumentTestDiscoveryTests.swift | 44 +++++++++---------- .../ExecuteCommandTests.swift | 4 +- .../SourceKitLSPTests/FoldingRangeTests.swift | 2 +- Tests/SourceKitLSPTests/FormattingTests.swift | 8 ++-- Tests/SourceKitLSPTests/HoverTests.swift | 2 +- Tests/SourceKitLSPTests/InlayHintTests.swift | 2 +- Tests/SourceKitLSPTests/LifecycleTests.swift | 2 +- Tests/SourceKitLSPTests/LocalClangTests.swift | 10 ++--- Tests/SourceKitLSPTests/LocalSwiftTests.swift | 2 +- .../PublishDiagnosticsTests.swift | 6 +-- .../PullDiagnosticsTests.swift | 4 +- .../SourceKitLSPTests/RenameAssertions.swift | 2 +- Tests/SourceKitLSPTests/RenameTests.swift | 4 +- .../SwiftCompletionTests.swift | 36 +++++++-------- .../SwiftInterfaceTests.swift | 2 +- .../WorkspaceTestDiscoveryTests.swift | 2 +- 25 files changed, 98 insertions(+), 97 deletions(-) diff --git a/Sources/SKTestSupport/SkipUnless.swift b/Sources/SKTestSupport/SkipUnless.swift index 803b8a507..323f4aa11 100644 --- a/Sources/SKTestSupport/SkipUnless.swift +++ b/Sources/SKTestSupport/SkipUnless.swift @@ -103,7 +103,7 @@ public actor SkipUnless { ) async throws { try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument("0.bitPattern", uri: uri) let response = try unwrap( await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri))) @@ -134,7 +134,7 @@ public actor SkipUnless { ) async throws { try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument("func 1️⃣test() {}", uri: uri) do { _ = try await testClient.send( @@ -154,7 +154,7 @@ public actor SkipUnless { ) async throws { try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.c) + let uri = DocumentURI(for: .c) let positions = testClient.openDocument("void 1️⃣test() {}", uri: uri) do { _ = try await testClient.send( @@ -214,7 +214,7 @@ public actor SkipUnless { return try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(6, 0), file: file, line: line) { // The XML-based doc comment conversion did not preserve `Precondition`. let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ /// - Precondition: Must have an apple diff --git a/Sources/SKTestSupport/Utils.swift b/Sources/SKTestSupport/Utils.swift index f55ed78ae..8d7b1fd00 100644 --- a/Sources/SKTestSupport/Utils.swift +++ b/Sources/SKTestSupport/Utils.swift @@ -36,8 +36,8 @@ extension Language { } extension DocumentURI { - /// Create a unique URI for a document of the given language. - public static func `for`(_ language: Language, testName: String = #function) -> DocumentURI { + /// Construct a `DocumentURI` by creating a unique URI for a document of the given language. + public init(for language: Language, testName: String = #function) { let testBaseName = testName.prefix(while: \.isLetter) #if os(Windows) @@ -45,7 +45,8 @@ extension DocumentURI { #else let url = URL(fileURLWithPath: "/\(testBaseName)/\(UUID())/test.\(language.fileExtension)") #endif - return DocumentURI(url) + + self.init(url) } } diff --git a/Tests/SourceKitDTests/CrashRecoveryTests.swift b/Tests/SourceKitDTests/CrashRecoveryTests.swift index 3feebc7eb..eba61bf47 100644 --- a/Tests/SourceKitDTests/CrashRecoveryTests.swift +++ b/Tests/SourceKitDTests/CrashRecoveryTests.swift @@ -52,7 +52,7 @@ final class CrashRecoveryTests: XCTestCase { capabilities: ClientCapabilities(window: WindowClientCapabilities(workDoneProgress: true)), usePullDiagnostics: false ) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -150,7 +150,7 @@ final class CrashRecoveryTests: XCTestCase { try SkipUnless.longTestsEnabled() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.cpp) + let uri = DocumentURI(for: .cpp) let positions = testClient.openDocument("1️⃣", uri: uri) @@ -256,7 +256,7 @@ final class CrashRecoveryTests: XCTestCase { try SkipUnless.longTestsEnabled() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.cpp) + let uri = DocumentURI(for: .cpp) let positions = testClient.openDocument("1️⃣", uri: uri) diff --git a/Tests/SourceKitDTests/SourceKitDTests.swift b/Tests/SourceKitDTests/SourceKitDTests.swift index 6a4dcb10f..0d5a3c5bc 100644 --- a/Tests/SourceKitDTests/SourceKitDTests.swift +++ b/Tests/SourceKitDTests/SourceKitDTests.swift @@ -30,7 +30,7 @@ final class SourceKitDTests: XCTestCase { let sourcekitdPath = await ToolchainRegistry.forTesting.default!.sourcekitd! let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitdPath) let keys = sourcekitd.keys - let path = DocumentURI.for(.swift).pseudoPath + let path = DocumentURI(for: .swift).pseudoPath let isExpectedNotification = { @Sendable (response: SKDResponse) -> Bool in if let notification: sourcekitd_api_uid_t = response.value?[keys.notification], diff --git a/Tests/SourceKitLSPTests/BuildSystemTests.swift b/Tests/SourceKitLSPTests/BuildSystemTests.swift index 378df97f8..a94c48eff 100644 --- a/Tests/SourceKitLSPTests/BuildSystemTests.swift +++ b/Tests/SourceKitLSPTests/BuildSystemTests.swift @@ -159,7 +159,7 @@ final class BuildSystemTests: XCTestCase { func testClangdDocumentUpdatedBuildSettings() async throws { guard haveClangd else { return } - let doc = DocumentURI.for(.objective_c) + let doc = DocumentURI(for: .objective_c) let args = [doc.pseudoPath, "-DDEBUG"] let text = """ #ifdef FOO @@ -201,7 +201,7 @@ final class BuildSystemTests: XCTestCase { } func testSwiftDocumentUpdatedBuildSettings() async throws { - let doc = DocumentURI.for(.swift) + let doc = DocumentURI(for: .swift) let args = await FallbackBuildSystem(buildSetup: .default) .buildSettings(for: doc, language: .swift)! .compilerArguments @@ -236,7 +236,7 @@ final class BuildSystemTests: XCTestCase { } func testClangdDocumentFallbackWithholdsDiagnostics() async throws { - let doc = DocumentURI.for(.objective_c) + let doc = DocumentURI(for: .objective_c) let args = [doc.pseudoPath, "-DDEBUG"] let text = """ #ifdef FOO @@ -270,7 +270,7 @@ final class BuildSystemTests: XCTestCase { } func testSwiftDocumentFallbackWithholdsSemanticDiagnostics() async throws { - let doc = DocumentURI.for(.swift) + let doc = DocumentURI(for: .swift) // Primary settings must be different than the fallback settings. var primarySettings = await FallbackBuildSystem(buildSetup: .default).buildSettings(for: doc, language: .swift)! diff --git a/Tests/SourceKitLSPTests/CodeActionTests.swift b/Tests/SourceKitLSPTests/CodeActionTests.swift index e490e7369..63a021833 100644 --- a/Tests/SourceKitLSPTests/CodeActionTests.swift +++ b/Tests/SourceKitLSPTests/CodeActionTests.swift @@ -191,7 +191,7 @@ final class CodeActionTests: XCTestCase { func testEmptyCodeActionResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ func foo() -> String { @@ -214,7 +214,7 @@ final class CodeActionTests: XCTestCase { func testSemanticRefactorLocalRenameResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ func localRename() { @@ -240,7 +240,7 @@ final class CodeActionTests: XCTestCase { func testSemanticRefactorLocationCodeActionResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ func foo() -> String { @@ -297,7 +297,7 @@ final class CodeActionTests: XCTestCase { func testJSONCodableCodeActionResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ 1️⃣{ @@ -339,7 +339,7 @@ final class CodeActionTests: XCTestCase { func testSemanticRefactorRangeCodeActionResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ func foo() -> String { @@ -410,7 +410,7 @@ final class CodeActionTests: XCTestCase { capabilities: clientCapabilitiesWithCodeActionSupport, usePullDiagnostics: false ) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -497,7 +497,7 @@ final class CodeActionTests: XCTestCase { func testAddDocumentationCodeActionResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ 2️⃣func refacto1️⃣r(syntax: DeclSyntax, in context: Void) -> DeclSyntax? { }3️⃣ @@ -576,7 +576,7 @@ final class CodeActionTests: XCTestCase { func testPackageManifestEditingCodeActionResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ // swift-tools-version: 5.5 @@ -650,7 +650,7 @@ final class CodeActionTests: XCTestCase { func testPackageManifestEditingCodeActionNoTestResult() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ // swift-tools-version: 5.5 @@ -1014,7 +1014,7 @@ final class CodeActionTests: XCTestCase { line: UInt = #line ) async throws { let testClient = try await TestSourceKitLSPClient(capabilities: clientCapabilitiesWithCodeActionSupport) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument(markedText, uri: uri) var ranges = ranges diff --git a/Tests/SourceKitLSPTests/DefinitionTests.swift b/Tests/SourceKitLSPTests/DefinitionTests.swift index 23b5d6946..cdaa34739 100644 --- a/Tests/SourceKitLSPTests/DefinitionTests.swift +++ b/Tests/SourceKitLSPTests/DefinitionTests.swift @@ -19,7 +19,7 @@ import enum PackageLoading.Platform class DefinitionTests: XCTestCase { func testJumpToDefinitionAtEndOfIdentifier() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -200,7 +200,7 @@ class DefinitionTests: XCTestCase { func testReportInitializerOnDefinitionForType() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ struct 1️⃣Foo { @@ -353,7 +353,7 @@ class DefinitionTests: XCTestCase { func testDefinitionOfImplicitInitializer() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ diff --git a/Tests/SourceKitLSPTests/DocumentColorTests.swift b/Tests/SourceKitLSPTests/DocumentColorTests.swift index 66744d057..c92cce7fb 100644 --- a/Tests/SourceKitLSPTests/DocumentColorTests.swift +++ b/Tests/SourceKitLSPTests/DocumentColorTests.swift @@ -22,7 +22,7 @@ final class DocumentColorTests: XCTestCase { private func performDocumentColorRequest(text: String) async throws -> [ColorInformation] { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument(text, uri: uri) @@ -37,7 +37,7 @@ final class DocumentColorTests: XCTestCase { ) async throws -> [ColorPresentation] { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument(text, uri: uri) diff --git a/Tests/SourceKitLSPTests/DocumentSymbolTests.swift b/Tests/SourceKitLSPTests/DocumentSymbolTests.swift index f41020f58..03b853f82 100644 --- a/Tests/SourceKitLSPTests/DocumentSymbolTests.swift +++ b/Tests/SourceKitLSPTests/DocumentSymbolTests.swift @@ -738,7 +738,7 @@ fileprivate func assertDocumentSymbols( line: UInt = #line ) async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument(markedText, uri: uri) let symbols = try unwrap(try await testClient.send(DocumentSymbolRequest(textDocument: TextDocumentIdentifier(uri)))) diff --git a/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift b/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift index cd4a835fb..cf3cb4180 100644 --- a/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift +++ b/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift @@ -81,7 +81,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSyntacticDocumentTestsSwift() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -197,7 +197,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingDocumentTests() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -286,7 +286,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testNestedSwiftTestingSuites() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -343,7 +343,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testParameterizedSwiftTestingTest() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -388,7 +388,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testParameterizedSwiftTestingTestWithAnonymousArgument() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -433,7 +433,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testParameterizedSwiftTestingTestWithCommentInSignature() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -478,7 +478,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingSuiteWithNoTests() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -510,7 +510,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingSuiteWithCustomName() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -542,7 +542,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingTestWithCustomName() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -575,7 +575,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testDisabledSwiftTestingTest() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -608,7 +608,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingTestInDisabledSuite() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -654,7 +654,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testHiddenSwiftTestingTest() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ @@ -677,7 +677,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingTestWithTags() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -723,7 +723,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingTestWithCustomTags() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -785,7 +785,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingTestsWithExtension() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -846,7 +846,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingTestSuitesWithExtension() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -901,7 +901,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testXCTestTestsWithExtension() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -945,7 +945,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingNestedTestSuiteWithExtension() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -1012,7 +1012,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingExtensionOfTypeInAnotherFile() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -1057,7 +1057,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingExtensionOfNestedType() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -1106,7 +1106,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testSwiftTestingTwoExtensionsNoDeclaration() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -1161,7 +1161,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { func testFullyQualifySwiftTestingTestAttribute() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ diff --git a/Tests/SourceKitLSPTests/ExecuteCommandTests.swift b/Tests/SourceKitLSPTests/ExecuteCommandTests.swift index 3fce1b50f..68536b259 100644 --- a/Tests/SourceKitLSPTests/ExecuteCommandTests.swift +++ b/Tests/SourceKitLSPTests/ExecuteCommandTests.swift @@ -19,7 +19,7 @@ import XCTest final class ExecuteCommandTests: XCTestCase { func testLocationSemanticRefactoring() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -74,7 +74,7 @@ final class ExecuteCommandTests: XCTestCase { func testRangeSemanticRefactoring() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ diff --git a/Tests/SourceKitLSPTests/FoldingRangeTests.swift b/Tests/SourceKitLSPTests/FoldingRangeTests.swift index dad8ea8f8..0798edc67 100644 --- a/Tests/SourceKitLSPTests/FoldingRangeTests.swift +++ b/Tests/SourceKitLSPTests/FoldingRangeTests.swift @@ -57,7 +57,7 @@ func assertFoldingRanges( ) ) let testClient = try await TestSourceKitLSPClient(capabilities: capabilities) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument(markedSource, uri: uri) let foldingRanges = try unwrap(await testClient.send(FoldingRangeRequest(textDocument: TextDocumentIdentifier(uri)))) if foldingRanges.count != expectedRanges.count { diff --git a/Tests/SourceKitLSPTests/FormattingTests.swift b/Tests/SourceKitLSPTests/FormattingTests.swift index bac9f6a8d..db2c3fe34 100644 --- a/Tests/SourceKitLSPTests/FormattingTests.swift +++ b/Tests/SourceKitLSPTests/FormattingTests.swift @@ -21,7 +21,7 @@ final class FormattingTests: XCTestCase { func testFormatting() async throws { try await SkipUnless.toolchainContainsSwiftFormat() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -55,7 +55,7 @@ final class FormattingTests: XCTestCase { func testFormattingNoEdits() async throws { try await SkipUnless.toolchainContainsSwiftFormat() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ @@ -219,7 +219,7 @@ final class FormattingTests: XCTestCase { func testInsertAndRemove() async throws { try await SkipUnless.toolchainContainsSwiftFormat() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ @@ -251,7 +251,7 @@ final class FormattingTests: XCTestCase { func testMultiLineStringInsertion() async throws { try await SkipUnless.toolchainContainsSwiftFormat() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( #""" diff --git a/Tests/SourceKitLSPTests/HoverTests.swift b/Tests/SourceKitLSPTests/HoverTests.swift index 95594ee67..747972c12 100644 --- a/Tests/SourceKitLSPTests/HoverTests.swift +++ b/Tests/SourceKitLSPTests/HoverTests.swift @@ -181,7 +181,7 @@ private func assertHover( line: UInt = #line ) async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument(markedSource, uri: uri) diff --git a/Tests/SourceKitLSPTests/InlayHintTests.swift b/Tests/SourceKitLSPTests/InlayHintTests.swift index acb61bd36..34dd19780 100644 --- a/Tests/SourceKitLSPTests/InlayHintTests.swift +++ b/Tests/SourceKitLSPTests/InlayHintTests.swift @@ -21,7 +21,7 @@ final class InlayHintTests: XCTestCase { func performInlayHintRequest(text: String, range: Range? = nil) async throws -> [InlayHint] { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument(text, uri: uri) diff --git a/Tests/SourceKitLSPTests/LifecycleTests.swift b/Tests/SourceKitLSPTests/LifecycleTests.swift index 991b85e25..0ca5b4524 100644 --- a/Tests/SourceKitLSPTests/LifecycleTests.swift +++ b/Tests/SourceKitLSPTests/LifecycleTests.swift @@ -41,7 +41,7 @@ final class LifecycleTests: XCTestCase { func testCancellation() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ class Foo { diff --git a/Tests/SourceKitLSPTests/LocalClangTests.swift b/Tests/SourceKitLSPTests/LocalClangTests.swift index c06a9cb11..aabd2d6ec 100644 --- a/Tests/SourceKitLSPTests/LocalClangTests.swift +++ b/Tests/SourceKitLSPTests/LocalClangTests.swift @@ -21,7 +21,7 @@ final class LocalClangTests: XCTestCase { func testSymbolInfo() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.cpp) + let uri = DocumentURI(for: .cpp) let locations = testClient.openDocument( """ @@ -96,7 +96,7 @@ final class LocalClangTests: XCTestCase { func testFoldingRange() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.cpp) + let uri = DocumentURI(for: .cpp) testClient.openDocument( """ @@ -133,7 +133,7 @@ final class LocalClangTests: XCTestCase { ) ) ) - let uri = DocumentURI.for(.cpp) + let uri = DocumentURI(for: .cpp) testClient.openDocument( """ @@ -161,7 +161,7 @@ final class LocalClangTests: XCTestCase { func testCodeAction() async throws { let testClient = try await TestSourceKitLSPClient(usePullDiagnostics: false) - let uri = DocumentURI.for(.cpp) + let uri = DocumentURI(for: .cpp) let positions = testClient.openDocument( """ enum Color { RED, GREEN, BLUE }; @@ -284,7 +284,7 @@ final class LocalClangTests: XCTestCase { func testSemanticHighlighting() async throws { let testClient = try await TestSourceKitLSPClient(usePullDiagnostics: false) - let uri = DocumentURI.for(.c) + let uri = DocumentURI(for: .c) testClient.openDocument( """ diff --git a/Tests/SourceKitLSPTests/LocalSwiftTests.swift b/Tests/SourceKitLSPTests/LocalSwiftTests.swift index bbde96fbf..58d2fcb3f 100644 --- a/Tests/SourceKitLSPTests/LocalSwiftTests.swift +++ b/Tests/SourceKitLSPTests/LocalSwiftTests.swift @@ -35,7 +35,7 @@ final class LocalSwiftTests: XCTestCase { func testEditing() async throws { let testClient = try await TestSourceKitLSPClient(usePullDiagnostics: false) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let documentManager = await testClient.server._documentManager diff --git a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift index 10799451b..e8e5bd605 100644 --- a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift @@ -18,7 +18,7 @@ import XCTest final class PublishDiagnosticsTests: XCTestCase { func testUnknownIdentifierDiagnostic() async throws { let testClient = try await TestSourceKitLSPClient(usePullDiagnostics: false) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ @@ -39,7 +39,7 @@ final class PublishDiagnosticsTests: XCTestCase { func testRangeShiftAfterNewlineAdded() async throws { let testClient = try await TestSourceKitLSPClient(usePullDiagnostics: false) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ @@ -80,7 +80,7 @@ final class PublishDiagnosticsTests: XCTestCase { func testRangeShiftAfterNewlineRemoved() async throws { let testClient = try await TestSourceKitLSPClient(usePullDiagnostics: false) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ diff --git a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift index 5b4268bc8..e871574f9 100644 --- a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift @@ -18,7 +18,7 @@ import XCTest final class PullDiagnosticsTests: XCTestCase { func testUnknownIdentifierDiagnostic() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ @@ -50,7 +50,7 @@ final class PullDiagnosticsTests: XCTestCase { ) ) ) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ diff --git a/Tests/SourceKitLSPTests/RenameAssertions.swift b/Tests/SourceKitLSPTests/RenameAssertions.swift index e61a1f09c..eb2f88256 100644 --- a/Tests/SourceKitLSPTests/RenameAssertions.swift +++ b/Tests/SourceKitLSPTests/RenameAssertions.swift @@ -44,7 +44,7 @@ func assertSingleFileRename( ) async throws { try await SkipUnless.sourcekitdSupportsRename() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(language, testName: testName) + let uri = DocumentURI(for: language, testName: testName) let positions = testClient.openDocument(markedSource, uri: uri, language: language) guard !positions.allMarkers.isEmpty else { XCTFail("Test case did not contain any markers at which to invoke the rename", file: file, line: line) diff --git a/Tests/SourceKitLSPTests/RenameTests.swift b/Tests/SourceKitLSPTests/RenameTests.swift index 75cc5ad68..b169a1676 100644 --- a/Tests/SourceKitLSPTests/RenameTests.swift +++ b/Tests/SourceKitLSPTests/RenameTests.swift @@ -658,7 +658,7 @@ final class RenameTests: XCTestCase { func testPrepeareRenameOnDefinition() async throws { try await SkipUnless.sourcekitdSupportsRename() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ func 1️⃣foo2️⃣(3️⃣a: Int) {} @@ -677,7 +677,7 @@ final class RenameTests: XCTestCase { func testPrepeareRenameOnReference() async throws { try await SkipUnless.sourcekitdSupportsRename() let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ func foo(a: Int, b: Int = 1) {} diff --git a/Tests/SourceKitLSPTests/SwiftCompletionTests.swift b/Tests/SourceKitLSPTests/SwiftCompletionTests.swift index 51601db7d..fd6faa017 100644 --- a/Tests/SourceKitLSPTests/SwiftCompletionTests.swift +++ b/Tests/SourceKitLSPTests/SwiftCompletionTests.swift @@ -63,7 +63,7 @@ final class SwiftCompletionTests: XCTestCase { func testCompletionBasic(options: SKCompletionOptions) async throws { let testClient = try await TestSourceKitLSPClient(initializationOptions: initializationOptions(for: options)) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument(text, uri: uri) @@ -129,7 +129,7 @@ final class SwiftCompletionTests: XCTestCase { func testCompletionSnippetSupport() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: snippetCapabilities) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument(text, uri: uri) func getTestMethodCompletion(_ position: Position, label: String) async throws -> CompletionItem? { @@ -183,7 +183,7 @@ final class SwiftCompletionTests: XCTestCase { func testCompletionNoSnippetSupport() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument(text, uri: uri) func getTestMethodCompletion(_ position: Position, label: String) async throws -> CompletionItem? { @@ -232,7 +232,7 @@ final class SwiftCompletionTests: XCTestCase { func testCompletionPositionServerFilter() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument("foo", uri: uri) for col in 0...3 { @@ -265,7 +265,7 @@ final class SwiftCompletionTests: XCTestCase { func testCompletionOptional() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ struct Foo { @@ -301,7 +301,7 @@ final class SwiftCompletionTests: XCTestCase { func testCompletionOverride() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ class Base { @@ -339,7 +339,7 @@ final class SwiftCompletionTests: XCTestCase { func testCompletionOverrideInNewLine() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ class Base { @@ -378,7 +378,7 @@ final class SwiftCompletionTests: XCTestCase { func testMaxResults() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ struct S { @@ -511,7 +511,7 @@ final class SwiftCompletionTests: XCTestCase { for: SKCompletionOptions(maxResults: 20) ) ) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ struct S { @@ -630,7 +630,7 @@ final class SwiftCompletionTests: XCTestCase { func testRefilterAfterIncompleteResultsWithEdits() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ struct S { @@ -784,7 +784,7 @@ final class SwiftCompletionTests: XCTestCase { /// close waits for its respective open to finish to prevent a session geting stuck open. func testSessionCloseWaitsforOpen() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) testClient.openDocument( """ struct S { @@ -888,7 +888,7 @@ final class SwiftCompletionTests: XCTestCase { func testTriggerFromIncompleteAfterStartingStringLiteral() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ func foo(_ x: String) {} @@ -947,7 +947,7 @@ final class SwiftCompletionTests: XCTestCase { func testNonAsciiCompletionFilter() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ struct Foo { @@ -981,7 +981,7 @@ final class SwiftCompletionTests: XCTestCase { func testExpandClosurePlaceholder() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: snippetCapabilities) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ struct MyArray { @@ -1029,7 +1029,7 @@ final class SwiftCompletionTests: XCTestCase { func testExpandClosurePlaceholderOnOptional() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: snippetCapabilities) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ struct MyArray { @@ -1077,7 +1077,7 @@ final class SwiftCompletionTests: XCTestCase { func testExpandMultipleClosurePlaceholders() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: snippetCapabilities) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ struct MyArray { @@ -1129,7 +1129,7 @@ final class SwiftCompletionTests: XCTestCase { func testExpandMultipleClosurePlaceholdersWithLabel() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: snippetCapabilities) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ struct MyArray { @@ -1181,7 +1181,7 @@ final class SwiftCompletionTests: XCTestCase { func testInferIndentationWhenExpandingClosurePlaceholder() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: snippetCapabilities) - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ struct MyArray { diff --git a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift index 043ea9234..fdc2fd547 100644 --- a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift +++ b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift @@ -212,7 +212,7 @@ final class SwiftInterfaceTests: XCTestCase { func testJumpToSynthesizedExtensionMethodInSystemModuleWithoutIndex() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ diff --git a/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift b/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift index 4bf07fc15..7a119d942 100644 --- a/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift @@ -626,7 +626,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { func testInMemoryFileWithFallbackBuildSystem() async throws { let testClient = try await TestSourceKitLSPClient() - let uri = DocumentURI.for(.swift) + let uri = DocumentURI(for: .swift) let positions = testClient.openDocument( """ From a858d30eaf30e76cc005b04375a1af8fc2fc45e0 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 24 May 2024 11:34:26 -0700 Subject: [PATCH 02/69] Address review comments on #1329 --- Sources/SourceKitLSP/SourceKitLSPServer.swift | 2 +- Sources/sourcekit-lsp/SourceKitLSP.swift | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index 1172400ef..85f4472f1 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -1185,7 +1185,7 @@ extension SourceKitLSPServer { nonisolated func indexTaskDidProduceResult(_ result: IndexProcessResult) { self.sendNotificationToClient( LogMessageNotification( - type: result.failed ? .info : .warning, + type: result.failed ? .warning : .info, message: """ \(result.taskDescription) finished in \(result.duration) \(result.command) diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index 57c63eec8..b8942db77 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -208,8 +208,8 @@ struct SourceKitLSP: AsyncParsableCommand { @Flag( help: """ - Show which index tasks are currently running in the indexing work done progress. \ - This produces a multi-line work done progress, which might render incorrectly depending in the editor. + When reporting index progress, show the currently running index tasks in addition to the task's count. \ + This produces a multi-line work done progress, which might render incorrectly, depending on the editor. """ ) var experimentalShowActivePreparationTasksInProgress = false From 9098e0928768fa85b105d4c2c51d6842189f42cb Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 24 May 2024 11:59:38 -0700 Subject: [PATCH 03/69] Address review comments from #1322 --- Sources/SemanticIndex/CMakeLists.txt | 2 +- .../PreparationTaskDescription.swift | 10 +++++----- .../SemanticIndex/SemanticIndexManager.swift | 20 +++++++++---------- ...tusManager.swift => UpToDateTracker.swift} | 4 ++-- .../UpdateIndexStoreTaskDescription.swift | 10 +++++----- 5 files changed, 23 insertions(+), 23 deletions(-) rename Sources/SemanticIndex/{IndexStatusManager.swift => UpToDateTracker.swift} (96%) diff --git a/Sources/SemanticIndex/CMakeLists.txt b/Sources/SemanticIndex/CMakeLists.txt index b087edaeb..e3efc74ae 100644 --- a/Sources/SemanticIndex/CMakeLists.txt +++ b/Sources/SemanticIndex/CMakeLists.txt @@ -2,12 +2,12 @@ add_library(SemanticIndex STATIC CheckedIndex.swift CompilerCommandLineOption.swift - IndexStatusManager.swift IndexTaskDescription.swift PreparationTaskDescription.swift SemanticIndexManager.swift TestHooks.swift UpdateIndexStoreTaskDescription.swift + UpToDateTracker.swift ) set_target_properties(SemanticIndex PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) diff --git a/Sources/SemanticIndex/PreparationTaskDescription.swift b/Sources/SemanticIndex/PreparationTaskDescription.swift index 66f78e8c6..c0adfeb31 100644 --- a/Sources/SemanticIndex/PreparationTaskDescription.swift +++ b/Sources/SemanticIndex/PreparationTaskDescription.swift @@ -35,7 +35,7 @@ public struct PreparationTaskDescription: IndexTaskDescription { /// The build system manager that is used to get the toolchain and build settings for the files to index. private let buildSystemManager: BuildSystemManager - private let preparationUpToDateStatus: IndexUpToDateStatusManager + private let preparationUpToDateTracker: UpToDateTracker /// See `SemanticIndexManager.indexProcessDidProduceResult` private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void @@ -59,13 +59,13 @@ public struct PreparationTaskDescription: IndexTaskDescription { init( targetsToPrepare: [ConfiguredTarget], buildSystemManager: BuildSystemManager, - preparationUpToDateStatus: IndexUpToDateStatusManager, + preparationUpToDateTracker: UpToDateTracker, indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, testHooks: IndexTestHooks ) { self.targetsToPrepare = targetsToPrepare self.buildSystemManager = buildSystemManager - self.preparationUpToDateStatus = preparationUpToDateStatus + self.preparationUpToDateTracker = preparationUpToDateTracker self.indexProcessDidProduceResult = indexProcessDidProduceResult self.testHooks = testHooks } @@ -76,7 +76,7 @@ public struct PreparationTaskDescription: IndexTaskDescription { // The last 2 digits should be sufficient to differentiate between multiple concurrently running preparation operations await withLoggingSubsystemAndScope(subsystem: indexLoggingSubsystem, scope: "preparation-\(id % 100)") { let targetsToPrepare = await targetsToPrepare.asyncFilter { - await !preparationUpToDateStatus.isUpToDate($0) + await !preparationUpToDateTracker.isUpToDate($0) }.sorted(by: { ($0.targetID, $0.runDestinationID) < ($1.targetID, $1.runDestinationID) }) @@ -114,7 +114,7 @@ public struct PreparationTaskDescription: IndexTaskDescription { } await testHooks.preparationTaskDidFinish?(self) if !Task.isCancelled { - await preparationUpToDateStatus.markUpToDate(targetsToPrepare, updateOperationStartDate: startDate) + await preparationUpToDateTracker.markUpToDate(targetsToPrepare, updateOperationStartDate: startDate) } } } diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index f01e7f218..e599557ef 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -79,9 +79,9 @@ public final actor SemanticIndexManager { /// ...). `nil` if no build graph is currently being generated. private var generateBuildGraphTask: Task? - private let preparationUpToDateStatus = IndexUpToDateStatusManager() + private let preparationUpToDateTracker = UpToDateTracker() - private let indexStoreUpToDateStatus = IndexUpToDateStatusManager() + private let indexStoreUpToDateTracker = UpToDateTracker() /// The preparation tasks that have been started and are either scheduled in the task scheduler or currently /// executing. @@ -273,23 +273,23 @@ public final actor SemanticIndexManager { // We only re-index the files that were changed and don't re-index any of their dependencies. See the // `Documentation/Files_To_Reindex.md` file. let changedFiles = events.map(\.uri) - await indexStoreUpToDateStatus.markOutOfDate(changedFiles) + await indexStoreUpToDateTracker.markOutOfDate(changedFiles) // Note that configured targets are the right abstraction layer here (instead of a non-configured target) because a // build system might have targets that include different source files. Hence a source file might be in target T // configured for macOS but not in target T configured for iOS. let targets = await changedFiles.asyncMap { await buildSystemManager.configuredTargets(for: $0) }.flatMap { $0 } if let dependentTargets = await buildSystemManager.targets(dependingOn: targets) { - await preparationUpToDateStatus.markOutOfDate(dependentTargets) + await preparationUpToDateTracker.markOutOfDate(dependentTargets) } else { - await preparationUpToDateStatus.markAllOutOfDate() + await preparationUpToDateTracker.markAllKnownOutOfDate() // `markAllOutOfDate` only marks targets out-of-date that have been indexed before. Also mark all targets with // in-progress preparation out of date. So we don't get into the following situation, which would result in an // incorrect up-to-date status of a target // - Target preparation starts for the first time // - Files changed // - Target preparation finishes. - await preparationUpToDateStatus.markOutOfDate(inProgressPreparationTasks.keys) + await preparationUpToDateTracker.markOutOfDate(inProgressPreparationTasks.keys) } await scheduleBackgroundIndex(files: changedFiles) @@ -367,7 +367,7 @@ public final actor SemanticIndexManager { // schedule two preparations of the same target in quick succession, only the first one actually performs a prepare // and the second one will be a no-op once it runs. let targetsToPrepare = await targets.asyncFilter { - await !preparationUpToDateStatus.isUpToDate($0) + await !preparationUpToDateTracker.isUpToDate($0) } guard !targetsToPrepare.isEmpty else { @@ -378,7 +378,7 @@ public final actor SemanticIndexManager { PreparationTaskDescription( targetsToPrepare: targetsToPrepare, buildSystemManager: self.buildSystemManager, - preparationUpToDateStatus: preparationUpToDateStatus, + preparationUpToDateTracker: preparationUpToDateTracker, indexProcessDidProduceResult: indexProcessDidProduceResult, testHooks: testHooks ) @@ -425,7 +425,7 @@ public final actor SemanticIndexManager { filesToIndex: filesAndTargets, buildSystemManager: self.buildSystemManager, index: index, - indexStoreUpToDateStatus: indexStoreUpToDateStatus, + indexStoreUpToDateTracker: indexStoreUpToDateTracker, indexProcessDidProduceResult: indexProcessDidProduceResult, testHooks: testHooks ) @@ -470,7 +470,7 @@ public final actor SemanticIndexManager { // schedule two indexing jobs for the same file in quick succession, only the first one actually updates the index // store and the second one will be a no-op once it runs. let outOfDateFiles = await filesToIndex(toCover: files).asyncFilter { - if await indexStoreUpToDateStatus.isUpToDate($0.sourceFile) { + if await indexStoreUpToDateTracker.isUpToDate($0.sourceFile) { return false } guard let language = await buildSystemManager.defaultLanguage(for: $0.mainFile), diff --git a/Sources/SemanticIndex/IndexStatusManager.swift b/Sources/SemanticIndex/UpToDateTracker.swift similarity index 96% rename from Sources/SemanticIndex/IndexStatusManager.swift rename to Sources/SemanticIndex/UpToDateTracker.swift index 016243bf8..a6490ec74 100644 --- a/Sources/SemanticIndex/IndexStatusManager.swift +++ b/Sources/SemanticIndex/UpToDateTracker.swift @@ -14,7 +14,7 @@ import Foundation import SKCore /// Keeps track of whether an item (a target or file to index) is up-to-date. -actor IndexUpToDateStatusManager { +actor UpToDateTracker { private enum Status { /// The item is up-to-date. case upToDate @@ -57,7 +57,7 @@ actor IndexUpToDateStatusManager { } } - func markAllOutOfDate() { + func markAllKnownOutOfDate() { markOutOfDate(status.keys) } diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift index 6e3222df3..545d17500 100644 --- a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -105,7 +105,7 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { /// The build system manager that is used to get the toolchain and build settings for the files to index. private let buildSystemManager: BuildSystemManager - private let indexStoreUpToDateStatus: IndexUpToDateStatusManager + private let indexStoreUpToDateTracker: UpToDateTracker /// A reference to the underlying index store. Used to check if the index is already up-to-date for a file, in which /// case we don't need to index it again. @@ -138,14 +138,14 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { filesToIndex: [FileAndTarget], buildSystemManager: BuildSystemManager, index: UncheckedIndex, - indexStoreUpToDateStatus: IndexUpToDateStatusManager, + indexStoreUpToDateTracker: UpToDateTracker, indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, testHooks: IndexTestHooks ) { self.filesToIndex = filesToIndex self.buildSystemManager = buildSystemManager self.index = index - self.indexStoreUpToDateStatus = indexStoreUpToDateStatus + self.indexStoreUpToDateTracker = indexStoreUpToDateTracker self.indexProcessDidProduceResult = indexProcessDidProduceResult self.testHooks = testHooks } @@ -202,7 +202,7 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { } private func updateIndexStore(forSingleFile file: FileToIndex, in target: ConfiguredTarget) async { - guard await !indexStoreUpToDateStatus.isUpToDate(file.sourceFile) else { + guard await !indexStoreUpToDateTracker.isUpToDate(file.sourceFile) else { // If we know that the file is up-to-date without having ot hit the index, do that because it's fastest. return } @@ -264,7 +264,7 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { "Not updating index store for \(file) because it is a language that is not supported by background indexing" ) } - await indexStoreUpToDateStatus.markUpToDate([file.sourceFile], updateOperationStartDate: startDate) + await indexStoreUpToDateTracker.markUpToDate([file.sourceFile], updateOperationStartDate: startDate) } private func updateIndexStore( From 5e55203f34f4ac30e36cce13c3194eb549333dad Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 25 May 2024 12:01:33 -0700 Subject: [PATCH 04/69] Switch most tests to use background indexing instead of building the project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are still a few tests left that explicitly check the before and after build state, which I didn’t modify. rdar://128697501 --- Sources/SKTestSupport/SkipUnless.swift | 6 ++--- .../SKTestSupport/SwiftPMTestProject.swift | 9 ------- .../CallHierarchyTests.swift | 2 +- Tests/SourceKitLSPTests/DefinitionTests.swift | 16 +++++-------- .../DependencyTrackingTests.swift | 2 -- .../DocumentTestDiscoveryTests.swift | 6 ++--- .../ImplementationTests.swift | 2 +- Tests/SourceKitLSPTests/IndexTests.swift | 2 +- .../MainFilesProviderTests.swift | 15 ++++++------ .../SourceKitLSPTests/RenameAssertions.swift | 2 +- Tests/SourceKitLSPTests/RenameTests.swift | 5 ++-- .../SwiftInterfaceTests.swift | 4 ++-- .../SwiftPMIntegration.swift | 4 ++-- .../WorkspaceSymbolsTests.swift | 5 ++-- .../WorkspaceTestDiscoveryTests.swift | 17 +++++++------ Tests/SourceKitLSPTests/WorkspaceTests.swift | 24 ++++++++++--------- 16 files changed, 51 insertions(+), 70 deletions(-) diff --git a/Sources/SKTestSupport/SkipUnless.swift b/Sources/SKTestSupport/SkipUnless.swift index f1ab51777..e89d12b61 100644 --- a/Sources/SKTestSupport/SkipUnless.swift +++ b/Sources/SKTestSupport/SkipUnless.swift @@ -183,10 +183,8 @@ public actor SkipUnless { line: UInt = #line ) async throws { try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) { - let workspace = try await SwiftPMTestProject( - files: ["test.swift": ""], - build: true - ) + let workspace = try await SwiftPMTestProject(files: ["test.swift": ""]) + try await SwiftPMTestProject.build(at: workspace.scratchDirectory) let modulesDirectory = workspace.scratchDirectory .appendingPathComponent(".build") .appendingPathComponent("debug") diff --git a/Sources/SKTestSupport/SwiftPMTestProject.swift b/Sources/SKTestSupport/SwiftPMTestProject.swift index 9e44786be..4861ef954 100644 --- a/Sources/SKTestSupport/SwiftPMTestProject.swift +++ b/Sources/SKTestSupport/SwiftPMTestProject.swift @@ -40,8 +40,6 @@ public class SwiftPMTestProject: MultiFileTestProject { files: [RelativeFileLocation: String], manifest: String = SwiftPMTestProject.defaultPackageManifest, workspaces: (URL) async throws -> [WorkspaceFolder] = { [WorkspaceFolder(uri: DocumentURI($0))] }, - build: Bool = false, - allowBuildFailure: Bool = false, capabilities: ClientCapabilities = ClientCapabilities(), serverOptions: SourceKitLSPServer.Options = .testDefault, enableBackgroundIndexing: Bool = false, @@ -79,13 +77,6 @@ public class SwiftPMTestProject: MultiFileTestProject { testName: testName ) - if build { - if allowBuildFailure { - try? await Self.build(at: self.scratchDirectory) - } else { - try await Self.build(at: self.scratchDirectory) - } - } if pollIndex { // Wait for the indexstore-db to finish indexing _ = try await testClient.send(PollIndexRequest()) diff --git a/Tests/SourceKitLSPTests/CallHierarchyTests.swift b/Tests/SourceKitLSPTests/CallHierarchyTests.swift index 39e15e131..bafd55af6 100644 --- a/Tests/SourceKitLSPTests/CallHierarchyTests.swift +++ b/Tests/SourceKitLSPTests/CallHierarchyTests.swift @@ -200,7 +200,7 @@ final class CallHierarchyTests: XCTestCase { void FilePathIndex::2️⃣foo() {} """, ], - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("lib.h", language: .cpp) let result = try await project.testClient.send( diff --git a/Tests/SourceKitLSPTests/DefinitionTests.swift b/Tests/SourceKitLSPTests/DefinitionTests.swift index 23b5d6946..3ce951b80 100644 --- a/Tests/SourceKitLSPTests/DefinitionTests.swift +++ b/Tests/SourceKitLSPTests/DefinitionTests.swift @@ -126,7 +126,7 @@ class DefinitionTests: XCTestCase { } """, ], - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("test.cpp") @@ -177,7 +177,7 @@ class DefinitionTests: XCTestCase { ] ) """, - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("main.swift") @@ -391,8 +391,7 @@ class DefinitionTests: XCTestCase { } """, ], - build: true, - allowBuildFailure: true + enableBackgroundIndexing: true ) let (bUri, bPositions) = try project.openDocument("FileB.swift") @@ -428,12 +427,11 @@ class DefinitionTests: XCTestCase { .locations([Location(uri: aUri, range: Range(updatedAPositions["2️⃣"]))]) ) - try await SwiftPMTestProject.build(at: project.scratchDirectory) - let afterBuilding = try await project.testClient.send( + let afterChange = try await project.testClient.send( DefinitionRequest(textDocument: TextDocumentIdentifier(bUri), position: bPositions["1️⃣"]) ) XCTAssertEqual( - afterBuilding, + afterChange, .locations([Location(uri: aUri, range: Range(updatedAPositions["2️⃣"]))]) ) } @@ -518,7 +516,7 @@ class DefinitionTests: XCTestCase { } """, ], - build: true + enableBackgroundIndexing: true ) let definitionUri = try project.uri(for: "definition.swift") @@ -550,8 +548,6 @@ class DefinitionTests: XCTestCase { ]) ) - try await SwiftPMTestProject.build(at: project.scratchDirectory) - let resultAfterFileMove = try await project.testClient.send( DefinitionRequest(textDocument: TextDocumentIdentifier(callerUri), position: callerPositions["2️⃣"]) ) diff --git a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift index 4da49dd0c..39270e89c 100644 --- a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift +++ b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift @@ -51,8 +51,6 @@ final class DependencyTrackingTests: XCTestCase { // Semantic analysis: expect module import error. XCTAssertEqual(initialDiags.diagnostics.count, 1) if let diagnostic = initialDiags.diagnostics.first { - // FIXME: The error message for the missing module is misleading on Darwin - // https://github.com/apple/swift-package-manager/issues/5925 XCTAssert( diagnostic.message.contains("Could not build Objective-C module") || diagnostic.message.contains("No such module"), diff --git a/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift b/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift index cd4a835fb..411eeccb0 100644 --- a/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift +++ b/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift @@ -48,7 +48,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { targets: [.testTarget(name: "MyLibraryTests")] ) """, - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("MyTests.swift") @@ -1304,7 +1304,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { targets: [.testTarget(name: "MyLibraryTests")] ) """, - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("Test.m") @@ -1363,7 +1363,7 @@ final class DocumentTestDiscoveryTests: XCTestCase { targets: [.testTarget(name: "MyLibraryTests")] ) """, - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("Test.m") diff --git a/Tests/SourceKitLSPTests/ImplementationTests.swift b/Tests/SourceKitLSPTests/ImplementationTests.swift index 341916e08..d3ea28163 100644 --- a/Tests/SourceKitLSPTests/ImplementationTests.swift +++ b/Tests/SourceKitLSPTests/ImplementationTests.swift @@ -296,7 +296,7 @@ final class ImplementationTests: XCTestCase { struct MyStruct: 2️⃣MyProto {} """, ], - build: true + enableBackgroundIndexing: true ) let (aUri, aPositions) = try project.openDocument("a.swift") diff --git a/Tests/SourceKitLSPTests/IndexTests.swift b/Tests/SourceKitLSPTests/IndexTests.swift index a32c42497..2c74cc992 100644 --- a/Tests/SourceKitLSPTests/IndexTests.swift +++ b/Tests/SourceKitLSPTests/IndexTests.swift @@ -49,7 +49,7 @@ final class IndexTests: XCTestCase { ] ) """, - build: true + enableBackgroundIndexing: true ) let (libAUri, libAPositions) = try project.openDocument("LibA.swift") diff --git a/Tests/SourceKitLSPTests/MainFilesProviderTests.swift b/Tests/SourceKitLSPTests/MainFilesProviderTests.swift index 492fb9e4d..9923bbd47 100644 --- a/Tests/SourceKitLSPTests/MainFilesProviderTests.swift +++ b/Tests/SourceKitLSPTests/MainFilesProviderTests.swift @@ -45,7 +45,6 @@ final class MainFilesProviderTests: XCTestCase { ] ) """, - build: false, usePullDiagnostics: false ) @@ -86,7 +85,6 @@ final class MainFilesProviderTests: XCTestCase { ] ) """, - build: false, usePullDiagnostics: false ) @@ -143,7 +141,7 @@ final class MainFilesProviderTests: XCTestCase { ] ) """, - build: true, + enableBackgroundIndexing: true, usePullDiagnostics: false ) @@ -192,7 +190,7 @@ final class MainFilesProviderTests: XCTestCase { ] ) """, - build: true, + enableBackgroundIndexing: true, usePullDiagnostics: false ) @@ -208,10 +206,11 @@ final class MainFilesProviderTests: XCTestCase { let newFancyLibraryContents = """ #include "\(project.scratchDirectory.path)/Sources/shared.h" """ - let fancyLibraryURL = try project.uri(for: "MyFancyLibrary.c").fileURL! - try newFancyLibraryContents.write(to: fancyLibraryURL, atomically: false, encoding: .utf8) - - try await SwiftPMTestProject.build(at: project.scratchDirectory) + let fancyLibraryUri = try project.uri(for: "MyFancyLibrary.c") + try newFancyLibraryContents.write(to: try XCTUnwrap(fancyLibraryUri.fileURL), atomically: false, encoding: .utf8) + project.testClient.send( + DidChangeWatchedFilesNotification(changes: [FileEvent(uri: fancyLibraryUri, type: .changed)]) + ) // 'MyFancyLibrary.c' now also includes 'shared.h'. Since it lexicographically preceeds MyLibrary, we should use its // build settings. diff --git a/Tests/SourceKitLSPTests/RenameAssertions.swift b/Tests/SourceKitLSPTests/RenameAssertions.swift index e61a1f09c..be3958604 100644 --- a/Tests/SourceKitLSPTests/RenameAssertions.swift +++ b/Tests/SourceKitLSPTests/RenameAssertions.swift @@ -145,7 +145,7 @@ func assertMultiFileRename( let project = try await SwiftPMTestProject( files: files, manifest: manifest, - build: true, + enableBackgroundIndexing: true, testName: testName ) try preRenameActions(project) diff --git a/Tests/SourceKitLSPTests/RenameTests.swift b/Tests/SourceKitLSPTests/RenameTests.swift index 75cc5ad68..4fa507098 100644 --- a/Tests/SourceKitLSPTests/RenameTests.swift +++ b/Tests/SourceKitLSPTests/RenameTests.swift @@ -1114,7 +1114,7 @@ final class RenameTests: XCTestCase { } """, ], - build: true + enableBackgroundIndexing: true ) let definitionUri = try project.uri(for: "definition.swift") @@ -1157,7 +1157,6 @@ final class RenameTests: XCTestCase { ]) ) - try await SwiftPMTestProject.build(at: project.scratchDirectory) _ = try await project.testClient.send(PollIndexRequest()) let resultAfterFileMove = try await project.testClient.send( @@ -1250,7 +1249,7 @@ final class RenameTests: XCTestCase { } """, ], - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("FileA.swift") let result = try await project.testClient.send( diff --git a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift index 043ea9234..ff8ee7bda 100644 --- a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift +++ b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift @@ -79,7 +79,7 @@ final class SwiftInterfaceTests: XCTestCase { ] ) """, - build: true + enableBackgroundIndexing: true ) let (mainUri, _) = try project.openDocument("main.swift") @@ -176,7 +176,7 @@ final class SwiftInterfaceTests: XCTestCase { ] ) """, - build: true + enableBackgroundIndexing: true ) let (mainUri, mainPositions) = try project.openDocument("main.swift") diff --git a/Tests/SourceKitLSPTests/SwiftPMIntegration.swift b/Tests/SourceKitLSPTests/SwiftPMIntegration.swift index 94c6e1de7..737fb783a 100644 --- a/Tests/SourceKitLSPTests/SwiftPMIntegration.swift +++ b/Tests/SourceKitLSPTests/SwiftPMIntegration.swift @@ -31,7 +31,7 @@ final class SwiftPMIntegrationTests: XCTestCase { } """, ], - build: true + enableBackgroundIndexing: true ) let (otherUri, otherPositions) = try project.openDocument("Other.swift") @@ -101,7 +101,7 @@ final class SwiftPMIntegrationTests: XCTestCase { } """ ], - build: true + enableBackgroundIndexing: true ) let newFileUrl = project.scratchDirectory diff --git a/Tests/SourceKitLSPTests/WorkspaceSymbolsTests.swift b/Tests/SourceKitLSPTests/WorkspaceSymbolsTests.swift index 05ac41b98..651df3245 100644 --- a/Tests/SourceKitLSPTests/WorkspaceSymbolsTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceSymbolsTests.swift @@ -60,11 +60,10 @@ class WorkspaceSymbolsTests: XCTestCase { ], workspaces: { return [WorkspaceFolder(uri: DocumentURI($0.appendingPathComponent("packageB")))] - } + }, + enableBackgroundIndexing: true ) - try await SwiftPMTestProject.build(at: project.scratchDirectory.appendingPathComponent("packageB")) - _ = try await project.testClient.send(PollIndexRequest()) let response = try await project.testClient.send(WorkspaceSymbolsRequest(query: "funcFrom")) diff --git a/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift b/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift index 4bf07fc15..8802c73d9 100644 --- a/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift @@ -46,7 +46,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """ ], manifest: packageManifestWithTestTarget, - build: true + enableBackgroundIndexing: true ) let tests = try await project.testClient.send(WorkspaceTestsRequest()) @@ -141,7 +141,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """ ], manifest: packageManifestWithTestTarget, - build: true + enableBackgroundIndexing: true ) let myTestsUri = try project.uri(for: "MyTests.swift") @@ -304,8 +304,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """ ], manifest: packageManifestWithTestTarget, - build: true, - allowBuildFailure: true + enableBackgroundIndexing: true ) let tests = try await project.testClient.send(WorkspaceTestsRequest()) @@ -371,7 +370,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """ ], manifest: packageManifestWithTestTarget, - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("MyTests.swift") @@ -464,7 +463,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """, ], manifest: packageManifestWithTestTarget, - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("MyFirstTests.swift") @@ -536,7 +535,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """ ], manifest: packageManifestWithTestTarget, - build: true + enableBackgroundIndexing: true ) let uri = try project.uri(for: "MyTests.swift") @@ -935,7 +934,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { targets: [.testTarget(name: "MyLibraryTests")] ) """, - build: true + enableBackgroundIndexing: true ) let tests = try await project.testClient.send(WorkspaceTestsRequest()) @@ -992,7 +991,7 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { targets: [.testTarget(name: "MyLibraryTests")] ) """, - build: true + enableBackgroundIndexing: true ) let (uri, positions) = try project.openDocument("Test.m") diff --git a/Tests/SourceKitLSPTests/WorkspaceTests.swift b/Tests/SourceKitLSPTests/WorkspaceTests.swift index 10af11fce..58aa8eebe 100644 --- a/Tests/SourceKitLSPTests/WorkspaceTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceTests.swift @@ -76,11 +76,10 @@ final class WorkspaceTests: XCTestCase { WorkspaceFolder(uri: DocumentURI(scratchDir.appendingPathComponent("PackageA"))), WorkspaceFolder(uri: DocumentURI(scratchDir.appendingPathComponent("PackageB"))), ] - } + }, + enableBackgroundIndexing: true ) - - try await SwiftPMTestProject.build(at: project.scratchDirectory.appendingPathComponent("PackageA")) - try await SwiftPMTestProject.build(at: project.scratchDirectory.appendingPathComponent("PackageB")) + _ = try await project.testClient.send(PollIndexRequest()) let (bUri, bPositions) = try project.openDocument("execB.swift") @@ -228,12 +227,14 @@ final class WorkspaceTests: XCTestCase { """, "PackageA/Package.swift": packageManifest, - ] + ], + enableBackgroundIndexing: true ) - try await SwiftPMTestProject.build(at: project.scratchDirectory.appendingPathComponent("PackageA")) let (uri, positions) = try project.openDocument("execA.swift") + _ = try await project.testClient.send(PollIndexRequest()) + let otherCompletions = try await project.testClient.send( CompletionRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"]) ) @@ -321,11 +322,11 @@ final class WorkspaceTests: XCTestCase { Lib().2️⃣foo() """, "Package.swift": packageManifest, - ] + ], + enableBackgroundIndexing: true ) - try await SwiftPMTestProject.build(at: project.scratchDirectory.appendingPathComponent("PackageA")) - try await SwiftPMTestProject.build(at: project.scratchDirectory) + _ = try await project.testClient.send(PollIndexRequest()) let (bUri, bPositions) = try project.openDocument("execB.swift") @@ -367,6 +368,8 @@ final class WorkspaceTests: XCTestCase { let (aUri, aPositions) = try project.openDocument("execA.swift") + _ = try await project.testClient.send(PollIndexRequest()) + let otherCompletions = try await project.testClient.send( CompletionRequest(textDocument: TextDocumentIdentifier(aUri), position: aPositions["1️⃣"]) ) @@ -820,10 +823,9 @@ final class WorkspaceTests: XCTestCase { ] ) """, - build: true + enableBackgroundIndexing: true ) let (mainUri, mainPositions) = try project.openDocument("main.swift") - _ = try await project.testClient.send(PollIndexRequest()) let fooDefinitionResponse = try await project.testClient.send( DefinitionRequest(textDocument: TextDocumentIdentifier(mainUri), position: mainPositions["3️⃣"]) From 1b914a7519a4cd9ef90c1e6093b2ae34396cf848 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 28 May 2024 10:04:58 -0700 Subject: [PATCH 05/69] Fix a memory leak in `BackgroundIndexingTests.testPrepareTargetAfterEditToDependency` The closure that handled the `DiagnosticsRefreshRequest` was retaining `project`, which constituted a retain cycle. --- Tests/SourceKitLSPTests/BackgroundIndexingTests.swift | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index c1ef5db3e..d0f53000e 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -604,13 +604,14 @@ final class BackgroundIndexingTests: XCTestCase { ) let receivedEmptyDiagnostics = self.expectation(description: "Received diagnostic refresh request") + receivedEmptyDiagnostics.assertForOverFulfill = false project.testClient.handleMultipleRequests { (_: CreateWorkDoneProgressRequest) in return VoidResponse() } - project.testClient.handleMultipleRequests { (_: DiagnosticsRefreshRequest) in - Task { - let updatedDiagnostics = try await project.testClient.send( + project.testClient.handleMultipleRequests { [weak project] (_: DiagnosticsRefreshRequest) in + Task { [weak project] in + let updatedDiagnostics = try await project?.testClient.send( DocumentDiagnosticsRequest(textDocument: TextDocumentIdentifier(uri)) ) guard case .full(let updatedDiagnostics) = updatedDiagnostics else { From e87d9e8d5e5a61175bc5a7b0da43050cad523656 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 25 May 2024 14:26:08 -0700 Subject: [PATCH 06/69] =?UTF-8?q?Show=20message=20if=20background=20indexi?= =?UTF-8?q?ng=20is=20enabled=20but=20the=20workspace=20doesn=E2=80=99t=20s?= =?UTF-8?q?upport=20background=20indexing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the user has enabled background indexing in sourcekit-lsp but opens a project that doesn’t support background indexing (compilation database, build server), we should show a message after opening the workspace, informing the user that background indexing is only supported in SwiftPM projects at the moment. Fixes #1255 rdar://127474711 --- Sources/SKCore/BuildServerBuildSystem.swift | 2 ++ Sources/SKCore/BuildSystem.swift | 4 ++++ Sources/SKCore/BuildSystemManager.swift | 4 ++++ .../CompilationDatabaseBuildSystem.swift | 2 ++ .../SwiftPMBuildSystem.swift | 1 + Sources/SourceKitLSP/SourceKitLSPServer.swift | 23 ++++++++++++++++++- .../Swift/SwiftLanguageService.swift | 2 +- Sources/SourceKitLSP/Workspace.swift | 5 +++- .../SKCoreTests/BuildSystemManagerTests.swift | 2 ++ .../BackgroundIndexingTests.swift | 11 +++++++++ .../SourceKitLSPTests/BuildSystemTests.swift | 2 ++ 11 files changed, 55 insertions(+), 3 deletions(-) diff --git a/Sources/SKCore/BuildServerBuildSystem.swift b/Sources/SKCore/BuildServerBuildSystem.swift index 2ad9af434..5f19066ab 100644 --- a/Sources/SKCore/BuildServerBuildSystem.swift +++ b/Sources/SKCore/BuildServerBuildSystem.swift @@ -259,6 +259,8 @@ private func readReponseDataKey(data: LSPAny?, key: String) -> String? { } extension BuildServerBuildSystem: BuildSystem { + public nonisolated var supportsPreparation: Bool { false } + /// The build settings for the given file. /// /// Returns `nil` if no build settings have been received from the build diff --git a/Sources/SKCore/BuildSystem.swift b/Sources/SKCore/BuildSystem.swift index 718a03bc7..204dbefc3 100644 --- a/Sources/SKCore/BuildSystem.swift +++ b/Sources/SKCore/BuildSystem.swift @@ -121,6 +121,10 @@ public protocol BuildSystem: AnyObject, Sendable { /// context. func setDelegate(_ delegate: BuildSystemDelegate?) async + /// Whether the build system is capable of preparing a target for indexing, ie. if the `prepare` methods has been + /// implemented. + var supportsPreparation: Bool { get } + /// Retrieve build settings for the given document with the given source /// language. /// diff --git a/Sources/SKCore/BuildSystemManager.swift b/Sources/SKCore/BuildSystemManager.swift index 6b3b8be91..134453ed8 100644 --- a/Sources/SKCore/BuildSystemManager.swift +++ b/Sources/SKCore/BuildSystemManager.swift @@ -68,6 +68,10 @@ public actor BuildSystemManager { } } + public var supportsPreparation: Bool { + return buildSystem?.supportsPreparation ?? false + } + /// Create a BuildSystemManager that wraps the given build system. The new /// manager will modify the delegate of the underlying build system. public init( diff --git a/Sources/SKCore/CompilationDatabaseBuildSystem.swift b/Sources/SKCore/CompilationDatabaseBuildSystem.swift index 3d84a6beb..d601bd099 100644 --- a/Sources/SKCore/CompilationDatabaseBuildSystem.swift +++ b/Sources/SKCore/CompilationDatabaseBuildSystem.swift @@ -93,6 +93,8 @@ public actor CompilationDatabaseBuildSystem { } extension CompilationDatabaseBuildSystem: BuildSystem { + public nonisolated var supportsPreparation: Bool { false } + public var indexDatabasePath: AbsolutePath? { indexStorePath?.parentDirectory.appending(component: "IndexDatabase") } diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index a23dd40aa..e7598b59e 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -342,6 +342,7 @@ extension SwiftPMBuildSystem { } extension SwiftPMBuildSystem: SKCore.BuildSystem { + public nonisolated var supportsPreparation: Bool { true } public var buildPath: TSCAbsolutePath { return TSCAbsolutePath(buildParameters.buildPath) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index d69b82796..dddeb4a02 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -99,6 +99,12 @@ public actor SourceKitLSPServer { /// Initialization can be awaited using `waitUntilInitialized`. private var initialized: Bool = false + /// Set to `true` after the user has opened a project that doesn't support background indexing while having background + /// indexing enabled. + /// + /// This ensures that we only inform the user about background indexing not being supported for these projects once. + private var didSendBackgroundIndexingNotSupportedNotification = false + var options: Options let toolchainRegistry: ToolchainRegistry @@ -916,7 +922,7 @@ extension SourceKitLSPServer { "Created workspace at \(workspaceFolder.uri.forLogging) as \(type(of: buildSystem)) with project root \(projectRoot ?? "")" ) - return try? await Workspace( + let workspace = try? await Workspace( documentManager: self.documentManager, rootUri: workspaceFolder.uri, capabilityRegistry: capabilityRegistry, @@ -935,6 +941,21 @@ extension SourceKitLSPServer { self?.indexProgressManager.indexProgressStatusDidChange() } ) + if let workspace, options.indexOptions.enableBackgroundIndexing, workspace.semanticIndexManager == nil, + !self.didSendBackgroundIndexingNotSupportedNotification + { + self.sendNotificationToClient( + ShowMessageNotification( + type: .info, + message: """ + Background indexing is currently only supported for SwiftPM projects. \ + For all other project types, please run a build to update the index. + """ + ) + ) + self.didSendBackgroundIndexingNotSupportedNotification = true + } + return workspace } func initialize(_ req: InitializeRequest) async throws -> InitializeResult { diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index 7d577d2d1..0175d3525 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -405,7 +405,7 @@ extension SwiftLanguageService { if buildSettings == nil || buildSettings!.isFallback, let fileUrl = note.textDocument.uri.fileURL { // Do not show this notification for non-file URIs to make sure we don't see this notificaiton for newly created // files (which get opened as with a `untitled:Unitled-1` URI by VS Code. - await sourceKitLSPServer?.sendNotificationToClient( + sourceKitLSPServer?.sendNotificationToClient( ShowMessageNotification( type: .warning, message: """ diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index 65dfeb50a..9d7e999d3 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -108,7 +108,10 @@ public final class Workspace: Sendable { mainFilesProvider: uncheckedIndex, toolchainRegistry: toolchainRegistry ) - if let uncheckedIndex, options.indexOptions.enableBackgroundIndexing { + if options.indexOptions.enableBackgroundIndexing, + let uncheckedIndex, + await buildSystemManager.supportsPreparation + { self.semanticIndexManager = SemanticIndexManager( index: uncheckedIndex, buildSystemManager: buildSystemManager, diff --git a/Tests/SKCoreTests/BuildSystemManagerTests.swift b/Tests/SKCoreTests/BuildSystemManagerTests.swift index 5eb4fad08..10583e945 100644 --- a/Tests/SKCoreTests/BuildSystemManagerTests.swift +++ b/Tests/SKCoreTests/BuildSystemManagerTests.swift @@ -455,6 +455,8 @@ class ManualBuildSystem: BuildSystem { self.delegate = delegate } + public nonisolated var supportsPreparation: Bool { false } + func buildSettings(for uri: DocumentURI, in buildTarget: ConfiguredTarget, language: Language) -> FileBuildSettings? { return map[uri] } diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index ce91e4e84..70db8f691 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -862,4 +862,15 @@ final class BackgroundIndexingTests: XCTestCase { "No file should exist at \(nestedIndexBuildURL)" ) } + + func testShowMessageWhenOpeningAProjectThatDoesntSupportBackgroundIndexing() async throws { + let project = try await MultiFileTestProject( + files: [ + "compile_commands.json": "" + ], + enableBackgroundIndexing: true + ) + let message = try await project.testClient.nextNotification(ofType: ShowMessageNotification.self) + XCTAssert(message.message.contains("Background indexing"), "Received unexpected message: \(message.message)") + } } diff --git a/Tests/SourceKitLSPTests/BuildSystemTests.swift b/Tests/SourceKitLSPTests/BuildSystemTests.swift index 70af0a498..eec906818 100644 --- a/Tests/SourceKitLSPTests/BuildSystemTests.swift +++ b/Tests/SourceKitLSPTests/BuildSystemTests.swift @@ -43,6 +43,8 @@ actor TestBuildSystem: BuildSystem { buildSettingsByFile[uri] = buildSettings } + public nonisolated var supportsPreparation: Bool { false } + func buildSettings( for document: DocumentURI, in buildTarget: ConfiguredTarget, From 9677a005c1a5c83251bf1ed153d017ef0c1f2c9c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 28 May 2024 14:35:15 -0700 Subject: [PATCH 07/69] Add paths from compiler arguments spelled with `=` to the diagnose bundle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eg. if the crash contained `-fmodule-map-file=/path/to/module.modulemap`, we weren’t including `/path/to/module.modulemap` in the diagnose bundle. --- Sources/Diagnose/ReproducerBundle.swift | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Sources/Diagnose/ReproducerBundle.swift b/Sources/Diagnose/ReproducerBundle.swift index 3095e6da6..c3f160214 100644 --- a/Sources/Diagnose/ReproducerBundle.swift +++ b/Sources/Diagnose/ReproducerBundle.swift @@ -47,14 +47,20 @@ func makeReproducerBundle(for requestInfo: RequestInfo, toolchain: Toolchain, bu ) } for compilerArg in requestInfo.compilerArgs { - // Copy all files from the compiler arguments into the reproducer bundle. - // Don't include files in Xcode (.app), Xcode toolchains or usr because they are most likely binary files that aren't user specific and would bloat the reproducer bundle. - if compilerArg.hasPrefix("/"), !compilerArg.contains(".app"), !compilerArg.contains(".xctoolchain"), - !compilerArg.contains("/usr/") - { - let dest = URL(fileURLWithPath: bundlePath.path + compilerArg) - try? FileManager.default.createDirectory(at: dest.deletingLastPathComponent(), withIntermediateDirectories: true) - try? FileManager.default.copyItem(at: URL(fileURLWithPath: compilerArg), to: dest) + // Find the first slash so we are also able to copy files from eg. + // `-fmodule-map-file=/path/to/module.modulemap` + // `-I/foo/bar` + guard let firstSlash = compilerArg.firstIndex(of: "/") else { + continue } + let path = compilerArg[firstSlash...] + guard !path.contains(".app"), !path.contains(".xctoolchain"), !path.contains("/usr/") else { + // Don't include files in Xcode (.app), Xcode toolchains or usr because they are most likely binary files that + // aren't user specific and would bloat the reproducer bundle. + continue + } + let dest = URL(fileURLWithPath: bundlePath.path + path) + try? FileManager.default.createDirectory(at: dest.deletingLastPathComponent(), withIntermediateDirectories: true) + try? FileManager.default.copyItem(at: URL(fileURLWithPath: String(path)), to: dest) } } From 087ea64ed321b0cdea3ba450aa6a4ce97ab888ea Mon Sep 17 00:00:00 2001 From: "Lokesh.T.R" Date: Sat, 25 May 2024 15:07:38 +0000 Subject: [PATCH 08/69] Rename `note` to `notification` throughout the codebase wherever necessary --- .../TestJSONRPCConnection.swift | 8 ++-- .../Clang/ClangLanguageService.swift | 32 ++++++------- Sources/SourceKitLSP/DocumentManager.swift | 16 +++---- Sources/SourceKitLSP/LanguageService.swift | 10 ++-- Sources/SourceKitLSP/SourceKitLSPServer.swift | 20 ++++---- .../Swift/SwiftLanguageService.swift | 38 +++++++-------- .../ConnectionTests.swift | 48 +++++++++---------- .../ConnectionTests.swift | 8 ++-- Tests/SourceKitLSPTests/LocalSwiftTests.swift | 6 +-- 9 files changed, 93 insertions(+), 93 deletions(-) diff --git a/Sources/LSPTestSupport/TestJSONRPCConnection.swift b/Sources/LSPTestSupport/TestJSONRPCConnection.swift index 992ec51fe..d175f4afe 100644 --- a/Sources/LSPTestSupport/TestJSONRPCConnection.swift +++ b/Sources/LSPTestSupport/TestJSONRPCConnection.swift @@ -111,11 +111,11 @@ public actor TestClient: MessageHandler { } public func appendOneShotNotificationHandler(_ handler: @escaping (N) -> Void) { - oneShotNotificationHandlers.append({ anyNote in - guard let note = anyNote as? N else { - fatalError("received notification of the wrong type \(anyNote); expected \(N.self)") + oneShotNotificationHandlers.append({ anyNotification in + guard let notification = anyNotification as? N else { + fatalError("received notification of the wrong type \(anyNotification); expected \(N.self)") } - handler(note) + handler(notification) }) } diff --git a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift index 5c1822c21..965d7b01e 100644 --- a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift +++ b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift @@ -464,30 +464,30 @@ extension ClangLanguageService { // MARK: - Text synchronization - public func openDocument(_ note: DidOpenTextDocumentNotification) async { - openDocuments[note.textDocument.uri] = note.textDocument.language + public func openDocument(_ notification: DidOpenTextDocumentNotification) async { + openDocuments[notification.textDocument.uri] = notification.textDocument.language // Send clangd the build settings for the new file. We need to do this before // sending the open notification, so that the initial diagnostics already // have build settings. - await documentUpdatedBuildSettings(note.textDocument.uri) - clangd.send(note) + await documentUpdatedBuildSettings(notification.textDocument.uri) + clangd.send(notification) } - public func closeDocument(_ note: DidCloseTextDocumentNotification) { - openDocuments[note.textDocument.uri] = nil - clangd.send(note) + public func closeDocument(_ notification: DidCloseTextDocumentNotification) { + openDocuments[notification.textDocument.uri] = nil + clangd.send(notification) } - public func changeDocument(_ note: DidChangeTextDocumentNotification) { - clangd.send(note) + public func changeDocument(_ notification: DidChangeTextDocumentNotification) { + clangd.send(notification) } - public func willSaveDocument(_ note: WillSaveTextDocumentNotification) { + public func willSaveDocument(_ notification: WillSaveTextDocumentNotification) { } - public func didSaveDocument(_ note: DidSaveTextDocumentNotification) { - clangd.send(note) + public func didSaveDocument(_ notification: DidSaveTextDocumentNotification) { + clangd.send(notification) } // MARK: - Build System Integration @@ -505,13 +505,13 @@ extension ClangLanguageService { if let compileCommand = clangBuildSettings?.compileCommand, let pathString = (try? AbsolutePath(validating: url.path))?.pathString { - let note = DidChangeConfigurationNotification( + let notification = DidChangeConfigurationNotification( settings: .clangd( ClangWorkspaceSettings( compilationDatabaseChanges: [pathString: compileCommand]) ) ) - clangd.send(note) + clangd.send(notification) } } @@ -519,12 +519,12 @@ extension ClangLanguageService { // In order to tell clangd to reload an AST, we send it an empty `didChangeTextDocument` // with `forceRebuild` set in case any missing header files have been added. // This works well for us as the moment since clangd ignores the document version. - let note = DidChangeTextDocumentNotification( + let notification = DidChangeTextDocumentNotification( textDocument: VersionedTextDocumentIdentifier(uri, version: 0), contentChanges: [], forceRebuild: true ) - clangd.send(note) + clangd.send(notification) } // MARK: - Text Document diff --git a/Sources/SourceKitLSP/DocumentManager.swift b/Sources/SourceKitLSP/DocumentManager.swift index 243b4eb4a..6ade8e8b1 100644 --- a/Sources/SourceKitLSP/DocumentManager.swift +++ b/Sources/SourceKitLSP/DocumentManager.swift @@ -210,17 +210,17 @@ extension DocumentManager { /// Convenience wrapper for `open(_:language:version:text:)` that logs on failure. @discardableResult - func open(_ note: DidOpenTextDocumentNotification) -> DocumentSnapshot? { - let doc = note.textDocument + func open(_ notification: DidOpenTextDocumentNotification) -> DocumentSnapshot? { + let doc = notification.textDocument return orLog("failed to open document", level: .error) { try open(doc.uri, language: doc.language, version: doc.version, text: doc.text) } } /// Convenience wrapper for `close(_:)` that logs on failure. - func close(_ note: DidCloseTextDocumentNotification) { + func close(_ notification: DidCloseTextDocumentNotification) { orLog("failed to close document", level: .error) { - try close(note.textDocument.uri) + try close(notification.textDocument.uri) } } @@ -228,13 +228,13 @@ extension DocumentManager { /// that logs on failure. @discardableResult func edit( - _ note: DidChangeTextDocumentNotification + _ notification: DidChangeTextDocumentNotification ) -> (preEditSnapshot: DocumentSnapshot, postEditSnapshot: DocumentSnapshot, edits: [SourceEdit])? { return orLog("failed to edit document", level: .error) { return try edit( - note.textDocument.uri, - newVersion: note.textDocument.version, - edits: note.contentChanges + notification.textDocument.uri, + newVersion: notification.textDocument.version, + edits: notification.contentChanges ) } } diff --git a/Sources/SourceKitLSP/LanguageService.swift b/Sources/SourceKitLSP/LanguageService.swift index 517ce98d9..856c929fa 100644 --- a/Sources/SourceKitLSP/LanguageService.swift +++ b/Sources/SourceKitLSP/LanguageService.swift @@ -98,13 +98,13 @@ public protocol LanguageService: AnyObject, Sendable { /// Sent to open up a document on the Language Server. /// This may be called before or after a corresponding /// `documentUpdatedBuildSettings` call for the same document. - func openDocument(_ note: DidOpenTextDocumentNotification) async + func openDocument(_ notification: DidOpenTextDocumentNotification) async /// Sent to close a document on the Language Server. - func closeDocument(_ note: DidCloseTextDocumentNotification) async - func changeDocument(_ note: DidChangeTextDocumentNotification) async - func willSaveDocument(_ note: WillSaveTextDocumentNotification) async - func didSaveDocument(_ note: DidSaveTextDocumentNotification) async + func closeDocument(_ notification: DidCloseTextDocumentNotification) async + func changeDocument(_ notification: DidChangeTextDocumentNotification) async + func willSaveDocument(_ notification: WillSaveTextDocumentNotification) async + func didSaveDocument(_ notification: DidSaveTextDocumentNotification) async // MARK: - Build System Integration diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index 071c8302e..51132453c 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -1269,12 +1269,12 @@ extension SourceKitLSPServer { await openDocument(notification, workspace: workspace) } - private func openDocument(_ note: DidOpenTextDocumentNotification, workspace: Workspace) async { + private func openDocument(_ notification: DidOpenTextDocumentNotification, workspace: Workspace) async { // Immediately open the document even if the build system isn't ready. This is important since // we check that the document is open when we receive messages from the build system. - documentManager.open(note) + documentManager.open(notification) - let textDocument = note.textDocument + let textDocument = notification.textDocument let uri = textDocument.uri let language = textDocument.language @@ -1286,7 +1286,7 @@ extension SourceKitLSPServer { await workspace.buildSystemManager.registerForChangeNotifications(for: uri, language: language) // If the document is ready, we can immediately send the notification. - await service.openDocument(note) + await service.openDocument(notification) } func closeDocument(_ notification: DidCloseTextDocumentNotification) async { @@ -1300,16 +1300,16 @@ extension SourceKitLSPServer { await self.closeDocument(notification, workspace: workspace) } - func closeDocument(_ note: DidCloseTextDocumentNotification, workspace: Workspace) async { + func closeDocument(_ notification: DidCloseTextDocumentNotification, workspace: Workspace) async { // Immediately close the document. We need to be sure to clear our pending work queue in case // the build system still isn't ready. - documentManager.close(note) + documentManager.close(notification) - let uri = note.textDocument.uri + let uri = notification.textDocument.uri await workspace.buildSystemManager.unregisterForChangeNotifications(for: uri) - await workspace.documentService.value[uri]?.closeDocument(note) + await workspace.documentService.value[uri]?.closeDocument(notification) } func changeDocument(_ notification: DidChangeTextDocumentNotification) async { @@ -1336,10 +1336,10 @@ extension SourceKitLSPServer { } func didSaveDocument( - _ note: DidSaveTextDocumentNotification, + _ notification: DidSaveTextDocumentNotification, languageService: LanguageService ) async { - await languageService.didSaveDocument(note) + await languageService.didSaveDocument(notification) } func didChangeWorkspaceFolders(_ notification: DidChangeWorkspaceFoldersNotification) async { diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index 0175d3525..48d88e098 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -392,17 +392,17 @@ extension SwiftLanguageService { ]) } - public func openDocument(_ note: DidOpenTextDocumentNotification) async { - cancelInFlightPublishDiagnosticsTask(for: note.textDocument.uri) - await diagnosticReportManager.removeItemsFromCache(with: note.textDocument.uri) + public func openDocument(_ notification: DidOpenTextDocumentNotification) async { + cancelInFlightPublishDiagnosticsTask(for: notification.textDocument.uri) + await diagnosticReportManager.removeItemsFromCache(with: notification.textDocument.uri) - guard let snapshot = self.documentManager.open(note) else { + guard let snapshot = self.documentManager.open(notification) else { // Already logged failure. return } let buildSettings = await self.buildSettings(for: snapshot.uri) - if buildSettings == nil || buildSettings!.isFallback, let fileUrl = note.textDocument.uri.fileURL { + if buildSettings == nil || buildSettings!.isFallback, let fileUrl = notification.textDocument.uri.fileURL { // Do not show this notification for non-file URIs to make sure we don't see this notificaiton for newly created // files (which get opened as with a `untitled:Unitled-1` URI by VS Code. sourceKitLSPServer?.sendNotificationToClient( @@ -419,17 +419,17 @@ extension SwiftLanguageService { let req = openDocumentSourcekitdRequest(snapshot: snapshot, compileCommand: buildSettings) _ = try? await self.sourcekitd.send(req, fileContents: snapshot.text) - await publishDiagnosticsIfNeeded(for: note.textDocument.uri) + await publishDiagnosticsIfNeeded(for: notification.textDocument.uri) } - public func closeDocument(_ note: DidCloseTextDocumentNotification) async { - cancelInFlightPublishDiagnosticsTask(for: note.textDocument.uri) - inFlightPublishDiagnosticsTasks[note.textDocument.uri] = nil - await diagnosticReportManager.removeItemsFromCache(with: note.textDocument.uri) + public func closeDocument(_ notification: DidCloseTextDocumentNotification) async { + cancelInFlightPublishDiagnosticsTask(for: notification.textDocument.uri) + inFlightPublishDiagnosticsTasks[notification.textDocument.uri] = nil + await diagnosticReportManager.removeItemsFromCache(with: notification.textDocument.uri) - self.documentManager.close(note) + self.documentManager.close(notification) - let req = closeDocumentSourcekitdRequest(uri: note.textDocument.uri) + let req = closeDocumentSourcekitdRequest(uri: notification.textDocument.uri) _ = try? await self.sourcekitd.send(req, fileContents: nil) } @@ -511,8 +511,8 @@ extension SwiftLanguageService { } } - public func changeDocument(_ note: DidChangeTextDocumentNotification) async { - cancelInFlightPublishDiagnosticsTask(for: note.textDocument.uri) + public func changeDocument(_ notification: DidChangeTextDocumentNotification) async { + cancelInFlightPublishDiagnosticsTask(for: notification.textDocument.uri) let keys = self.keys struct Edit { @@ -521,14 +521,14 @@ extension SwiftLanguageService { let replacement: String } - guard let (preEditSnapshot, postEditSnapshot, edits) = self.documentManager.edit(note) else { + guard let (preEditSnapshot, postEditSnapshot, edits) = self.documentManager.edit(notification) else { return } for edit in edits { let req = sourcekitd.dictionary([ keys.request: self.requests.editorReplaceText, - keys.name: note.textDocument.uri.pseudoPath, + keys.name: notification.textDocument.uri.pseudoPath, keys.enableSyntaxMap: 0, keys.enableStructure: 0, keys.enableDiagnostics: 0, @@ -558,14 +558,14 @@ extension SwiftLanguageService { edits: concurrentEdits ) - await publishDiagnosticsIfNeeded(for: note.textDocument.uri) + await publishDiagnosticsIfNeeded(for: notification.textDocument.uri) } - public func willSaveDocument(_ note: WillSaveTextDocumentNotification) { + public func willSaveDocument(_ notification: WillSaveTextDocumentNotification) { } - public func didSaveDocument(_ note: DidSaveTextDocumentNotification) { + public func didSaveDocument(_ notification: DidSaveTextDocumentNotification) { } diff --git a/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift b/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift index 3f4838974..285950b7c 100644 --- a/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift +++ b/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift @@ -54,37 +54,37 @@ class ConnectionTests: XCTestCase { func testMessageBuffer() async throws { let client = connection.client let clientConnection = connection.clientToServerConnection - let expectation = self.expectation(description: "note received") + let expectation = self.expectation(description: "notfication received") - await client.appendOneShotNotificationHandler { (note: EchoNotification) in - XCTAssertEqual(note.string, "hello!") + await client.appendOneShotNotificationHandler { (notification: EchoNotification) in + XCTAssertEqual(notification.string, "hello!") expectation.fulfill() } - let note1 = try JSONEncoder().encode(JSONRPCMessage.notification(EchoNotification(string: "hello!"))) - let note2 = try JSONEncoder().encode(JSONRPCMessage.notification(EchoNotification(string: "no way!"))) + let notification1 = try JSONEncoder().encode(JSONRPCMessage.notification(EchoNotification(string: "hello!"))) + let notification2 = try JSONEncoder().encode(JSONRPCMessage.notification(EchoNotification(string: "no way!"))) - let note1Str: String = "Content-Length: \(note1.count)\r\n\r\n\(String(data: note1, encoding: .utf8)!)" - let note2Str: String = "Content-Length: \(note2.count)\r\n\r\n\(String(data: note2, encoding: .utf8)!)" + let notification1Str: String = "Content-Length: \(notification1.count)\r\n\r\n\(String(data: notification1, encoding: .utf8)!)" + let notfication2Str: String = "Content-Length: \(notification2.count)\r\n\r\n\(String(data: notification2, encoding: .utf8)!)" - for b in note1Str.utf8.dropLast() { + for b in notification1Str.utf8.dropLast() { clientConnection.send(_rawData: [b].withUnsafeBytes { DispatchData(bytes: $0) }) } clientConnection.send( - _rawData: [note1Str.utf8.last!, note2Str.utf8.first!].withUnsafeBytes { DispatchData(bytes: $0) } + _rawData: [notification1Str.utf8.last!, notfication2Str.utf8.first!].withUnsafeBytes { DispatchData(bytes: $0) } ) try await fulfillmentOfOrThrow([expectation]) - let expectation2 = self.expectation(description: "note received") + let expectation2 = self.expectation(description: "notification received") - await client.appendOneShotNotificationHandler { (note: EchoNotification) in - XCTAssertEqual(note.string, "no way!") + await client.appendOneShotNotificationHandler { (notification: EchoNotification) in + XCTAssertEqual(notification.string, "no way!") expectation2.fulfill() } - for b in note2Str.utf8.dropFirst() { + for b in notfication2Str.utf8.dropFirst() { clientConnection.send(_rawData: [b].withUnsafeBytes { DispatchData(bytes: $0) }) } @@ -117,12 +117,12 @@ class ConnectionTests: XCTestCase { try await fulfillmentOfOrThrow([expectation, expectation2]) } - func testEchoNote() async throws { + func testEchoNotification() async throws { let client = connection.client - let expectation = self.expectation(description: "note received") + let expectation = self.expectation(description: "notification received") - await client.appendOneShotNotificationHandler { (note: EchoNotification) in - XCTAssertEqual(note.string, "hello!") + await client.appendOneShotNotificationHandler { (notification: EchoNotification) in + XCTAssertEqual(notification.string, "hello!") expectation.fulfill() } @@ -150,13 +150,13 @@ class ConnectionTests: XCTestCase { func testUnknownNotification() async throws { let client = connection.client - let expectation = self.expectation(description: "note received") + let expectation = self.expectation(description: "notification received") - struct UnknownNote: NotificationType { + struct UnknownNotification: NotificationType { static let method: String = "unknown" } - client.send(UnknownNote()) + client.send(UnknownNotification()) // Nothing bad should happen; check that the next request works. @@ -191,7 +191,7 @@ class ConnectionTests: XCTestCase { func testSendAfterClose() async throws { let client = connection.client - let expectation = self.expectation(description: "note received") + let expectation = self.expectation(description: "notification received") connection.clientToServerConnection.close() @@ -214,7 +214,7 @@ class ConnectionTests: XCTestCase { let server = connection.server let expectation = self.expectation(description: "received notification") - await client.appendOneShotNotificationHandler { (note: EchoNotification) in + await client.appendOneShotNotificationHandler { (notification: EchoNotification) in expectation.fulfill() } @@ -281,8 +281,8 @@ class ConnectionTests: XCTestCase { func testMessageWithMissingParameter() async throws { let expectation = self.expectation(description: "Received ShowMessageNotification") - await connection.client.appendOneShotNotificationHandler { (note: ShowMessageNotification) in - XCTAssertEqual(note.type, .error) + await connection.client.appendOneShotNotificationHandler { (notification: ShowMessageNotification) in + XCTAssertEqual(notification.type, .error) expectation.fulfill() } diff --git a/Tests/LanguageServerProtocolTests/ConnectionTests.swift b/Tests/LanguageServerProtocolTests/ConnectionTests.swift index 73e5c447a..5a6c703e5 100644 --- a/Tests/LanguageServerProtocolTests/ConnectionTests.swift +++ b/Tests/LanguageServerProtocolTests/ConnectionTests.swift @@ -60,12 +60,12 @@ class ConnectionTests: XCTestCase { try await fulfillmentOfOrThrow([expectation, expectation2]) } - func testEchoNote() async throws { + func testEchoNotification() async throws { let client = connection.client - let expectation = self.expectation(description: "note received") + let expectation = self.expectation(description: "notification received") - await client.appendOneShotNotificationHandler { (note: EchoNotification) in - XCTAssertEqual(note.string, "hello!") + await client.appendOneShotNotificationHandler { (notification: EchoNotification) in + XCTAssertEqual(notification.string, "hello!") expectation.fulfill() } diff --git a/Tests/SourceKitLSPTests/LocalSwiftTests.swift b/Tests/SourceKitLSPTests/LocalSwiftTests.swift index 58d2fcb3f..54e0bcda9 100644 --- a/Tests/SourceKitLSPTests/LocalSwiftTests.swift +++ b/Tests/SourceKitLSPTests/LocalSwiftTests.swift @@ -419,7 +419,7 @@ final class LocalSwiftTests: XCTestCase { ) } - func testFixitsAreIncludedInPublishDiagnosticsNotes() async throws { + func testFixitsAreIncludedInPublishDiagnosticsNotifications() async throws { let testClient = try await TestSourceKitLSPClient(usePullDiagnostics: false) let url = URL(fileURLWithPath: "/\(UUID())/a.swift") let uri = DocumentURI(url) @@ -585,7 +585,7 @@ final class LocalSwiftTests: XCTestCase { ) } - func testFixitsAreReturnedFromCodeActionsNotes() async throws { + func testFixitsAreReturnedFromCodeActionsNotifications() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: quickFixCapabilities, usePullDiagnostics: false) let url = URL(fileURLWithPath: "/\(UUID())/a.swift") let uri = DocumentURI(url) @@ -691,7 +691,7 @@ final class LocalSwiftTests: XCTestCase { ) } - func testMuliEditFixitCodeActionNote() async throws { + func testMuliEditFixitCodeActionNotifications() async throws { let testClient = try await TestSourceKitLSPClient(capabilities: quickFixCapabilities, usePullDiagnostics: false) let url = URL(fileURLWithPath: "/\(UUID())/a.swift") let uri = DocumentURI(url) From 1a45220bddf37a66b8976a6ecf9ff6593644ab18 Mon Sep 17 00:00:00 2001 From: Lokesh T R Date: Wed, 29 May 2024 18:00:36 +0000 Subject: [PATCH 09/69] Fixed minor formatting issues --- .../ConnectionTests.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift b/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift index 285950b7c..daceb8b9f 100644 --- a/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift +++ b/Tests/LanguageServerProtocolJSONRPCTests/ConnectionTests.swift @@ -64,8 +64,10 @@ class ConnectionTests: XCTestCase { let notification1 = try JSONEncoder().encode(JSONRPCMessage.notification(EchoNotification(string: "hello!"))) let notification2 = try JSONEncoder().encode(JSONRPCMessage.notification(EchoNotification(string: "no way!"))) - let notification1Str: String = "Content-Length: \(notification1.count)\r\n\r\n\(String(data: notification1, encoding: .utf8)!)" - let notfication2Str: String = "Content-Length: \(notification2.count)\r\n\r\n\(String(data: notification2, encoding: .utf8)!)" + let notification1Str = + "Content-Length: \(notification1.count)\r\n\r\n\(String(data: notification1, encoding: .utf8)!)" + let notfication2Str = + "Content-Length: \(notification2.count)\r\n\r\n\(String(data: notification2, encoding: .utf8)!)" for b in notification1Str.utf8.dropLast() { clientConnection.send(_rawData: [b].withUnsafeBytes { DispatchData(bytes: $0) }) From 129d3a4f938b62b1ceb8ded06c25e7baff4ae74d Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 May 2024 13:59:20 -0700 Subject: [PATCH 10/69] Move Swift compiler version parsing from `SkipUnless.swift` to `Toolchain` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to inspect the Swift version and fix up compiler arguments from SwiftPM to account for an older version of SwiftPM being invoked using `swift build` than the one that’s bundled in-process with SourceKit-LSP. --- Sources/SKCore/Toolchain.swift | 79 ++++++++++++++++++++++++++ Sources/SKTestSupport/SkipUnless.swift | 63 +------------------- 2 files changed, 80 insertions(+), 62 deletions(-) diff --git a/Sources/SKCore/Toolchain.swift b/Sources/SKCore/Toolchain.swift index fcb369620..8b5edb612 100644 --- a/Sources/SKCore/Toolchain.swift +++ b/Sources/SKCore/Toolchain.swift @@ -12,13 +12,51 @@ import LSPLogging import LanguageServerProtocol +import RegexBuilder import SKSupport import enum PackageLoading.Platform import struct TSCBasic.AbsolutePath import protocol TSCBasic.FileSystem +import class TSCBasic.Process import var TSCBasic.localFileSystem +/// A Swift version consisting of the major and minor component. +public struct SwiftVersion: Sendable, Comparable, CustomStringConvertible { + public let major: Int + public let minor: Int + + public static func < (lhs: SwiftVersion, rhs: SwiftVersion) -> Bool { + return (lhs.major, lhs.minor) < (rhs.major, rhs.minor) + } + + public init(_ major: Int, _ minor: Int) { + self.major = major + self.minor = minor + } + + public var description: String { + return "\(major).\(minor)" + } +} + +fileprivate enum SwiftVersionParsingError: Error, CustomStringConvertible { + case failedToFindSwiftc + case failedToParseOutput(output: String?) + + var description: String { + switch self { + case .failedToFindSwiftc: + return "Default toolchain does not contain a swiftc executable" + case .failedToParseOutput(let output): + return """ + Failed to parse Swift version. Output of swift --version: + \(output ?? "") + """ + } + } +} + /// A Toolchain is a collection of related compilers and libraries meant to be used together to /// build and edit source code. /// @@ -63,6 +101,47 @@ public final class Toolchain: Sendable { /// The path to the indexstore library if available. public let libIndexStore: AbsolutePath? + private let swiftVersionTask = ThreadSafeBox?>(initialValue: nil) + + /// The Swift version installed in the toolchain. Throws an error if the version could not be parsed or if no Swift + /// compiler is installed in the toolchain. + public var swiftVersion: SwiftVersion { + get async throws { + let task = swiftVersionTask.withLock { task in + if let task { + return task + } + let newTask = Task { () -> SwiftVersion in + guard let swiftc else { + throw SwiftVersionParsingError.failedToFindSwiftc + } + + let process = Process(args: swiftc.pathString, "--version") + try process.launch() + let result = try await process.waitUntilExit() + let output = String(bytes: try result.output.get(), encoding: .utf8) + let regex = Regex { + "Swift version " + Capture { OneOrMore(.digit) } + "." + Capture { OneOrMore(.digit) } + } + guard let match = output?.firstMatch(of: regex) else { + throw SwiftVersionParsingError.failedToParseOutput(output: output) + } + guard let major = Int(match.1), let minor = Int(match.2) else { + throw SwiftVersionParsingError.failedToParseOutput(output: output) + } + return SwiftVersion(major, minor) + } + task = newTask + return newTask + } + + return try await task.value + } + } + public init( identifier: String, displayName: String, diff --git a/Sources/SKTestSupport/SkipUnless.swift b/Sources/SKTestSupport/SkipUnless.swift index 3753ad5ee..b4595c057 100644 --- a/Sources/SKTestSupport/SkipUnless.swift +++ b/Sources/SKTestSupport/SkipUnless.swift @@ -65,11 +65,7 @@ public actor SkipUnless { // Never skip tests in CI. Toolchain should be up-to-date checkResult = .featureSupported } else { - guard let swiftc = await ToolchainRegistry.forTesting.default?.swiftc else { - throw SwiftVersionParsingError.failedToFindSwiftc - } - - let toolchainSwiftVersion = try await getSwiftVersion(swiftc) + let toolchainSwiftVersion = try await unwrap(ToolchainRegistry.forTesting.default).swiftVersion let requiredSwiftVersion = SwiftVersion(swiftVersion.major, swiftVersion.minor) if toolchainSwiftVersion < requiredSwiftVersion { checkResult = .featureUnsupported( @@ -297,60 +293,3 @@ fileprivate extension String { } } } - -/// A Swift version consisting of the major and minor component. -fileprivate struct SwiftVersion: Comparable, CustomStringConvertible { - let major: Int - let minor: Int - - static func < (lhs: SwiftVersion, rhs: SwiftVersion) -> Bool { - return (lhs.major, lhs.minor) < (rhs.major, rhs.minor) - } - - init(_ major: Int, _ minor: Int) { - self.major = major - self.minor = minor - } - - var description: String { - return "\(major).\(minor)" - } -} - -fileprivate enum SwiftVersionParsingError: Error, CustomStringConvertible { - case failedToFindSwiftc - case failedToParseOutput(output: String?) - - var description: String { - switch self { - case .failedToFindSwiftc: - return "Default toolchain does not contain a swiftc executable" - case .failedToParseOutput(let output): - return """ - Failed to parse Swift version. Output of swift --version: - \(output ?? "") - """ - } - } -} - -/// Return the major and minor version of Swift for a `swiftc` compiler at `swiftcPath`. -private func getSwiftVersion(_ swiftcPath: AbsolutePath) async throws -> SwiftVersion { - let process = Process(args: swiftcPath.pathString, "--version") - try process.launch() - let result = try await process.waitUntilExit() - let output = String(bytes: try result.output.get(), encoding: .utf8) - let regex = Regex { - "Swift version " - Capture { OneOrMore(.digit) } - "." - Capture { OneOrMore(.digit) } - } - guard let match = output?.firstMatch(of: regex) else { - throw SwiftVersionParsingError.failedToParseOutput(output: output) - } - guard let major = Int(match.1), let minor = Int(match.2) else { - throw SwiftVersionParsingError.failedToParseOutput(output: output) - } - return SwiftVersion(major, minor) -} From 48df05478ca44415d210d6258220e649fdf685a8 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 May 2024 14:18:24 -0700 Subject: [PATCH 11/69] Support running tests that require building with a Swift 5.10 toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we detect that we’re running using a 5.10 toolchain, adjust the compiler arguments that we received from SwiftPM to drop any `/Modules` suffixes in the build directory, to account for the fact that 5.10 stores the modules one directory higher up (https://github.com/apple/swift-package-manager/pull/7103). --- .../SwiftPMBuildSystem.swift | 25 ++++++++++++++++--- Sources/SKTestSupport/SkipUnless.swift | 4 --- .../SwiftPMBuildSystemTests.swift | 5 ++-- .../BackgroundIndexingTests.swift | 6 ----- Tests/SourceKitLSPTests/DefinitionTests.swift | 1 - .../DependencyTrackingTests.swift | 1 - Tests/SourceKitLSPTests/IndexTests.swift | 1 - .../PublishDiagnosticsTests.swift | 1 - .../PullDiagnosticsTests.swift | 3 --- .../SwiftInterfaceTests.swift | 2 -- Tests/SourceKitLSPTests/WorkspaceTests.swift | 10 -------- 11 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index a23dd40aa..0c410ea15 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -357,11 +357,30 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { public var indexPrefixMappings: [PathPrefixMapping] { return [] } + /// Return the compiler arguments for the given source file within a target, making any necessary adjustments to + /// account for differences in the SwiftPM versions being linked into SwiftPM and being installed in the toolchain. + private func compilerArguments(for file: URL, in buildTarget: any SwiftBuildTarget) async throws -> [String] { + let compileArguments = try buildTarget.compileArguments(for: file) + + // Fix up compiler arguments that point to a `/Modules` subdirectory if the Swift version in the toolchain is less + // than 6.0 because it places the modules one level higher up. + let toolchainVersion = await orLog("Getting Swift version") { try await toolchainRegistry.default?.swiftVersion } + guard let toolchainVersion, toolchainVersion < SwiftVersion(6, 0) else { + return compileArguments + } + return compileArguments.map { argument in + if argument.hasSuffix("/Modules"), argument.contains(self.workspace.location.scratchDirectory.pathString) { + return String(argument.dropLast(8)) + } + return argument + } + } + public func buildSettings( for uri: DocumentURI, in configuredTarget: ConfiguredTarget, language: Language - ) throws -> FileBuildSettings? { + ) async throws -> FileBuildSettings? { guard let url = uri.fileURL, let path = try? AbsolutePath(validating: url.path) else { // We can't determine build settings for non-file URIs. return nil @@ -387,13 +406,13 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { // getting its compiler arguments and then patching up the compiler arguments by replacing the substitute file // with the `.cpp` file. return FileBuildSettings( - compilerArguments: try buildTarget.compileArguments(for: substituteFile), + compilerArguments: try await compilerArguments(for: substituteFile, in: buildTarget), workingDirectory: workspacePath.pathString ).patching(newFile: try resolveSymlinks(path).pathString, originalFile: substituteFile.absoluteString) } return FileBuildSettings( - compilerArguments: try buildTarget.compileArguments(for: url), + compilerArguments: try await compilerArguments(for: url, in: buildTarget), workingDirectory: workspacePath.pathString ) } diff --git a/Sources/SKTestSupport/SkipUnless.swift b/Sources/SKTestSupport/SkipUnless.swift index b4595c057..fca5f0267 100644 --- a/Sources/SKTestSupport/SkipUnless.swift +++ b/Sources/SKTestSupport/SkipUnless.swift @@ -170,10 +170,6 @@ public actor SkipUnless { /// SwiftPM moved the location where it stores Swift modules to a subdirectory in /// https://github.com/apple/swift-package-manager/pull/7103. - /// - /// sourcekit-lsp uses the built-in SwiftPM to synthesize compiler arguments and cross-module tests fail if the host - /// toolchain’s SwiftPM stores the Swift modules on the top level but we synthesize compiler arguments expecting the - /// modules to be in a `Modules` subdirectory. public static func swiftpmStoresModulesInSubdirectory( file: StaticString = #filePath, line: UInt = #line diff --git a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift index 92ec238dd..7302b187e 100644 --- a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift +++ b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift @@ -28,11 +28,11 @@ import struct PackageModel.BuildFlags #endif fileprivate extension SwiftPMBuildSystem { - func buildSettings(for uri: DocumentURI, language: Language) throws -> FileBuildSettings? { + func buildSettings(for uri: DocumentURI, language: Language) async throws -> FileBuildSettings? { guard let target = self.configuredTargets(for: uri).only else { return nil } - return try buildSettings(for: uri, in: target, language: language) + return try await buildSettings(for: uri, in: target, language: language) } } @@ -116,6 +116,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { } func testBasicSwiftArgs() async throws { + try await SkipUnless.swiftpmStoresModulesInSubdirectory() let fs = localFileSystem try await withTestScratchDir { tempDir in try fs.createFiles( diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index ce91e4e84..79235e7ab 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -104,7 +104,6 @@ final class BackgroundIndexingTests: XCTestCase { } func testBackgroundIndexingOfMultiModuleProject() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ "LibA/MyFile.swift": """ @@ -212,7 +211,6 @@ final class BackgroundIndexingTests: XCTestCase { } func testBackgroundIndexingOfPackageDependency() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let dependencyContents = """ public func 1️⃣doSomething() {} """ @@ -541,7 +539,6 @@ final class BackgroundIndexingTests: XCTestCase { } func testPrepareTargetAfterEditToDependency() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() var serverOptions = SourceKitLSPServer.Options.testDefault let expectedPreparationTracker = ExpectedIndexTaskTracker(expectedPreparations: [ [ @@ -651,7 +648,6 @@ final class BackgroundIndexingTests: XCTestCase { let libBStartedPreparation = self.expectation(description: "LibB started preparing") let libDPreparedForEditing = self.expectation(description: "LibD prepared for editing") - try await SkipUnless.swiftpmStoresModulesInSubdirectory() var serverOptions = SourceKitLSPServer.Options.testDefault let expectedPreparationTracker = ExpectedIndexTaskTracker(expectedPreparations: [ // Preparation of targets during the initial of the target @@ -749,8 +745,6 @@ final class BackgroundIndexingTests: XCTestCase { } func testIndexingHappensInParallel() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() - let fileAIndexingStarted = self.expectation(description: "FileA indexing started") let fileBIndexingStarted = self.expectation(description: "FileB indexing started") diff --git a/Tests/SourceKitLSPTests/DefinitionTests.swift b/Tests/SourceKitLSPTests/DefinitionTests.swift index cdaa34739..26f396ba4 100644 --- a/Tests/SourceKitLSPTests/DefinitionTests.swift +++ b/Tests/SourceKitLSPTests/DefinitionTests.swift @@ -440,7 +440,6 @@ class DefinitionTests: XCTestCase { func testDependentModuleGotBuilt() async throws { try SkipUnless.longTestsEnabled() - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ "LibA/LibA.swift": """ diff --git a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift index 4da49dd0c..6b4df7106 100644 --- a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift +++ b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift @@ -16,7 +16,6 @@ import XCTest final class DependencyTrackingTests: XCTestCase { func testDependenciesUpdatedSwift() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ "LibA/LibA.swift": """ diff --git a/Tests/SourceKitLSPTests/IndexTests.swift b/Tests/SourceKitLSPTests/IndexTests.swift index a32c42497..d67e18071 100644 --- a/Tests/SourceKitLSPTests/IndexTests.swift +++ b/Tests/SourceKitLSPTests/IndexTests.swift @@ -16,7 +16,6 @@ import XCTest final class IndexTests: XCTestCase { func testIndexSwiftModules() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ "LibA/LibA.swift": """ diff --git a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift index e8e5bd605..22e8d86df 100644 --- a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift @@ -154,7 +154,6 @@ final class PublishDiagnosticsTests: XCTestCase { func testDiagnosticUpdatedAfterDependentModuleIsBuilt() async throws { try SkipUnless.longTestsEnabled() - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ diff --git a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift index d225823ee..b68ec59c7 100644 --- a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift @@ -185,7 +185,6 @@ final class PullDiagnosticsTests: XCTestCase { func testDiagnosticUpdatedAfterDependentModuleIsBuilt() async throws { try SkipUnless.longTestsEnabled() - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ @@ -251,8 +250,6 @@ final class PullDiagnosticsTests: XCTestCase { } func testDiagnosticsWaitForDocumentToBePrepared() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() - nonisolated(unsafe) var diagnosticRequestSent = AtomicBool(initialValue: false) var serverOptions = SourceKitLSPServer.Options.testDefault serverOptions.indexTestHooks.preparationTaskDidStart = { @Sendable taskDescription in diff --git a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift index fdc2fd547..c510730dc 100644 --- a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift +++ b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift @@ -55,7 +55,6 @@ final class SwiftInterfaceTests: XCTestCase { } func testOpenInterface() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ "MyLibrary/MyLibrary.swift": """ @@ -152,7 +151,6 @@ final class SwiftInterfaceTests: XCTestCase { } func testSwiftInterfaceAcrossModules() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() let project = try await SwiftPMTestProject( files: [ "MyLibrary/MyLibrary.swift": """ diff --git a/Tests/SourceKitLSPTests/WorkspaceTests.swift b/Tests/SourceKitLSPTests/WorkspaceTests.swift index 10af11fce..13a295f3e 100644 --- a/Tests/SourceKitLSPTests/WorkspaceTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceTests.swift @@ -23,8 +23,6 @@ import XCTest final class WorkspaceTests: XCTestCase { func testMultipleSwiftPMWorkspaces() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() - // The package manifest is the same for both packages we open. let packageManifest = """ // swift-tools-version: 5.7 @@ -195,8 +193,6 @@ final class WorkspaceTests: XCTestCase { } func testSwiftPMPackageInSubfolder() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() - let packageManifest = """ // swift-tools-version: 5.7 @@ -274,8 +270,6 @@ final class WorkspaceTests: XCTestCase { } func testNestedSwiftPMWorkspacesWithoutDedicatedWorkspaceFolder() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() - // The package manifest is the same for both packages we open. let packageManifest = """ // swift-tools-version: 5.7 @@ -600,8 +594,6 @@ final class WorkspaceTests: XCTestCase { } func testChangeWorkspaceFolders() async throws { - try await SkipUnless.swiftpmStoresModulesInSubdirectory() - let project = try await MultiFileTestProject( files: [ "subdir/Sources/otherPackage/otherPackage.swift": """ @@ -775,8 +767,6 @@ final class WorkspaceTests: XCTestCase { func testIntegrationTest() async throws { // This test is doing the same as `test-sourcekit-lsp` in the `swift-integration-tests` repo. - try await SkipUnless.swiftpmStoresModulesInSubdirectory() - let project = try await SwiftPMTestProject( files: [ "Sources/clib/include/clib.h": """ From f3efac4d9da1141a7a039de7e5b87436ce656274 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 May 2024 16:56:17 -0700 Subject: [PATCH 12/69] Relax assertion around `testIntegrationTest` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depending on whether clangd has finished indexing/built an AST/something, we get different code completion results. The actual integration test is only checking that we have a result with `insertText` `clib_func`, which is satisfied independently of whether clangd’s progress. Adopt the same assertion in this test case. --- Tests/SourceKitLSPTests/WorkspaceTests.swift | 38 ++------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/Tests/SourceKitLSPTests/WorkspaceTests.swift b/Tests/SourceKitLSPTests/WorkspaceTests.swift index 10af11fce..e54028bcc 100644 --- a/Tests/SourceKitLSPTests/WorkspaceTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceTests.swift @@ -879,41 +879,7 @@ final class WorkspaceTests: XCTestCase { let cCompletionResponse = try await project.testClient.send( CompletionRequest(textDocument: TextDocumentIdentifier(clibcUri), position: clibcPositions["2️⃣"]) ) - XCTAssertEqual( - cCompletionResponse.items, - [ - // rdar://73762053: This should also suggest clib_other - CompletionItem( - label: " clib_func", - kind: .text, - deprecated: true, - sortText: "41b99800clib_func", - filterText: "clib_func", - insertText: "clib_func", - insertTextFormat: .plain, - textEdit: .textEdit(TextEdit(range: Range(clibcPositions["2️⃣"]), newText: "clib_func")) - ), - CompletionItem( - label: " include", - kind: .text, - deprecated: true, - sortText: "41d85b70include", - filterText: "include", - insertText: "include", - insertTextFormat: .plain, - textEdit: .textEdit(TextEdit(range: Range(clibcPositions["2️⃣"]), newText: "include")) - ), - CompletionItem( - label: " void", - kind: .text, - deprecated: true, - sortText: "41e677bbvoid", - filterText: "void", - insertText: "void", - insertTextFormat: .plain, - textEdit: .textEdit(TextEdit(range: Range(clibcPositions["2️⃣"]), newText: "void")) - ), - ] - ) + // rdar://73762053: This should also suggest clib_other + XCTAssert(cCompletionResponse.items.contains(where: { $0.insertText == "clib_func" })) } } From 6dabb1bec77f6df7852d98dd0bcfc703ead24f2e Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 May 2024 16:59:23 -0700 Subject: [PATCH 13/69] Fix a negation issue in the type of log message sent to the index log --- Sources/SourceKitLSP/SourceKitLSPServer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index d69b82796..174abf5da 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -834,7 +834,7 @@ extension SourceKitLSPServer { nonisolated func indexTaskDidProduceResult(_ result: IndexProcessResult) { self.sendNotificationToClient( LogMessageNotification( - type: result.failed ? .info : .warning, + type: result.failed ? .warning : .info, message: """ \(result.taskDescription) finished in \(result.duration) \(result.command) From 12b848ed58ad1aa1a49234a329213d2d34173a4c Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Tue, 28 May 2024 09:51:28 -0700 Subject: [PATCH 14/69] [SwiftPMBuildSystem] Adjust `BuildParameters` use to indicate a destination The change is introduced by https://github.com/apple/swift-package-manager/pull/7593 --- .../SwiftPMBuildSystem.swift | 23 ++++++++++++++----- .../SwiftPMBuildSystemTests.swift | 10 ++++---- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index e54cbfe4d..c22512334 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -102,7 +102,8 @@ public actor SwiftPMBuildSystem { public var projectRoot: TSCAbsolutePath var modulesGraph: ModulesGraph let workspace: Workspace - public let buildParameters: BuildParameters + public let toolsBuildParameters: BuildParameters + public let destinationBuildParameters: BuildParameters let fileSystem: FileSystem private let toolchainRegistry: ToolchainRegistry @@ -200,7 +201,16 @@ public actor SwiftPMBuildSystem { buildConfiguration = .release } - self.buildParameters = try BuildParameters( + self.toolsBuildParameters = try BuildParameters( + destination: .host, + dataPath: location.scratchDirectory.appending(component: toolchain.targetTriple.platformBuildPathComponent), + configuration: buildConfiguration, + toolchain: toolchain, + flags: buildSetup.flags + ) + + self.destinationBuildParameters = try BuildParameters( + destination: .target, dataPath: location.scratchDirectory.appending(component: toolchain.targetTriple.platformBuildPathComponent), configuration: buildConfiguration, toolchain: toolchain, @@ -282,8 +292,8 @@ extension SwiftPMBuildSystem { ) let plan = try BuildPlan( - productsBuildParameters: buildParameters, - toolsBuildParameters: buildParameters, + productsBuildParameters: destinationBuildParameters, + toolsBuildParameters: toolsBuildParameters, graph: modulesGraph, fileSystem: fileSystem, observabilityScope: observabilitySystem.topScope @@ -347,11 +357,12 @@ extension SwiftPMBuildSystem { extension SwiftPMBuildSystem: SKCore.BuildSystem { public var buildPath: TSCAbsolutePath { - return TSCAbsolutePath(buildParameters.buildPath) + return TSCAbsolutePath(destinationBuildParameters.buildPath) } public var indexStorePath: TSCAbsolutePath? { - return buildParameters.indexStoreMode == .off ? nil : TSCAbsolutePath(buildParameters.indexStore) + return destinationBuildParameters.indexStoreMode == .off + ? nil : TSCAbsolutePath(destinationBuildParameters.indexStore) } public var indexDatabasePath: TSCAbsolutePath? { diff --git a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift index 69cc4b0f2..3ad0a6a4d 100644 --- a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift +++ b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift @@ -142,7 +142,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { ) let aswift = packageRoot.appending(components: "Sources", "lib", "a.swift") - let hostTriple = await swiftpmBuildSystem.buildParameters.triple + let hostTriple = await swiftpmBuildSystem.destinationBuildParameters.triple let build = buildPath(root: packageRoot, platform: hostTriple.platformBuildPathComponent) assertEqual(await swiftpmBuildSystem.buildPath, build) @@ -211,7 +211,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { ) let aswift = packageRoot.appending(components: "Sources", "lib", "a.swift") - let hostTriple = await swiftpmBuildSystem.buildParameters.triple + let hostTriple = await swiftpmBuildSystem.destinationBuildParameters.triple let build = buildPath(root: packageRoot, config: config, platform: hostTriple.platformBuildPathComponent) assertEqual(await swiftpmBuildSystem.buildPath, build) @@ -435,7 +435,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { let acxx = packageRoot.appending(components: "Sources", "lib", "a.cpp") let bcxx = packageRoot.appending(components: "Sources", "lib", "b.cpp") let header = packageRoot.appending(components: "Sources", "lib", "include", "a.h") - let hostTriple = await swiftpmBuildSystem.buildParameters.triple + let hostTriple = await swiftpmBuildSystem.destinationBuildParameters.triple let build = buildPath(root: packageRoot, platform: hostTriple.platformBuildPathComponent) assertEqual(await swiftpmBuildSystem.buildPath, build) @@ -515,7 +515,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { let aswift = packageRoot.appending(components: "Sources", "lib", "a.swift") let arguments = try await swiftpmBuildSystem.buildSettings(for: aswift.asURI, language: .swift)!.compilerArguments assertArgumentsContain("-target", arguments: arguments) // Only one! - let hostTriple = await swiftpmBuildSystem.buildParameters.triple + let hostTriple = await swiftpmBuildSystem.destinationBuildParameters.triple #if os(macOS) assertArgumentsContain( @@ -741,7 +741,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { ) let aswift = packageRoot.appending(components: "Plugins", "MyPlugin", "a.swift") - let hostTriple = await swiftpmBuildSystem.buildParameters.triple + let hostTriple = await swiftpmBuildSystem.destinationBuildParameters.triple let build = buildPath(root: packageRoot, platform: hostTriple.platformBuildPathComponent) assertEqual(await swiftpmBuildSystem.buildPath, build) From 6d4953b3efe02da1130902924842b5ce8847e7bc Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 30 May 2024 19:23:45 -0700 Subject: [PATCH 15/69] Make tests pass with Xcode 15.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are no longer skipping these tests, which means that we need to make them account for different error messages emitted by Swift 5.10. I added warnings behind `#if compiler(>=6.1)` to give us a reminder that we can remove these checks when we no longer support running SourceKit-LSP with SwiftPM from Swift 5.10. Swift 6.1 doesn’t have to be this cut-off point but it’s the most likely candidate for now if we want to support the current and last Swift version from tip SourceKit-LSP. --- Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift | 3 +++ Tests/SourceKitLSPTests/DependencyTrackingTests.swift | 3 +++ Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift | 9 ++++++++- Tests/SourceKitLSPTests/PullDiagnosticsTests.swift | 9 ++++++++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index 8abbf5d07..5438dfacc 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -363,6 +363,9 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { private func compilerArguments(for file: URL, in buildTarget: any SwiftBuildTarget) async throws -> [String] { let compileArguments = try buildTarget.compileArguments(for: file) + #if compiler(>=6.1) + #warning("When we drop support for Swift 5.10 we no longer need to adjust compiler arguments for the Modules move") + #endif // Fix up compiler arguments that point to a `/Modules` subdirectory if the Swift version in the toolchain is less // than 6.0 because it places the modules one level higher up. let toolchainVersion = await orLog("Getting Swift version") { try await toolchainRegistry.default?.swiftVersion } diff --git a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift index 2780cf765..5262ef07c 100644 --- a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift +++ b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift @@ -50,6 +50,9 @@ final class DependencyTrackingTests: XCTestCase { // Semantic analysis: expect module import error. XCTAssertEqual(initialDiags.diagnostics.count, 1) if let diagnostic = initialDiags.diagnostics.first { + #if compiler(>=6.1) + #warning("When we drop support for Swift 5.10 we no longer need to check for the Objective-C error message") + #endif XCTAssert( diagnostic.message.contains("Could not build Objective-C module") || diagnostic.message.contains("No such module"), diff --git a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift index 22e8d86df..8535ca5cd 100644 --- a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift @@ -186,7 +186,14 @@ final class PublishDiagnosticsTests: XCTestCase { _ = try project.openDocument("LibB.swift") let diagnosticsBeforeBuilding = try await project.testClient.nextDiagnosticsNotification() - XCTAssert(diagnosticsBeforeBuilding.diagnostics.contains(where: { $0.message == "No such module 'LibA'" })) + XCTAssert( + diagnosticsBeforeBuilding.diagnostics.contains(where: { + #if compiler(>=6.1) + #warning("When we drop support for Swift 5.10 we no longer need to check for the Objective-C error message") + #endif + return $0.message == "No such module 'LibA'" || $0.message == "Could not build Objective-C module 'LibA'" + }) + ) try await SwiftPMTestProject.build(at: project.scratchDirectory) diff --git a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift index b68ec59c7..98b3039b0 100644 --- a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift @@ -222,7 +222,14 @@ final class PullDiagnosticsTests: XCTestCase { XCTFail("Expected full diagnostics report") return } - XCTAssert(fullReportBeforeBuilding.items.contains(where: { $0.message == "No such module 'LibA'" })) + XCTAssert( + fullReportBeforeBuilding.items.contains(where: { + #if compiler(>=6.1) + #warning("When we drop support for Swift 5.10 we no longer need to check for the Objective-C error message") + #endif + return $0.message == "No such module 'LibA'" || $0.message == "Could not build Objective-C module 'LibA'" + }) + ) let diagnosticsRefreshRequestReceived = self.expectation(description: "DiagnosticsRefreshRequest received") project.testClient.handleSingleRequest { (request: DiagnosticsRefreshRequest) in From 094eb0c237bd99081b765a63e2a84c5a3a8db307 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 30 May 2024 19:40:47 -0700 Subject: [PATCH 16/69] Relax `testMainFileChangesIfIncludeIsAdded` to allow `clangd` to return diagnostics from old build settings `clangd` may return diagnostics from the old build settings sometimes (I believe when it's still building the preamble for shared.h when the new build settings come in). Check that it eventually returns the correct diagnostics and allow it to return outdated diagnostics for a short while. --- .../MainFilesProviderTests.swift | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Tests/SourceKitLSPTests/MainFilesProviderTests.swift b/Tests/SourceKitLSPTests/MainFilesProviderTests.swift index 9923bbd47..779c8c05a 100644 --- a/Tests/SourceKitLSPTests/MainFilesProviderTests.swift +++ b/Tests/SourceKitLSPTests/MainFilesProviderTests.swift @@ -214,9 +214,19 @@ final class MainFilesProviderTests: XCTestCase { // 'MyFancyLibrary.c' now also includes 'shared.h'. Since it lexicographically preceeds MyLibrary, we should use its // build settings. - let postEditDiags = try await project.testClient.nextDiagnosticsNotification() - XCTAssertEqual(postEditDiags.diagnostics.count, 1) - let postEditDiag = try XCTUnwrap(postEditDiags.diagnostics.first) - XCTAssertEqual(postEditDiag.message, "Unused variable 'fromMyFancyLibrary'") + // `clangd` may return diagnostics from the old build settings sometimes (I believe when it's still building the + // preamble for shared.h when the new build settings come in). Check that it eventually returns the correct + // diagnostics. + var receivedCorrectDiagnostic = false + for _ in 0.. Date: Thu, 30 May 2024 23:04:52 -0700 Subject: [PATCH 17/69] Log targets marked out-of-date I am seeing more preparation tasks going on than I expect. This should help us figure out why targets are being marked as out-of-date. --- Sources/SemanticIndex/SemanticIndexManager.swift | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index e15699e16..6e5c8680c 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -312,8 +312,15 @@ public final actor SemanticIndexManager { // configured for macOS but not in target T configured for iOS. let targets = await changedFiles.asyncMap { await buildSystemManager.configuredTargets(for: $0) }.flatMap { $0 } if let dependentTargets = await buildSystemManager.targets(dependingOn: targets) { + logger.info( + """ + Marking targets as out-of-date: \ + \(String(dependentTargets.map(\.description).joined(separator: ", "))) + """ + ) await preparationUpToDateTracker.markOutOfDate(dependentTargets) } else { + logger.info("Marking all targets as out-of-date") await preparationUpToDateTracker.markAllKnownOutOfDate() // `markAllOutOfDate` only marks targets out-of-date that have been indexed before. Also mark all targets with // in-progress preparation out of date. So we don't get into the following situation, which would result in an From 6e952f9de7cd9ab6e168932d4f7f593b976316bd Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 30 May 2024 23:18:00 -0700 Subject: [PATCH 18/69] Use default toolchain in `sourcekit-lsp run-sourcekitd-request` if no sourcekitd is specified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes local reduction of failing sourcekitd requests easier if you don’t always need to add the `--sourcekitd` parameter. --- Sources/Diagnose/CMakeLists.txt | 2 +- ...nd.swift => RunSourcekitdRequestCommand.swift} | 15 +++++++++++++-- Sources/sourcekit-lsp/SourceKitLSP.swift | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) rename Sources/Diagnose/{SourcekitdRequestCommand.swift => RunSourcekitdRequestCommand.swift} (85%) diff --git a/Sources/Diagnose/CMakeLists.txt b/Sources/Diagnose/CMakeLists.txt index b93da89cd..32caa5515 100644 --- a/Sources/Diagnose/CMakeLists.txt +++ b/Sources/Diagnose/CMakeLists.txt @@ -12,8 +12,8 @@ add_library(Diagnose STATIC ReductionError.swift ReproducerBundle.swift RequestInfo.swift + RunSourcekitdRequestCommand.swift SourceKitD+RunWithYaml.swift - SourcekitdRequestCommand.swift SourceKitDRequestExecutor.swift SourceReducer.swift StderrStreamConcurrencySafe.swift diff --git a/Sources/Diagnose/SourcekitdRequestCommand.swift b/Sources/Diagnose/RunSourcekitdRequestCommand.swift similarity index 85% rename from Sources/Diagnose/SourcekitdRequestCommand.swift rename to Sources/Diagnose/RunSourcekitdRequestCommand.swift index ba4e0686f..771fc4777 100644 --- a/Sources/Diagnose/SourcekitdRequestCommand.swift +++ b/Sources/Diagnose/RunSourcekitdRequestCommand.swift @@ -12,12 +12,13 @@ import ArgumentParser import Foundation +import SKCore import SKSupport import SourceKitD import struct TSCBasic.AbsolutePath -public struct SourceKitdRequestCommand: AsyncParsableCommand { +public struct RunSourceKitdRequestCommand: AsyncParsableCommand { public static let configuration = CommandConfiguration( commandName: "run-sourcekitd-request", abstract: "Run a sourcekitd request and print its result", @@ -28,7 +29,7 @@ public struct SourceKitdRequestCommand: AsyncParsableCommand { name: .customLong("sourcekitd"), help: "Path to sourcekitd.framework/sourcekitd" ) - var sourcekitdPath: String + var sourcekitdPath: String? @Option( name: .customLong("request-file"), @@ -44,6 +45,16 @@ public struct SourceKitdRequestCommand: AsyncParsableCommand { public func run() async throws { var requestString = try String(contentsOf: URL(fileURLWithPath: sourcekitdRequestPath)) + let installPath = try AbsolutePath(validating: Bundle.main.bundlePath) + let sourcekitdPath = + if let sourcekitdPath { + sourcekitdPath + } else if let path = await ToolchainRegistry(installPath: installPath).default?.sourcekitd?.pathString { + path + } else { + print("Did not find sourcekitd in the toolchain. Specify path to sourcekitd manually by passing --sourcekitd") + throw ExitCode(1) + } let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( dylibPath: try! AbsolutePath(validating: sourcekitdPath) ) diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index b8942db77..0cece17c0 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -108,7 +108,7 @@ struct SourceKitLSP: AsyncParsableCommand { IndexCommand.self, ReduceCommand.self, ReduceFrontendCommand.self, - SourceKitdRequestCommand.self, + RunSourceKitdRequestCommand.self, ] ) From e2f6f5d991eca7bcfe9251e6375d3a9027034be2 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 30 May 2024 23:45:41 -0700 Subject: [PATCH 19/69] Flush stdout after issuing message that we re-run tests in serial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some reason the log message that we’re re-running tests in serial always showed up after the serial run in the build log output. Test if flushing stdout fixes the issue. --- Utilities/build-script-helper.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Utilities/build-script-helper.py b/Utilities/build-script-helper.py index 27f00cbd8..2d50aafba 100755 --- a/Utilities/build-script-helper.py +++ b/Utilities/build-script-helper.py @@ -206,8 +206,8 @@ def run_tests(swift_exec: str, args: argparse.Namespace) -> None: """ swiftpm_args = get_swiftpm_options(swift_exec, args, suppress_verbose=True) additional_env = get_swiftpm_environment_variables(swift_exec, args) - # 'swift test' doesn't print os_log output to the command line. Use the - # `NonDarwinLogger` that prints to stderr so we can view the log output in CI test + # 'swift test' doesn't print os_log output to the command line. Use the + # `NonDarwinLogger` that prints to stderr so we can view the log output in CI test # runs. additional_env['SOURCEKITLSP_FORCE_NON_DARWIN_LOGGER'] = '1' @@ -233,6 +233,7 @@ def run_tests(swift_exec: str, args: argparse.Namespace) -> None: check_call(cmd + ['--parallel'], additional_env=additional_env, verbose=args.verbose) except: print('--- Running tests in parallel failed. Re-running tests serially to capture more actionable output.') + sys.stdout.flush() check_call(cmd, additional_env=additional_env, verbose=args.verbose) # Return with non-zero exit code even if serial test execution succeeds. raise SystemExit(1) @@ -249,7 +250,7 @@ def install(swift_exec: str, args: argparse.Namespace) -> None: swiftpm_args = get_swiftpm_options(swift_exec, args) additional_env = get_swiftpm_environment_variables(swift_exec, args) bin_path = swiftpm_bin_path(swift_exec, swiftpm_args=swiftpm_args, additional_env=additional_env) - + for prefix in args.install_prefixes: install_binary('sourcekit-lsp', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose) From 6124c9d5297aaba8b9820d39412b04c48461e476 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 00:06:23 -0700 Subject: [PATCH 20/69] Replace an assertion by a fault log in `WorkDoneProgressState` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We didn’t hit this assertion but I noticed it and the best practice in sourcekit-lsp is to log a fault instead of throwing an assertion for any kind of recoverable error. --- Sources/SourceKitLSP/WorkDoneProgressState.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Sources/SourceKitLSP/WorkDoneProgressState.swift b/Sources/SourceKitLSP/WorkDoneProgressState.swift index c6022662a..2a2076a58 100644 --- a/Sources/SourceKitLSP/WorkDoneProgressState.swift +++ b/Sources/SourceKitLSP/WorkDoneProgressState.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +import LSPLogging import LanguageServerProtocol import SKSupport @@ -108,7 +109,10 @@ final actor WorkDoneProgressState { } func endProgressImpl(server: SourceKitLSPServer) async { - assert(activeTasks > 0, "Unbalanced startProgress/endProgress calls") + guard activeTasks > 0 else { + logger.fault("Unbalanced startProgress/endProgress calls") + return + } activeTasks -= 1 guard await server.capabilityRegistry?.clientCapabilities.window?.workDoneProgress ?? false else { return From 58d448a678c5a8a381be43166b8f9ae504c00f0c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 00:09:53 -0700 Subject: [PATCH 21/69] Create all notification/request handling signposts in the same logging category MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This shows up nicer in Instruments because it doesn’t open a new lane for every signpost. Instead, they all share the same lane. Concurrent signposts are still shown as overlapping. --- Sources/SourceKitLSP/SourceKitLSPServer.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index 51132453c..e883345c7 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -541,7 +541,7 @@ extension SourceKitLSPServer: MessageHandler { let notificationID = notificationIDForLogging.fetchAndIncrement() - let signposter = Logger(subsystem: LoggingScope.subsystem, category: "notification-\(notificationID)") + let signposter = Logger(subsystem: LoggingScope.subsystem, category: "message-handling") .makeSignposter() let signpostID = signposter.makeSignpostID() let state = signposter.beginInterval("Notification", id: signpostID, "\(type(of: params))") @@ -591,7 +591,7 @@ extension SourceKitLSPServer: MessageHandler { id: RequestID, reply: @Sendable @escaping (LSPResult) -> Void ) { - let signposter = Logger(subsystem: LoggingScope.subsystem, category: "request-\(id)").makeSignposter() + let signposter = Logger(subsystem: LoggingScope.subsystem, category: "message-handling").makeSignposter() let signpostID = signposter.makeSignpostID() let state = signposter.beginInterval("Request", id: signpostID, "\(R.self)") From 01e8858f7fa5f2aa8a632e2a2ca05cccbdff7eb0 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 07:05:43 -0700 Subject: [PATCH 22/69] =?UTF-8?q?Don=E2=80=99t=20show=20a=20=E2=80=9CPrepa?= =?UTF-8?q?ring=20current=20file=E2=80=9D=20progress=20while=20determining?= =?UTF-8?q?=20target=20of=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every time a request regarding a text document is sent, we call `schedulePreparationForEditorFunctionality` on the `SemanticIndexManager` with the assumption that it’s a no-op if the target is already prepared. We did, however, show the “Preparing current file” status while we were determining the file‘s target to determine that the file’s target is up-to-date. --- .../SemanticIndex/SemanticIndexManager.swift | 50 ++++++++++++++++--- .../BackgroundIndexingTests.swift | 19 +++++++ 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index e15699e16..158ccd4a2 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -96,6 +96,26 @@ public enum IndexProgressStatus { } } +/// See `SemanticIndexManager.inProgressPrepareForEditorTask`. +fileprivate struct InProgressPrepareForEditorTask { + fileprivate enum State { + case determiningCanonicalConfiguredTarget + case preparingTarget + } + /// A unique ID that identifies the preparation task and is used to set + /// `SemanticIndexManager.inProgressPrepareForEditorTask` to `nil` when the current in progress task finishes. + let id: UUID + + /// The document that is being prepared. + let document: DocumentURI + + /// The task that prepares the document. Should never be awaited and only be used to cancel the task. + let task: Task + + /// Whether the task is currently determining the file's target or actually preparing the document. + var state: State +} + /// Schedules index tasks and keeps track of the index status of files. public final actor SemanticIndexManager { /// The underlying index. This is used to check if the index of a file is already up-to-date, in which case it doesn't @@ -133,10 +153,7 @@ public final actor SemanticIndexManager { /// avoid the following scenario: The user browses through documents from targets A, B, and C in quick succession. We /// don't want stack preparation of A, B, and C. Instead we want to only prepare target C - and also finish /// preparation of A if it has already started when the user opens C. - /// - /// `id` is a unique ID that identifies the preparation task and is used to set `inProgressPrepareForEditorTask` to - /// `nil` when the current in progress task finishes. - private var inProgressPrepareForEditorTask: (id: UUID, document: DocumentURI, task: Task)? = nil + private var inProgressPrepareForEditorTask: InProgressPrepareForEditorTask? = nil /// The `TaskScheduler` that manages the scheduling of index tasks. This is shared among all `SemanticIndexManager`s /// in the process, to ensure that we don't schedule more index operations than processor cores from multiple @@ -161,7 +178,7 @@ public final actor SemanticIndexManager { /// A summary of the tasks that this `SemanticIndexManager` has currently scheduled or is currently indexing. public var progressStatus: IndexProgressStatus { - if inProgressPrepareForEditorTask != nil { + if let inProgressPrepareForEditorTask, inProgressPrepareForEditorTask.state == .preparingTarget { return .preparingFileForEditorFunctionality } if generateBuildGraphTask != nil { @@ -370,7 +387,20 @@ public final actor SemanticIndexManager { let id = UUID() let task = Task(priority: priority) { await withLoggingScope("preparation") { - await self.prepareFileForEditorFunctionality(uri) + // Should be kept in sync with `prepareFileForEditorFunctionality` + guard let target = await buildSystemManager.canonicalConfiguredTarget(for: uri) else { + return + } + if Task.isCancelled { + return + } + if inProgressPrepareForEditorTask?.id == id { + if inProgressPrepareForEditorTask?.state != .determiningCanonicalConfiguredTarget { + logger.fault("inProgressPrepareForEditorTask is in unexpected state") + } + inProgressPrepareForEditorTask?.state = .preparingTarget + } + await self.prepare(targets: [target], priority: nil) if inProgressPrepareForEditorTask?.id == id { inProgressPrepareForEditorTask = nil self.indexProgressStatusDidChange() @@ -378,7 +408,12 @@ public final actor SemanticIndexManager { } } inProgressPrepareForEditorTask?.task.cancel() - inProgressPrepareForEditorTask = (id, uri, task) + inProgressPrepareForEditorTask = InProgressPrepareForEditorTask( + id: id, + document: uri, + task: task, + state: .determiningCanonicalConfiguredTarget + ) self.indexProgressStatusDidChange() } @@ -387,6 +422,7 @@ public final actor SemanticIndexManager { /// /// If file's target is known to be up-to-date, this returns almost immediately. public func prepareFileForEditorFunctionality(_ uri: DocumentURI) async { + // Should be kept in sync with `schedulePreparationForEditorFunctionality`. guard let target = await buildSystemManager.canonicalConfiguredTarget(for: uri) else { return } diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index 50aac010f..d6bf57790 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -868,4 +868,23 @@ final class BackgroundIndexingTests: XCTestCase { let message = try await project.testClient.nextNotification(ofType: ShowMessageNotification.self) XCTAssert(message.message.contains("Background indexing"), "Received unexpected message: \(message.message)") } + + func testNoPreparationStatusIfTargetIsUpToDate() async throws { + let project = try await SwiftPMTestProject( + files: [ + "Lib.swift": "" + ], + capabilities: ClientCapabilities(window: WindowClientCapabilities(workDoneProgress: true)), + enableBackgroundIndexing: true + ) + + // Opening the document prepares it for editor functionality. Its target is already prepared, so we shouldn't show + // a work done progress for it. + project.testClient.handleSingleRequest { (request: CreateWorkDoneProgressRequest) in + XCTFail("Received unexpected create work done progress: \(request)") + return VoidResponse() + } + _ = try project.openDocument("Lib.swift") + _ = try await project.testClient.send(BarrierRequest()) + } } From 2beae820b37ef9ab2af822190c2f1d9b0bb21773 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 07:59:47 -0700 Subject: [PATCH 23/69] =?UTF-8?q?Don=E2=80=99t=20catch=20`CancellationErro?= =?UTF-8?q?r`=20for=20document=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We were catching errors during diagnostic generation because VS Code will not request diagnostics again if SourceKit-LSP returns an error to any diagnostic request. We were, however, a little over-eager when doing this and also caught cancellation errors, which means that if the user made multiple edits in quick succession (which would cancel the diagnostics request from the first edit), we would return empty diagnostics, clearing the displayed diagnostics. --- .../TestSourceKitLSPClient.swift | 10 +++++-- .../Swift/SwiftLanguageService.swift | 2 ++ .../PullDiagnosticsTests.swift | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index 9f61a9b9a..83915ad10 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -206,10 +206,16 @@ public final class TestSourceKitLSPClient: MessageHandler { /// /// This version of the `send` function should only be used if some action needs to be performed after the request is /// sent but before it returns a result. - public func send(_ request: R, completionHandler: @escaping (LSPResult) -> Void) { - server.handle(request, id: .number(Int(nextRequestID.fetchAndIncrement()))) { result in + @discardableResult + public func send( + _ request: R, + completionHandler: @escaping (LSPResult) -> Void + ) -> RequestID { + let requestID = RequestID.number(Int(nextRequestID.fetchAndIncrement())) + server.handle(request, id: requestID) { result in completionHandler(result) } + return requestID } /// Send the notification to `server`. diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index 48d88e098..1f4dbae92 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -888,6 +888,8 @@ extension SwiftLanguageService { buildSettings: buildSettings ) return .full(diagnosticReport) + } catch let error as CancellationError { + throw error } catch { // VS Code does not request diagnostics again for a document if the diagnostics request failed. // Since sourcekit-lsp usually recovers from failures (e.g. after sourcekitd crashes), this is undesirable. diff --git a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift index b68ec59c7..4998cd8f8 100644 --- a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift @@ -309,4 +309,32 @@ final class PullDiagnosticsTests: XCTestCase { diagnosticRequestSent.value = true try await fulfillmentOfOrThrow([receivedDiagnostics]) } + + func testDontReturnEmptyDiagnosticsIfDiagnosticRequestIsCancelled() async throws { + let diagnosticRequestCancelled = self.expectation(description: "diagnostic request cancelled") + var serverOptions = SourceKitLSPServer.Options.testDefault + serverOptions.indexTestHooks.preparationTaskDidStart = { _ in + await self.fulfillment(of: [diagnosticRequestCancelled], timeout: defaultTimeout) + } + let project = try await SwiftPMTestProject( + files: [ + "Lib.swift": "let x: String = 1" + ], + serverOptions: serverOptions, + enableBackgroundIndexing: true, + pollIndex: false + ) + let (uri, _) = try project.openDocument("Lib.swift") + + let diagnosticResponseReceived = self.expectation(description: "Received diagnostic response") + let requestID = project.testClient.send( + DocumentDiagnosticsRequest(textDocument: TextDocumentIdentifier(uri)) + ) { result in + XCTAssertEqual(result.failure?.code, .cancelled) + diagnosticResponseReceived.fulfill() + } + project.testClient.send(CancelRequestNotification(id: requestID)) + diagnosticRequestCancelled.fulfill() + try await fulfillmentOfOrThrow([diagnosticResponseReceived]) + } } From 2da283b2ff916c4778fa3cd386c3bc61215be2f8 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 10:10:29 -0700 Subject: [PATCH 24/69] =?UTF-8?q?Fix=20two=20cases=20where=20we=20weren?= =?UTF-8?q?=E2=80=99t=20cleaning=20up=20test=20scratch=20directories?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/SKTestSupport/SwiftPMDependencyProject.swift | 11 ++++++++--- Tests/SourceKitLSPTests/IndexTests.swift | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Sources/SKTestSupport/SwiftPMDependencyProject.swift b/Sources/SKTestSupport/SwiftPMDependencyProject.swift index 496834c5c..e0f38f2aa 100644 --- a/Sources/SKTestSupport/SwiftPMDependencyProject.swift +++ b/Sources/SKTestSupport/SwiftPMDependencyProject.swift @@ -22,8 +22,13 @@ import struct TSCBasic.ProcessResult /// A SwiftPM package that gets written to disk and for which a Git repository is initialized with a commit tagged /// `1.0.0`. This repository can then be used as a dependency for another package, usually a `SwiftPMTestProject`. public class SwiftPMDependencyProject { + /// The scratch directory created for the dependency project. + public let scratchDirectory: URL + /// The directory in which the repository lives. - public let packageDirectory: URL + public var packageDirectory: URL { + return scratchDirectory.appendingPathComponent("MyDependency") + } private func runGitCommand(_ arguments: [String], workingDirectory: URL) async throws { enum Error: Swift.Error { @@ -66,7 +71,7 @@ public class SwiftPMDependencyProject { manifest: String = defaultPackageManifest, testName: String = #function ) async throws { - packageDirectory = try testScratchDir(testName: testName).appendingPathComponent("MyDependency") + scratchDirectory = try testScratchDir(testName: testName) var files = files files["Package.swift"] = manifest @@ -94,7 +99,7 @@ public class SwiftPMDependencyProject { deinit { if cleanScratchDirectories { - try? FileManager.default.removeItem(at: packageDirectory) + try? FileManager.default.removeItem(at: scratchDirectory) } } diff --git a/Tests/SourceKitLSPTests/IndexTests.swift b/Tests/SourceKitLSPTests/IndexTests.swift index c5ae0d14a..52be924a0 100644 --- a/Tests/SourceKitLSPTests/IndexTests.swift +++ b/Tests/SourceKitLSPTests/IndexTests.swift @@ -120,6 +120,7 @@ final class IndexTests: XCTestCase { 2️⃣foo() } """, + workspaceDirectory: workspaceDirectory, cleanUp: cleanUp ) From b08895b82cf2e7ed04ea1ee63a60db4efb4fdc26 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 15:36:40 -0700 Subject: [PATCH 25/69] Add a few more arguments to allow errors during indexing I noticed that we were missing a bunch more arguments (in particular for clang index invocations), while diagnosing rdar://129071600, which is fixed by passing `-experimental-allow-module-with-compiler-errors` to the Swift index invocations. rdar://129071600 --- .../UpdateIndexStoreTaskDescription.swift | 47 +++++++++++++------ .../BackgroundIndexingTests.swift | 43 +++++++++++++++++ 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift index cd0cc1721..54d7dbeaa 100644 --- a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -430,11 +430,6 @@ private func adjustSwiftCompilerArgumentsForIndexStoreUpdate( .option("output-file-map", [.singleDash], [.separatedBySpace, .separatedByEqualSign]), ] - let removeFrontendFlags = [ - "-experimental-skip-non-inlinable-function-bodies", - "-experimental-skip-all-function-bodies", - ] - var result: [String] = [] result.reserveCapacity(compilerArguments.count) var iterator = compilerArguments.makeIterator() @@ -448,18 +443,14 @@ private func adjustSwiftCompilerArgumentsForIndexStoreUpdate( case nil: break } - if argument == "-Xfrontend" { - if let nextArgument = iterator.next() { - if removeFrontendFlags.contains(nextArgument) { - continue - } - result += [argument, nextArgument] - continue - } - } result.append(argument) } + result += supplementalClangIndexingArgs.flatMap { ["-Xcc", $0] } result += [ + // Preparation produces modules with errors. We should allow reading them. + "-Xfrontend", "-experimental-allow-module-with-compiler-errors", + // Avoid emitting the ABI descriptor, we don't need it + "-Xfrontend", "-empty-abi-descriptor", "-index-file", "-index-file-path", fileToIndex.pseudoPath, // batch mode is not compatible with -index-file @@ -520,12 +511,40 @@ private func adjustClangCompilerArgumentsForIndexStoreUpdate( } result.append(argument) } + result += supplementalClangIndexingArgs result.append( "-fsyntax-only" ) return result } +#if compiler(>=6.1) +#warning( + "Remove -fmodules-validate-system-headers from supplementalClangIndexingArgs once all supported Swift compilers have https://github.com/apple/swift/pull/74063" +) +#endif + +fileprivate let supplementalClangIndexingArgs: [String] = [ + // Retain extra information for indexing + "-fretain-comments-from-system-headers", + // Pick up macro definitions during indexing + "-Xclang", "-detailed-preprocessing-record", + + // libclang uses 'raw' module-format. Match it so we can reuse the module cache and PCHs that libclang uses. + "-Xclang", "-fmodule-format=raw", + + // Be less strict - we want to continue and typecheck/index as much as possible + "-Xclang", "-fallow-pch-with-compiler-errors", + "-Xclang", "-fallow-pcm-with-compiler-errors", + "-Wno-non-modular-include-in-framework-module", + "-Wno-incomplete-umbrella", + + // sourcekitd adds `-fno-modules-validate-system-headers` before https://github.com/apple/swift/pull/74063. + // This completely disables system module validation and never re-builds pcm for system modules. The intended behavior + // is to only re-build those PCMs once per sourcekitd session. + "-fmodules-validate-system-headers", +] + fileprivate extension Sequence { /// Returns `true` if this sequence contains an element that is equal to an element in `otherSequence` when /// considering two elements as equal if they satisfy `predicate`. diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index d6bf57790..ada1ee3fb 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -887,4 +887,47 @@ final class BackgroundIndexingTests: XCTestCase { _ = try project.openDocument("Lib.swift") _ = try await project.testClient.send(BarrierRequest()) } + + func testImportPreparedModuleWithFunctionBodiesSkipped() async throws { + // This test case was crashing the indexing compiler invocation for Client if Lib was built for index preparation + // (using `-enable-library-evolution -experimental-skip-all-function-bodies -experimental-lazy-typecheck`) but x + // Client was not indexed with `-experimental-allow-module-with-compiler-errors`. rdar://129071600 + let project = try await SwiftPMTestProject( + files: [ + "Lib/Lib.swift": """ + public class TerminalController { + public var 1️⃣width: Int { 1 } + } + """, + "Client/Client.swift": """ + import Lib + + func test(terminal: TerminalController) { + let width = terminal.width + } + """, + ], + manifest: """ + // swift-tools-version: 5.7 + + import PackageDescription + + let package = Package( + name: "MyLibrary", + targets: [ + .target(name: "Lib"), + .target(name: "Client", dependencies: ["Lib"]), + ] + ) + """, + enableBackgroundIndexing: true + ) + let (uri, positions) = try project.openDocument("Lib.swift") + + // Check that we indexed `Client.swift` by checking that we return a rename location within it. + let result = try await project.testClient.send( + RenameRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"], newName: "height") + ) + XCTAssertEqual((result?.changes?.keys).map(Set.init), [uri, try project.uri(for: "Client.swift")]) + } } From b7a502c1f46ff4a4a4a0012c0c43ffe2f149875c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 30 May 2024 23:00:52 -0700 Subject: [PATCH 26/69] When SwiftPM supports `--experimental-prepare-for-indexing` pass it to the prepare command The main purpose for now is that this makes it easier for me to live on background indexing combined with https://github.com/apple/swift-package-manager/pull/7574. --- .../SwiftPMBuildSystem.swift | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index 106d08a1e..19bf8a197 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -107,6 +107,45 @@ public actor SwiftPMBuildSystem { let fileSystem: FileSystem private let toolchainRegistry: ToolchainRegistry + private let swiftBuildSupportsPrepareForIndexingTask = SKSupport.ThreadSafeBox?>(initialValue: nil) + + #if compiler(>=6.1) + #warning( + "Remove swiftBuildSupportsPrepareForIndexing when we no longer need to support SwiftPM versions that don't have support for `--experimental-prepare-for-indexing`" + ) + #endif + /// Whether `swift build` supports the `--experimental-prepare-for-indexing` flag. + private var swiftBuildSupportsPrepareForIndexing: Bool { + get async { + let task = swiftBuildSupportsPrepareForIndexingTask.withLock { task in + if let task { + return task + } + let newTask = Task { () -> Bool in + guard let swift = await toolchainRegistry.default?.swift else { + return false + } + + do { + let process = Process(args: swift.pathString, "build", "--help-hidden") + try process.launch() + let result = try await process.waitUntilExit() + guard let output = String(bytes: try result.output.get(), encoding: .utf8) else { + return false + } + return output.contains("--experimental-prepare-for-indexing") + } catch { + return false + } + } + task = newTask + return newTask + } + + return await task.value + } + } + var fileToTarget: [AbsolutePath: SwiftBuildTarget] = [:] var sourceDirToTarget: [AbsolutePath: SwiftBuildTarget] = [:] @@ -523,13 +562,16 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { ) return } - let arguments = [ + var arguments = [ swift.pathString, "build", "--package-path", workspacePath.pathString, "--scratch-path", self.workspace.location.scratchDirectory.pathString, "--disable-index-store", "--target", target.targetID, ] + if await swiftBuildSupportsPrepareForIndexing { + arguments.append("--experimental-prepare-for-indexing") + } if Task.isCancelled { return } From 2da648efd30666de8c6599a74f88b9e7e5c6fe59 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 16:41:20 -0700 Subject: [PATCH 27/69] Improve folding ranges if editor only supports line folding If the folding range doesn't end at the end of the last line, exclude that line from the folding range since the end line gets folded away. This means if we reported `end.line`, we would eg. fold away the `}` that matches a `{`, which looks surprising. If the folding range does end at the end of the line we are in cases that don't have a closing indicator (like comments), so we can fold the last line as well. --- Sources/SourceKitLSP/Swift/FoldingRange.swift | 28 ++++++++- .../SourceKitLSPTests/FoldingRangeTests.swift | 58 +++++++++++++++++-- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/Sources/SourceKitLSP/Swift/FoldingRange.swift b/Sources/SourceKitLSP/Swift/FoldingRange.swift index cfe99be5b..3d71b2204 100644 --- a/Sources/SourceKitLSP/Swift/FoldingRange.swift +++ b/Sources/SourceKitLSP/Swift/FoldingRange.swift @@ -12,6 +12,7 @@ import LSPLogging import LanguageServerProtocol +import SKSupport import SwiftSyntax fileprivate final class FoldingRangeFinder: SyntaxAnyVisitor { @@ -210,9 +211,21 @@ fileprivate final class FoldingRangeFinder: SyntaxAnyVisitor { let end = snapshot.positionOf(utf8Offset: end.utf8Offset) let range: FoldingRange if lineFoldingOnly { + // If the folding range doesn't end at the end of the last line, exclude that line from the folding range since + // the end line gets folded away. This means if we reported `end.line`, we would eg. fold away the `}` that + // matches a `{`, which looks surprising. + // If the folding range does end at the end of the line we are in cases that don't have a closing indicator (like + // comments), so we can fold the last line as well. + let endLine: Int + if snapshot.lineTable.isAtEndOfLine(end) { + endLine = end.line + } else { + endLine = end.line - 1 + } + // Since the client cannot fold less than a single line, if the // fold would span 1 line there's no point in reporting it. - guard end.line > start.line else { + guard endLine > start.line else { return .visitChildren } @@ -221,7 +234,7 @@ fileprivate final class FoldingRangeFinder: SyntaxAnyVisitor { range = FoldingRange( startLine: start.line, startUTF16Index: nil, - endLine: end.line, + endLine: endLine, endUTF16Index: nil, kind: kind ) @@ -264,3 +277,14 @@ extension SwiftLanguageService { return ranges.sorted() } } + +fileprivate extension LineTable { + func isAtEndOfLine(_ position: Position) -> Bool { + guard position.line >= 0, position.line < self.count else { + return false + } + let line = self[position.line] + let suffixAfterPositionColumn = line[line.utf16.index(line.startIndex, offsetBy: position.utf16index)...] + return suffixAfterPositionColumn.allSatisfy(\.isNewline) + } +} diff --git a/Tests/SourceKitLSPTests/FoldingRangeTests.swift b/Tests/SourceKitLSPTests/FoldingRangeTests.swift index 0798edc67..8fb9500c7 100644 --- a/Tests/SourceKitLSPTests/FoldingRangeTests.swift +++ b/Tests/SourceKitLSPTests/FoldingRangeTests.swift @@ -96,10 +96,9 @@ final class FoldingRangeTests: XCTestCase { try await assertFoldingRanges( markedSource: """ 1️⃣func foo() { - - 2️⃣} - """ - , + 2️⃣ + } + """, expectedRanges: [ FoldingRangeSpec(from: "1️⃣", to: "2️⃣") ], @@ -107,6 +106,53 @@ final class FoldingRangeTests: XCTestCase { ) } + func testLineFoldingOfFunctionWithMultiLineParameters() async throws { + try await assertFoldingRanges( + markedSource: """ + 1️⃣func foo( + 2️⃣ param: Int + 3️⃣) { + print(param) + 4️⃣ + } + """, + expectedRanges: [ + FoldingRangeSpec(from: "1️⃣", to: "2️⃣"), + FoldingRangeSpec(from: "3️⃣", to: "4️⃣"), + ], + lineFoldingOnly: true + ) + } + + func testLineFoldingOfComment() async throws { + try await assertFoldingRanges( + markedSource: """ + 1️⃣// abc + // def + 2️⃣// ghi + + """, + expectedRanges: [ + FoldingRangeSpec(from: "1️⃣", to: "2️⃣", kind: .comment) + ], + lineFoldingOnly: true + ) + } + + func testLineFoldingOfCommentAtEndOfFile() async throws { + try await assertFoldingRanges( + markedSource: """ + 1️⃣// abc + // def + 2️⃣// ghi + """, + expectedRanges: [ + FoldingRangeSpec(from: "1️⃣", to: "2️⃣", kind: .comment) + ], + lineFoldingOnly: true + ) + } + func testLineFoldingDoesntReportSingleLine() async throws { try await assertFoldingRanges( markedSource: """ @@ -272,8 +318,8 @@ final class FoldingRangeTests: XCTestCase { try await assertFoldingRanges( markedSource: """ let x = [1️⃣ - 1: "one", - 2: "two", + 1: "one", + 2: "two", 3: "three" 2️⃣] """, From 02576d6e9952b2672e69df5a42bd10254f36e253 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 30 May 2024 18:47:09 -0700 Subject: [PATCH 28/69] Change `QueuedTask.cancelledToBeRescheduled` to be a normal Bool instead of an atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s only modified from method isolated to `QueuedTask`, so there’s no need for it to be an `AtomicBool`. --- Sources/SKCore/TaskScheduler.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/SKCore/TaskScheduler.swift b/Sources/SKCore/TaskScheduler.swift index 753344976..4332776eb 100644 --- a/Sources/SKCore/TaskScheduler.swift +++ b/Sources/SKCore/TaskScheduler.swift @@ -144,7 +144,7 @@ public actor QueuedTask { /// Whether `cancelToBeRescheduled` has been called on this `QueuedTask`. /// /// Gets reset every time `executionTask` finishes. - nonisolated(unsafe) private var cancelledToBeRescheduled: AtomicBool = .init(initialValue: false) + private var cancelledToBeRescheduled: Bool = false /// Whether `resultTask` has been cancelled. private nonisolated(unsafe) var resultTaskCancelled: AtomicBool = .init(initialValue: false) @@ -246,9 +246,9 @@ public actor QueuedTask { private func finalizeExecution() async -> ExecutionTaskFinishStatus { self.executionTask = nil _isExecuting.value = false - if Task.isCancelled && self.cancelledToBeRescheduled.value { + if Task.isCancelled && self.cancelledToBeRescheduled { await executionStateChangedCallback?(self, .cancelledToBeRescheduled) - self.cancelledToBeRescheduled.value = false + self.cancelledToBeRescheduled = false return ExecutionTaskFinishStatus.cancelledToBeRescheduled } else { await executionStateChangedCallback?(self, .finished) @@ -263,7 +263,7 @@ public actor QueuedTask { guard let executionTask else { return } - self.cancelledToBeRescheduled.value = true + self.cancelledToBeRescheduled = true executionTask.cancel() self.executionTask = nil } From cc1280f5e08f7abed5a6abb37fe0aa8d4eb3454c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 30 May 2024 18:54:46 -0700 Subject: [PATCH 29/69] =?UTF-8?q?Don=E2=80=99t=20start=20executing=20a=20t?= =?UTF-8?q?ask=20when=20`cancelToBeRescheduled`=20is=20called=20before=20`?= =?UTF-8?q?execute`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `QueuedTask.execute` is called from a detached task in `TaskScheduler.poke` but we insert it into the `currentlyExecutingTasks` queue beforehand. This left a short windows in which we could cancel the task to reschedule it before it actually started executing. Change `QueuedTask` to just return immediately from `execute` if it has been cancelled to be rescheduled beforehand. --- Sources/SKCore/TaskScheduler.swift | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Sources/SKCore/TaskScheduler.swift b/Sources/SKCore/TaskScheduler.swift index 4332776eb..2bae9c161 100644 --- a/Sources/SKCore/TaskScheduler.swift +++ b/Sources/SKCore/TaskScheduler.swift @@ -228,6 +228,15 @@ public actor QueuedTask { /// Execution might be canceled to be rescheduled, in which case this returns `.cancelledToBeRescheduled`. In that /// case the `TaskScheduler` is expected to call `execute` again. func execute() async -> ExecutionTaskFinishStatus { + if cancelledToBeRescheduled { + // `QueuedTask.execute` is called from a detached task in `TaskScheduler.poke` but we insert it into the + // `currentlyExecutingTasks` queue beforehand. This leaves a short windows in which we could cancel the task to + // reschedule it before it actually starts executing. + // If this happens, we don't have to do anything in `execute` and can immediately return. `execute` will be called + // again when the task gets rescheduled. + cancelledToBeRescheduled = false + return .cancelledToBeRescheduled + } precondition(executionTask == nil, "Task started twice") let task = Task.detached(priority: self.priority) { if !Task.isCancelled && !self.resultTaskCancelled.value { @@ -260,10 +269,10 @@ public actor QueuedTask { /// /// If the task has not been started yet or has already finished execution, this is a no-op. func cancelToBeRescheduled() { + self.cancelledToBeRescheduled = true guard let executionTask else { return } - self.cancelledToBeRescheduled = true executionTask.cancel() self.executionTask = nil } From 6599292ff5bf5ae0e07ef09ed3158dc185406df6 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 May 2024 16:44:08 -0700 Subject: [PATCH 30/69] Add infrastructure to write tests for Swift macros Add a default package manifest that defines two targets: - A macro target named `MyMacro` - And executable target named `MyMacroClient` It builds the macro using the swift-syntax that was already built as part of the SourceKit-LSP build. Re-using the SwiftSyntax modules that are already built is significantly faster than building swift-syntax in each test case run and does not require internet access. --- Package.swift | 4 + Sources/SKTestSupport/SkipUnless.swift | 67 +++++++++-- .../SKTestSupport/SwiftPMTestProject.swift | 110 ++++++++++++++++++ Sources/SKTestSupport/TestBundle.swift | 34 ++++++ .../BuildServerBuildSystemTests.swift | 21 ---- .../SwiftPMBuildSystemTests.swift | 47 ++++++++ 6 files changed, 251 insertions(+), 32 deletions(-) create mode 100644 Sources/SKTestSupport/TestBundle.swift diff --git a/Package.swift b/Package.swift index bc00da05d..6300efcbf 100644 --- a/Package.swift +++ b/Package.swift @@ -381,6 +381,10 @@ let package = Package( .product(name: "SwiftParser", package: "swift-syntax"), .product(name: "SwiftSyntax", package: "swift-syntax"), .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), + // Depend on `SwiftCompilerPlugin` and `SwiftSyntaxMacros` so the modules are built before running tests and can + // be used by test cases that test macros (see `SwiftPMTestProject.macroPackageManifest`). + .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), ] ), ] diff --git a/Sources/SKTestSupport/SkipUnless.swift b/Sources/SKTestSupport/SkipUnless.swift index d3248b1bc..9289c6fae 100644 --- a/Sources/SKTestSupport/SkipUnless.swift +++ b/Sources/SKTestSupport/SkipUnless.swift @@ -58,17 +58,11 @@ public actor SkipUnless { line: UInt, featureCheck: () async throws -> Bool ) async throws { - let checkResult: FeatureCheckResult - if let cachedResult = checkCache[featureName] { - checkResult = cachedResult - } else if ProcessEnv.block["SWIFTCI_USE_LOCAL_DEPS"] != nil { - // Never skip tests in CI. Toolchain should be up-to-date - checkResult = .featureSupported - } else { + return try await skipUnlessSupported(featureName: featureName, file: file, line: line) { let toolchainSwiftVersion = try await unwrap(ToolchainRegistry.forTesting.default).swiftVersion let requiredSwiftVersion = SwiftVersion(swiftVersion.major, swiftVersion.minor) if toolchainSwiftVersion < requiredSwiftVersion { - checkResult = .featureUnsupported( + return .featureUnsupported( skipMessage: """ Skipping because toolchain has Swift version \(toolchainSwiftVersion) \ but test requires at least \(requiredSwiftVersion) @@ -77,15 +71,32 @@ public actor SkipUnless { } else if toolchainSwiftVersion == requiredSwiftVersion { logger.info("Checking if feature '\(featureName)' is supported") if try await !featureCheck() { - checkResult = .featureUnsupported(skipMessage: "Skipping because toolchain doesn't contain \(featureName)") + return .featureUnsupported(skipMessage: "Skipping because toolchain doesn't contain \(featureName)") } else { - checkResult = .featureSupported + return .featureSupported } logger.info("Done checking if feature '\(featureName)' is supported") } else { - checkResult = .featureSupported + return .featureSupported } } + } + + private func skipUnlessSupported( + featureName: String = #function, + file: StaticString, + line: UInt, + featureCheck: () async throws -> FeatureCheckResult + ) async throws { + let checkResult: FeatureCheckResult + if let cachedResult = checkCache[featureName] { + checkResult = cachedResult + } else if ProcessEnv.block["SWIFTCI_USE_LOCAL_DEPS"] != nil { + // Never skip tests in CI. Toolchain should be up-to-date + checkResult = .featureSupported + } else { + checkResult = try await featureCheck() + } checkCache[featureName] = checkResult if case .featureUnsupported(let skipMessage) = checkResult { @@ -272,6 +283,40 @@ public actor SkipUnless { } #endif } + + /// Check if we can use the build artifacts in the sourcekit-lsp build directory to build a macro package without + /// re-building swift-syntax. + public static func canBuildMacroUsingSwiftSyntaxFromSourceKitLSPBuild( + file: StaticString = #filePath, + line: UInt = #line + ) async throws { + return try await shared.skipUnlessSupported(file: file, line: line) { + do { + let project = try await SwiftPMTestProject( + files: [ + "MyMacros/MyMacros.swift": #""" + import SwiftCompilerPlugin + import SwiftSyntax + import SwiftSyntaxBuilder + import SwiftSyntaxMacros + """#, + "MyMacroClient/MyMacroClient.swift": """ + """, + ], + manifest: SwiftPMTestProject.macroPackageManifest + ) + try await SwiftPMTestProject.build(at: project.scratchDirectory) + return .featureSupported + } catch { + return .featureUnsupported( + skipMessage: """ + Skipping because macro could not be built using build artifacts in the sourcekit-lsp build directory. \ + This usually happens if sourcekit-lsp was built using a different toolchain than the one used at test-time. + """ + ) + } + } + } } // MARK: - Parsing Swift compiler version diff --git a/Sources/SKTestSupport/SwiftPMTestProject.swift b/Sources/SKTestSupport/SwiftPMTestProject.swift index 4861ef954..e6aef9824 100644 --- a/Sources/SKTestSupport/SwiftPMTestProject.swift +++ b/Sources/SKTestSupport/SwiftPMTestProject.swift @@ -16,6 +16,8 @@ import LanguageServerProtocol import SourceKitLSP import TSCBasic +private struct SwiftSyntaxCShimsModulemapNotFoundError: Error {} + public class SwiftPMTestProject: MultiFileTestProject { enum Error: Swift.Error { /// The `swift` executable could not be found. @@ -33,6 +35,114 @@ public class SwiftPMTestProject: MultiFileTestProject { ) """ + /// A manifest that defines two targets: + /// - A macro target named `MyMacro` + /// - And executable target named `MyMacroClient` + /// + /// It builds the macro using the swift-syntax that was already built as part of the SourceKit-LSP build. + /// Re-using the SwiftSyntax modules that are already built is significantly faster than building swift-syntax in + /// each test case run and does not require internet access. + public static var macroPackageManifest: String { + get async throws { + // Directories that we should search for the swift-syntax package. + // We prefer a checkout in the build folder. If that doesn't exist, we are probably using local dependencies + // (SWIFTCI_USE_LOCAL_DEPS), so search next to the sourcekit-lsp source repo + let swiftSyntaxSearchPaths = [ + productsDirectory + .deletingLastPathComponent() // arm64-apple-macosx + .deletingLastPathComponent() // debug + .appendingPathComponent("checkouts"), + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // SwiftPMTestProject.swift + .deletingLastPathComponent() // SKTestSupport + .deletingLastPathComponent() // Sources + .deletingLastPathComponent(), // sourcekit-lsp + ] + + let swiftSyntaxCShimsModulemap = + swiftSyntaxSearchPaths.map { swiftSyntaxSearchPath in + swiftSyntaxSearchPath + .appendingPathComponent("swift-syntax") + .appendingPathComponent("Sources") + .appendingPathComponent("_SwiftSyntaxCShims") + .appendingPathComponent("include") + .appendingPathComponent("module.modulemap") + } + .first { FileManager.default.fileExists(atPath: $0.path) } + + guard let swiftSyntaxCShimsModulemap else { + throw SwiftSyntaxCShimsModulemapNotFoundError() + } + + let swiftSyntaxModulesToLink = [ + "SwiftBasicFormat", + "SwiftCompilerPlugin", + "SwiftCompilerPluginMessageHandling", + "SwiftDiagnostics", + "SwiftOperators", + "SwiftParser", + "SwiftParserDiagnostics", + "SwiftSyntax", + "SwiftSyntaxBuilder", + "SwiftSyntaxMacroExpansion", + "SwiftSyntaxMacros", + ] + + var objectFiles: [String] = [] + for moduleName in swiftSyntaxModulesToLink { + let dir = productsDirectory.appendingPathComponent("\(moduleName).build") + let enumerator = FileManager.default.enumerator(at: dir, includingPropertiesForKeys: nil) + while let file = enumerator?.nextObject() as? URL { + if file.pathExtension == "o" { + objectFiles.append(file.path) + } + } + } + + let linkerFlags = objectFiles.map { + """ + "-l", "\($0)", + """ + }.joined(separator: "\n") + + let moduleSearchPath: String + if let toolchainVersion = try await ToolchainRegistry.forTesting.default?.swiftVersion, + toolchainVersion < SwiftVersion(6, 0) + { + moduleSearchPath = productsDirectory.path + } else { + moduleSearchPath = "\(productsDirectory.path)/Modules" + } + + return """ + // swift-tools-version: 5.10 + + import PackageDescription + import CompilerPluginSupport + + let package = Package( + name: "MyMacro", + platforms: [.macOS(.v10_15)], + targets: [ + .macro( + name: "MyMacros", + swiftSettings: [.unsafeFlags([ + "-I", "\(moduleSearchPath)", + "-Xcc", "-fmodule-map-file=\(swiftSyntaxCShimsModulemap.path)" + ])], + linkerSettings: [ + .unsafeFlags([ + \(linkerFlags) + ]) + ] + ), + .executableTarget(name: "MyMacroClient", dependencies: ["MyMacros"]), + ] + ) + """ + } + } + /// Create a new SwiftPM package with the given files. /// /// If `index` is `true`, then the package will be built, indexing all modules within the package. diff --git a/Sources/SKTestSupport/TestBundle.swift b/Sources/SKTestSupport/TestBundle.swift new file mode 100644 index 000000000..e8351cecd --- /dev/null +++ b/Sources/SKTestSupport/TestBundle.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// The bundle of the currently executing test. +public let testBundle: Bundle = { + #if os(macOS) + if let bundle = Bundle.allBundles.first(where: { $0.bundlePath.hasSuffix(".xctest") }) { + return bundle + } + fatalError("couldn't find the test bundle") + #else + return Bundle.main + #endif +}() + +/// The path to the built products directory, ie. `.build/debug/arm64-apple-macosx` or the platform-specific equivalent. +public let productsDirectory: URL = { + #if os(macOS) + return testBundle.bundleURL.deletingLastPathComponent() + #else + return testBundle.bundleURL + #endif +}() diff --git a/Tests/SKCoreTests/BuildServerBuildSystemTests.swift b/Tests/SKCoreTests/BuildServerBuildSystemTests.swift index 26d1d19a3..934dcffaf 100644 --- a/Tests/SKCoreTests/BuildServerBuildSystemTests.swift +++ b/Tests/SKCoreTests/BuildServerBuildSystemTests.swift @@ -20,27 +20,6 @@ import SKTestSupport import TSCBasic import XCTest -/// The bundle of the currently executing test. -private let testBundle: Bundle = { - #if os(macOS) - if let bundle = Bundle.allBundles.first(where: { $0.bundlePath.hasSuffix(".xctest") }) { - return bundle - } - fatalError("couldn't find the test bundle") - #else - return Bundle.main - #endif -}() - -/// The path to the built products directory. -private let productsDirectory: URL = { - #if os(macOS) - return testBundle.bundleURL.deletingLastPathComponent() - #else - return testBundle.bundleURL - #endif -}() - /// The path to the INPUTS directory of shared test projects. private let skTestSupportInputsDirectory: URL = { #if os(macOS) diff --git a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift index 07fdcfe22..c1a426152 100644 --- a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift +++ b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift @@ -772,6 +772,53 @@ final class SwiftPMBuildSystemTests: XCTestCase { assertArgumentsContain(aswift.pathString, arguments: arguments) } } + + func testBuildMacro() async throws { + try await SkipUnless.canBuildMacroUsingSwiftSyntaxFromSourceKitLSPBuild() + // This test is just a dummy to show how to create a `SwiftPMTestProject` that builds a macro using the SwiftSyntax + // modules that were already built during the build of SourceKit-LSP. + // It should be removed once we have a real test that tests macros (like macro expansion). + let project = try await SwiftPMTestProject( + files: [ + "MyMacros/MyMacros.swift": #""" + import SwiftCompilerPlugin + import SwiftSyntax + import SwiftSyntaxBuilder + import SwiftSyntaxMacros + + public struct StringifyMacro: ExpressionMacro { + public static func expansion( + of node: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) -> ExprSyntax { + guard let argument = node.argumentList.first?.expression else { + fatalError("compiler bug: the macro does not have any arguments") + } + + return "(\(argument), \(literal: argument.description))" + } + } + + @main + struct MyMacroPlugin: CompilerPlugin { + let providingMacros: [Macro.Type] = [ + StringifyMacro.self, + ] + } + """#, + "MyMacroClient/MyMacroClient.swift": """ + @freestanding(expression) + public macro stringify(_ value: T) -> (T, String) = #externalMacro(module: "MyMacros", type: "StringifyMacro") + + func test() { + #stringify(1 + 2) + } + """, + ], + manifest: SwiftPMTestProject.macroPackageManifest + ) + try await SwiftPMTestProject.build(at: project.scratchDirectory) + } } private func assertArgumentsDoNotContain( From e56c71f4b31fbbe0b62fe5c00362878ec3f1c6dc Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 07:38:08 -0700 Subject: [PATCH 31/69] =?UTF-8?q?Don=E2=80=99t=20run=20a=20`swift=20build`?= =?UTF-8?q?=20command=20to=20prepare=20a=20package=20manifest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Package manifests don’t have an associated target to prepare and are represented by a `ConfiguredTarget` with an empty target ID. We were mistakingly running `swift build` with an empty target name to prepare them, which failed. There is nothing to prepare. --- .../SwiftPMBuildSystem.swift | 5 +++ .../TestSourceKitLSPClient.swift | 43 ++++++++++++++----- .../SemanticIndex/SemanticIndexManager.swift | 14 ++++-- .../SourceKitDTests/CrashRecoveryTests.swift | 4 +- .../BackgroundIndexingTests.swift | 23 ++++++++++ .../SourceKitLSPTests/BuildSystemTests.swift | 2 +- Tests/SourceKitLSPTests/LocalSwiftTests.swift | 2 +- .../MainFilesProviderTests.swift | 2 +- 8 files changed, 76 insertions(+), 19 deletions(-) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index 19bf8a197..c3a24b451 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -550,6 +550,11 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { singleTarget target: ConfiguredTarget, indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void ) async throws { + if target == .forPackageManifest { + // Nothing to prepare for package manifests. + return + } + // TODO (indexing): Add a proper 'prepare' job in SwiftPM instead of building the target. // https://github.com/apple/sourcekit-lsp/issues/1254 guard let toolchain = await toolchainRegistry.default else { diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index 83915ad10..80d07ea51 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -27,6 +27,10 @@ extension SourceKitLSPServer.Options { public static let testDefault = Self(swiftPublishDiagnosticsDebounceDuration: 0) } +fileprivate struct NotificationTimeoutError: Error, CustomStringConvertible { + var description: String = "Failed to receive next notification within timeout" +} + /// A mock SourceKit-LSP client (aka. a mock editor) that behaves like an editor /// for testing purposes. /// @@ -229,21 +233,17 @@ public final class TestSourceKitLSPClient: MessageHandler { /// /// - Note: This also returns any notifications sent before the call to /// `nextNotification`. - public func nextNotification(timeout: TimeInterval = defaultTimeout) async throws -> any NotificationType { - struct TimeoutError: Error, CustomStringConvertible { - var description: String = "Failed to receive next notification within timeout" - } - + public func nextNotification(timeout: Duration = .seconds(defaultTimeout)) async throws -> any NotificationType { return try await withThrowingTaskGroup(of: (any NotificationType).self) { taskGroup in taskGroup.addTask { for await notification in self.notifications { return notification } - throw TimeoutError() + throw NotificationTimeoutError() } taskGroup.addTask { - try await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000)) - throw TimeoutError() + try await Task.sleep(for: timeout) + throw NotificationTimeoutError() } let result = try await taskGroup.next()! taskGroup.cancelAll() @@ -256,7 +256,7 @@ public final class TestSourceKitLSPClient: MessageHandler { /// If the next notification is not a `PublishDiagnosticsNotification`, this /// methods throws. public func nextDiagnosticsNotification( - timeout: TimeInterval = defaultTimeout + timeout: Duration = .seconds(defaultTimeout) ) async throws -> PublishDiagnosticsNotification { guard !usePullDiagnostics else { struct PushDiagnosticsError: Error, CustomStringConvertible { @@ -272,7 +272,7 @@ public final class TestSourceKitLSPClient: MessageHandler { public func nextNotification( ofType: ExpectedNotificationType.Type, satisfying predicate: (ExpectedNotificationType) -> Bool = { _ in true }, - timeout: TimeInterval = defaultTimeout + timeout: Duration = .seconds(defaultTimeout) ) async throws -> ExpectedNotificationType { while true { let nextNotification = try await nextNotification(timeout: timeout) @@ -282,6 +282,29 @@ public final class TestSourceKitLSPClient: MessageHandler { } } + /// Asserts that the test client does not receive a notification of the given type and satisfying the given predicate + /// within the given duration. + /// + /// For stable tests, the code that triggered the notification should be run before this assertion instead of relying + /// on the duration. + /// + /// The duration should not be 0 because we need to allow `nextNotification` some time to get the notification out of + /// the `notifications` `AsyncStream`. + public func assertDoesNotReceiveNotification( + ofType: ExpectedNotificationType.Type, + satisfying predicate: (ExpectedNotificationType) -> Bool = { _ in true }, + within duration: Duration = .seconds(0.2) + ) async throws { + do { + let notification = try await nextNotification( + ofType: ExpectedNotificationType.self, + satisfying: predicate, + timeout: duration + ) + XCTFail("Did not expect to receive notification but received \(notification)") + } catch is NotificationTimeoutError {} + } + /// Handle the next request of the given type that is sent to the client. /// /// The request handler will only handle a single request. If the request is called again, the request handler won't diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index 50bebba9e..dc2b663c0 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -394,6 +394,12 @@ public final actor SemanticIndexManager { let id = UUID() let task = Task(priority: priority) { await withLoggingScope("preparation") { + defer { + if inProgressPrepareForEditorTask?.id == id { + inProgressPrepareForEditorTask = nil + self.indexProgressStatusDidChange() + } + } // Should be kept in sync with `prepareFileForEditorFunctionality` guard let target = await buildSystemManager.canonicalConfiguredTarget(for: uri) else { return @@ -401,6 +407,10 @@ public final actor SemanticIndexManager { if Task.isCancelled { return } + if await preparationUpToDateTracker.isUpToDate(target) { + // If the target is up-to-date, there is nothing to prepare. + return + } if inProgressPrepareForEditorTask?.id == id { if inProgressPrepareForEditorTask?.state != .determiningCanonicalConfiguredTarget { logger.fault("inProgressPrepareForEditorTask is in unexpected state") @@ -408,10 +418,6 @@ public final actor SemanticIndexManager { inProgressPrepareForEditorTask?.state = .preparingTarget } await self.prepare(targets: [target], priority: nil) - if inProgressPrepareForEditorTask?.id == id { - inProgressPrepareForEditorTask = nil - self.indexProgressStatusDidChange() - } } } inProgressPrepareForEditorTask?.task.cancel() diff --git a/Tests/SourceKitDTests/CrashRecoveryTests.swift b/Tests/SourceKitDTests/CrashRecoveryTests.swift index eba61bf47..339387b54 100644 --- a/Tests/SourceKitDTests/CrashRecoveryTests.swift +++ b/Tests/SourceKitDTests/CrashRecoveryTests.swift @@ -90,7 +90,7 @@ final class CrashRecoveryTests: XCTestCase { await swiftLanguageService._crash() - let crashedNotification = try await testClient.nextNotification(ofType: WorkDoneProgress.self, timeout: 5) + let crashedNotification = try await testClient.nextNotification(ofType: WorkDoneProgress.self, timeout: .seconds(5)) XCTAssertEqual( crashedNotification.value, .begin( @@ -107,7 +107,7 @@ final class CrashRecoveryTests: XCTestCase { _ = try? await testClient.send(hoverRequest) let semanticFunctionalityRestoredNotification = try await testClient.nextNotification( ofType: WorkDoneProgress.self, - timeout: 30 + timeout: .seconds(30) ) XCTAssertEqual(semanticFunctionalityRestoredNotification.value, .end(WorkDoneProgressEnd())) diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index ada1ee3fb..215a48c5b 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -930,4 +930,27 @@ final class BackgroundIndexingTests: XCTestCase { ) XCTAssertEqual((result?.changes?.keys).map(Set.init), [uri, try project.uri(for: "Client.swift")]) } + + func testDontPreparePackageManifest() async throws { + let project = try await SwiftPMTestProject( + files: [ + "Lib.swift": "" + ], + enableBackgroundIndexing: true + ) + + _ = try await project.testClient.nextNotification( + ofType: LogMessageNotification.self, + satisfying: { $0.message.contains("Preparing MyLibrary") } + ) + + // Opening the package manifest shouldn't cause any `swift build` calls to prepare them because they are not part of + // a target that can be prepared. + let (uri, _) = try project.openDocument("Package.swift") + _ = try await project.testClient.send(DocumentDiagnosticsRequest(textDocument: TextDocumentIdentifier(uri))) + try await project.testClient.assertDoesNotReceiveNotification( + ofType: LogMessageNotification.self, + satisfying: { $0.message.contains("Preparing") } + ) + } } diff --git a/Tests/SourceKitLSPTests/BuildSystemTests.swift b/Tests/SourceKitLSPTests/BuildSystemTests.swift index eec906818..44c4031e9 100644 --- a/Tests/SourceKitLSPTests/BuildSystemTests.swift +++ b/Tests/SourceKitLSPTests/BuildSystemTests.swift @@ -193,7 +193,7 @@ final class BuildSystemTests: XCTestCase { var receivedCorrectDiagnostic = false for _ in 0.. Date: Fri, 31 May 2024 22:02:58 -0700 Subject: [PATCH 32/69] Forward all errors that translate to `.cancelled` from document diagnostics request We were only forwarding `CancellationError` but depending on when cancellation happens, we would get `SKDError.cancelled`, which we would translate to empty diagnostics, causing the test to fail. --- Sources/SourceKitLSP/Swift/SwiftLanguageService.swift | 7 +++++-- Tests/SourceKitLSPTests/PullDiagnosticsTests.swift | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index 1f4dbae92..e488264e0 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -888,12 +888,15 @@ extension SwiftLanguageService { buildSettings: buildSettings ) return .full(diagnosticReport) - } catch let error as CancellationError { - throw error } catch { // VS Code does not request diagnostics again for a document if the diagnostics request failed. // Since sourcekit-lsp usually recovers from failures (e.g. after sourcekitd crashes), this is undesirable. // Instead of returning an error, return empty results. + // Do forward cancellation because we don't want to clear diagnostics in the client if they cancel the diagnostic + // request. + if ResponseError(error) == .cancelled { + throw error + } logger.error( """ Loading diagnostic failed with the following error. Returning empty diagnostics. diff --git a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift index 110d3182b..b22c91d4c 100644 --- a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift @@ -337,7 +337,7 @@ final class PullDiagnosticsTests: XCTestCase { let requestID = project.testClient.send( DocumentDiagnosticsRequest(textDocument: TextDocumentIdentifier(uri)) ) { result in - XCTAssertEqual(result.failure?.code, .cancelled) + XCTAssertEqual(result, .failure(ResponseError.cancelled)) diagnosticResponseReceived.fulfill() } project.testClient.send(CancelRequestNotification(id: requestID)) From 20840c418a6475baeb5a5dd4938b0cce4f985eda Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 22:04:57 -0700 Subject: [PATCH 33/69] =?UTF-8?q?Skip=20testImportPreparedModuleWithFuncti?= =?UTF-8?q?onBodiesSkipped=20if=20toolchain=20doesn=E2=80=99t=20support=20?= =?UTF-8?q?rename?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test uses rename and should thus be skipped if sourcekitd doesn’t support rename. --- Tests/SourceKitLSPTests/BackgroundIndexingTests.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index ada1ee3fb..13004f8a2 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -889,6 +889,7 @@ final class BackgroundIndexingTests: XCTestCase { } func testImportPreparedModuleWithFunctionBodiesSkipped() async throws { + try await SkipUnless.sourcekitdSupportsRename() // This test case was crashing the indexing compiler invocation for Client if Lib was built for index preparation // (using `-enable-library-evolution -experimental-skip-all-function-bodies -experimental-lazy-typecheck`) but x // Client was not indexed with `-experimental-allow-module-with-compiler-errors`. rdar://129071600 From 4524e6b715de83d387ccd740ea5776e3f0cdd979 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 22:07:04 -0700 Subject: [PATCH 34/69] Set `SOURCEKITLSP_FORCE_NON_DARWIN_LOGGER` for parallel test execution Set `SOURCEKITLSP_FORCE_NON_DARWIN_LOGGER` to 1 so we generate a log to stderr if parallel test execution fails, which helps diagnose the issue. --- .vscode/tasks.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index c7792d33d..bfb4c5c29 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -10,6 +10,11 @@ "disableTaskQueue": true, "group": "test", "label": "Run all tests (parallel)", + "options": { + "env": { + "SOURCEKITLSP_FORCE_NON_DARWIN_LOGGER": "1" + } + }, "problemMatcher": [ "$swiftc" ], @@ -30,7 +35,8 @@ "label": "Run fast tests (parallel)", "options": { "env": { - "SKIP_LONG_TESTS": "1" + "SKIP_LONG_TESTS": "1", + "SOURCEKITLSP_FORCE_NON_DARWIN_LOGGER": "1" } }, "problemMatcher": [ From eb304c4759489c523bf5c30ec99a9f641c7a8bef Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 08:35:31 -0700 Subject: [PATCH 35/69] Add a general notion of experimental features to sourcekit-lsp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Background indexing probably won’t be the last experimental feature in sourcekit-lsp that we want to gate behind a feature flag. Instead of adding new parameters ad-hoc, introduce a general notion of experimental features. --- Sources/Diagnose/IndexCommand.swift | 2 +- .../TestSourceKitLSPClient.swift | 4 ++- Sources/SourceKitLSP/CMakeLists.txt | 1 + Sources/SourceKitLSP/CreateBuildSystem.swift | 2 +- .../SourceKitLSP/ExperimentalFeatures.swift | 26 +++++++++++++++++++ .../SourceKitLSP/IndexProgressManager.swift | 2 +- .../SourceKitLSPServer+Options.swift | 5 ++++ Sources/SourceKitLSP/SourceKitLSPServer.swift | 2 +- Sources/SourceKitLSP/Workspace.swift | 19 ++------------ Sources/sourcekit-lsp/SourceKitLSP.swift | 22 ++++++++-------- 10 files changed, 52 insertions(+), 33 deletions(-) create mode 100644 Sources/SourceKitLSP/ExperimentalFeatures.swift diff --git a/Sources/Diagnose/IndexCommand.swift b/Sources/Diagnose/IndexCommand.swift index a00872d29..9923a922d 100644 --- a/Sources/Diagnose/IndexCommand.swift +++ b/Sources/Diagnose/IndexCommand.swift @@ -75,7 +75,7 @@ public struct IndexCommand: AsyncParsableCommand { public func run() async throws { var serverOptions = SourceKitLSPServer.Options() - serverOptions.indexOptions.enableBackgroundIndexing = true + serverOptions.experimentalFeatures.append(.backgroundIndexing) let installPath = if let toolchainOverride, let toolchain = Toolchain(try AbsolutePath(validating: toolchainOverride)) { diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index 83915ad10..71f57c798 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -117,7 +117,9 @@ public final class TestSourceKitLSPClient: MessageHandler { if let moduleCache { serverOptions.buildSetup.flags.swiftCompilerFlags += ["-module-cache-path", moduleCache.path] } - serverOptions.indexOptions.enableBackgroundIndexing = enableBackgroundIndexing + if enableBackgroundIndexing { + serverOptions.experimentalFeatures.append(.backgroundIndexing) + } var notificationYielder: AsyncStream.Continuation! self.notifications = AsyncStream { continuation in diff --git a/Sources/SourceKitLSP/CMakeLists.txt b/Sources/SourceKitLSP/CMakeLists.txt index a76fd4faf..875c3d4e5 100644 --- a/Sources/SourceKitLSP/CMakeLists.txt +++ b/Sources/SourceKitLSP/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(SourceKitLSP STATIC CreateBuildSystem.swift DocumentManager.swift DocumentSnapshot+FromFileContents.swift + ExperimentalFeatures.swift IndexProgressManager.swift IndexStoreDB+MainFilesProvider.swift LanguageServerType.swift diff --git a/Sources/SourceKitLSP/CreateBuildSystem.swift b/Sources/SourceKitLSP/CreateBuildSystem.swift index 9eb963419..4b8d0b375 100644 --- a/Sources/SourceKitLSP/CreateBuildSystem.swift +++ b/Sources/SourceKitLSP/CreateBuildSystem.swift @@ -38,7 +38,7 @@ func createBuildSystem( url: rootUrl, toolchainRegistry: toolchainRegistry, buildSetup: options.buildSetup, - isForIndexBuild: options.indexOptions.enableBackgroundIndexing, + isForIndexBuild: options.experimentalFeatures.contains(.backgroundIndexing), reloadPackageStatusCallback: reloadPackageStatusCallback ) } diff --git a/Sources/SourceKitLSP/ExperimentalFeatures.swift b/Sources/SourceKitLSP/ExperimentalFeatures.swift new file mode 100644 index 000000000..ae0a677fd --- /dev/null +++ b/Sources/SourceKitLSP/ExperimentalFeatures.swift @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// An experimental feature that can be enabled by passing `--experimental-feature` to `sourcekit-lsp` on the command +/// line. The raw value of this feature is how it is named on the command line. +public enum ExperimentalFeature: String, Codable, Sendable, CaseIterable { + /// Enable background indexing. + case backgroundIndexing = "background-indexing" + + /// Show the files that are currently being indexed / the targets that are currently being prepared in the work done + /// progress. + /// + /// This is an option because VS Code tries to render a multi-line work done progress into a single line text field in + /// the status bar, which looks broken. But at the same time, it is very useful to get a feeling about what's + /// currently happening indexing-wise. + case showActivePreparationTasksInProgress = "show-active-preparation-tasks-in-progress" +} diff --git a/Sources/SourceKitLSP/IndexProgressManager.swift b/Sources/SourceKitLSP/IndexProgressManager.swift index 72d53ffa9..17ae4599d 100644 --- a/Sources/SourceKitLSP/IndexProgressManager.swift +++ b/Sources/SourceKitLSP/IndexProgressManager.swift @@ -99,7 +99,7 @@ actor IndexProgressManager { // Clip the finished tasks to 0 because showing a negative number there looks stupid. let finishedTasks = max(queuedIndexTasks - indexTasks.count, 0) message = "\(finishedTasks) / \(queuedIndexTasks)" - if await sourceKitLSPServer.options.indexOptions.showActivePreparationTasksInProgress { + if await sourceKitLSPServer.options.experimentalFeatures.contains(.showActivePreparationTasksInProgress) { var inProgressTasks: [String] = [] inProgressTasks += preparationTasks.filter { $0.value == .executing } .map { "- Preparing \($0.key.targetID)" } diff --git a/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift b/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift index 402013d20..78dc82d02 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift @@ -49,6 +49,9 @@ extension SourceKitLSPServer { /// notification when running unit tests. public var swiftPublishDiagnosticsDebounceDuration: TimeInterval + /// Experimental features that are enabled. + public var experimentalFeatures: [ExperimentalFeature] + public var indexTestHooks: IndexTestHooks public init( @@ -59,6 +62,7 @@ extension SourceKitLSPServer { completionOptions: SKCompletionOptions = .init(), generatedInterfacesPath: AbsolutePath = defaultDirectoryForGeneratedInterfaces, swiftPublishDiagnosticsDebounceDuration: TimeInterval = 2, /* 2s */ + experimentalFeatures: [ExperimentalFeature] = [], indexTestHooks: IndexTestHooks = IndexTestHooks() ) { self.buildSetup = buildSetup @@ -68,6 +72,7 @@ extension SourceKitLSPServer { self.completionOptions = completionOptions self.generatedInterfacesPath = generatedInterfacesPath self.swiftPublishDiagnosticsDebounceDuration = swiftPublishDiagnosticsDebounceDuration + self.experimentalFeatures = experimentalFeatures self.indexTestHooks = indexTestHooks } } diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index e883345c7..3d251f7ae 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -941,7 +941,7 @@ extension SourceKitLSPServer { self?.indexProgressManager.indexProgressStatusDidChange() } ) - if let workspace, options.indexOptions.enableBackgroundIndexing, workspace.semanticIndexManager == nil, + if let workspace, options.experimentalFeatures.contains(.backgroundIndexing), workspace.semanticIndexManager == nil, !self.didSendBackgroundIndexingNotSupportedNotification { self.sendNotificationToClient( diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index 9d7e999d3..43672de55 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -108,7 +108,7 @@ public final class Workspace: Sendable { mainFilesProvider: uncheckedIndex, toolchainRegistry: toolchainRegistry ) - if options.indexOptions.enableBackgroundIndexing, + if options.experimentalFeatures.contains(.backgroundIndexing), let uncheckedIndex, await buildSystemManager.supportsPreparation { @@ -247,37 +247,22 @@ public struct IndexOptions: Sendable { /// explicit calls to pollForUnitChangesAndWait(). public var listenToUnitEvents: Bool - /// Whether background indexing should be enabled. - public var enableBackgroundIndexing: Bool - /// The percentage of the machine's cores that should at most be used for background indexing. /// /// Setting this to a value < 1 ensures that background indexing doesn't use all CPU resources. public var maxCoresPercentageToUseForBackgroundIndexing: Double - /// Whether to show the files that are currently being indexed / the targets that are currently being prepared in the - /// work done progress. - /// - /// This is an option because VS Code tries to render a multi-line work done progress into a single line text field in - /// the status bar, which looks broken. But at the same time, it is very useful to get a feeling about what's - /// currently happening indexing-wise. - public var showActivePreparationTasksInProgress: Bool - public init( indexStorePath: AbsolutePath? = nil, indexDatabasePath: AbsolutePath? = nil, indexPrefixMappings: [PathPrefixMapping]? = nil, listenToUnitEvents: Bool = true, - enableBackgroundIndexing: Bool = false, - maxCoresPercentageToUseForBackgroundIndexing: Double = 1, - showActivePreparationTasksInProgress: Bool = false + maxCoresPercentageToUseForBackgroundIndexing: Double = 1 ) { self.indexStorePath = indexStorePath self.indexDatabasePath = indexDatabasePath self.indexPrefixMappings = indexPrefixMappings self.listenToUnitEvents = listenToUnitEvents - self.enableBackgroundIndexing = enableBackgroundIndexing self.maxCoresPercentageToUseForBackgroundIndexing = maxCoresPercentageToUseForBackgroundIndexing - self.showActivePreparationTasksInProgress = showActivePreparationTasksInProgress } } diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index 0cece17c0..467be6b8c 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -201,18 +201,14 @@ struct SourceKitLSP: AsyncParsableCommand { ) var completionMaxResults = 200 - @Flag( - help: "Enable background indexing. This feature is still under active development and may be incomplete." - ) - var experimentalEnableBackgroundIndexing = false - - @Flag( + @Option( + name: .customLong("experimental-feature"), help: """ - When reporting index progress, show the currently running index tasks in addition to the task's count. \ - This produces a multi-line work done progress, which might render incorrectly, depending on the editor. + Enable an experimental sourcekit-lsp feature. + Available features are: \(ExperimentalFeature.allCases.map(\.rawValue).joined(separator: ", ")) """ ) - var experimentalShowActivePreparationTasksInProgress = false + var experimentalFeatures: [ExperimentalFeature] func mapOptions() -> SourceKitLSPServer.Options { var serverOptions = SourceKitLSPServer.Options() @@ -229,8 +225,6 @@ struct SourceKitLSP: AsyncParsableCommand { serverOptions.indexOptions.indexStorePath = indexStorePath serverOptions.indexOptions.indexDatabasePath = indexDatabasePath serverOptions.indexOptions.indexPrefixMappings = indexPrefixMappings - serverOptions.indexOptions.enableBackgroundIndexing = experimentalEnableBackgroundIndexing - serverOptions.indexOptions.showActivePreparationTasksInProgress = experimentalShowActivePreparationTasksInProgress serverOptions.completionOptions.maxResults = completionMaxResults serverOptions.generatedInterfacesPath = generatedInterfacesPath @@ -285,3 +279,9 @@ struct SourceKitLSP: AsyncParsableCommand { try await Task.sleep(for: .seconds(60 * 60 * 24 * 365 * 10)) } } + +#if compiler(>=6) +extension ExperimentalFeature: @retroactive ExpressibleByArgument {} +#else +extension ExperimentalFeature: ExpressibleByArgument {} +#endif From 4b2ee40a522a1fed98d442fe15e57a7f03f24b3b Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 10:01:27 -0700 Subject: [PATCH 36/69] Automatically add `swift-tools-version` and `import PackageDescription` to manifest in SwiftPMTestProject This shortens the tests and allows them to focus on the important parts. rdar://126484621 --- .../SKTestSupport/SwiftPMTestProject.swift | 11 ++++++++ .../BackgroundIndexingTests.swift | 26 ------------------- .../CrossLanguageRenameTests.swift | 16 ------------ Tests/SourceKitLSPTests/DefinitionTests.swift | 16 ------------ .../DependencyTrackingTests.swift | 4 --- .../DocumentTestDiscoveryTests.swift | 16 ------------ Tests/SourceKitLSPTests/IndexTests.swift | 4 --- .../MainFilesProviderTests.swift | 16 ------------ .../PublishDiagnosticsTests.swift | 4 --- .../PullDiagnosticsTests.swift | 8 ------ Tests/SourceKitLSPTests/RenameTests.swift | 12 --------- .../SwiftInterfaceTests.swift | 8 ------ .../WorkspaceTestDiscoveryTests.swift | 16 ------------ Tests/SourceKitLSPTests/WorkspaceTests.swift | 4 --- 14 files changed, 11 insertions(+), 150 deletions(-) diff --git a/Sources/SKTestSupport/SwiftPMTestProject.swift b/Sources/SKTestSupport/SwiftPMTestProject.swift index e6aef9824..ba9dae445 100644 --- a/Sources/SKTestSupport/SwiftPMTestProject.swift +++ b/Sources/SKTestSupport/SwiftPMTestProject.swift @@ -173,6 +173,17 @@ public class SwiftPMTestProject: MultiFileTestProject { filesByPath[RelativeFileLocation(directories: directories, fileLocation.fileName)] = contents } + var manifest = manifest + if !manifest.contains("swift-tools-version") { + // Tests specify a shorthand package manifest that doesn't contain the tools version boilerplate. + manifest = """ + // swift-tools-version: 5.7 + + import PackageDescription + + \(manifest) + """ + } filesByPath["Package.swift"] = manifest try await super.init( diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index 13004f8a2..71cae20fb 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -117,10 +117,6 @@ final class BackgroundIndexingTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -181,10 +177,6 @@ final class BackgroundIndexingTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -231,8 +223,6 @@ final class BackgroundIndexingTests: XCTestCase { """ ], manifest: """ - // swift-tools-version: 5.7 - import PackageDescription let package = Package( name: "MyLibrary", dependencies: [.package(url: "\(dependencyProject.packageDirectory)", from: "1.0.0")], @@ -562,10 +552,6 @@ final class BackgroundIndexingTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -686,10 +672,6 @@ final class BackgroundIndexingTests: XCTestCase { "LibD/LibD.swift": "", ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -801,10 +783,6 @@ final class BackgroundIndexingTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -909,10 +887,6 @@ final class BackgroundIndexingTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift b/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift index 4a3bba952..e49225f32 100644 --- a/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift +++ b/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift @@ -14,10 +14,6 @@ import SKTestSupport import XCTest private let libAlibBPackageManifest = """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -564,10 +560,6 @@ final class CrossLanguageRenameTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -622,10 +614,6 @@ final class CrossLanguageRenameTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -680,10 +668,6 @@ final class CrossLanguageRenameTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/DefinitionTests.swift b/Tests/SourceKitLSPTests/DefinitionTests.swift index 22a162737..5fdfc3b8d 100644 --- a/Tests/SourceKitLSPTests/DefinitionTests.swift +++ b/Tests/SourceKitLSPTests/DefinitionTests.swift @@ -165,10 +165,6 @@ class DefinitionTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -272,10 +268,6 @@ class DefinitionTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -323,10 +315,6 @@ class DefinitionTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -452,10 +440,6 @@ class DefinitionTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift index 5262ef07c..95420816b 100644 --- a/Tests/SourceKitLSPTests/DependencyTrackingTests.swift +++ b/Tests/SourceKitLSPTests/DependencyTrackingTests.swift @@ -29,10 +29,6 @@ final class DependencyTrackingTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift b/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift index 4655e4e6d..a4bd75779 100644 --- a/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift +++ b/Tests/SourceKitLSPTests/DocumentTestDiscoveryTests.swift @@ -39,10 +39,6 @@ final class DocumentTestDiscoveryTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [.testTarget(name: "MyLibraryTests")] @@ -142,10 +138,6 @@ final class DocumentTestDiscoveryTests: XCTestCase { """ ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [.testTarget(name: "MyLibraryTests")] @@ -1295,10 +1287,6 @@ final class DocumentTestDiscoveryTests: XCTestCase { """ ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [.testTarget(name: "MyLibraryTests")] @@ -1354,10 +1342,6 @@ final class DocumentTestDiscoveryTests: XCTestCase { """ ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [.testTarget(name: "MyLibraryTests")] diff --git a/Tests/SourceKitLSPTests/IndexTests.swift b/Tests/SourceKitLSPTests/IndexTests.swift index 52be924a0..c66f4facb 100644 --- a/Tests/SourceKitLSPTests/IndexTests.swift +++ b/Tests/SourceKitLSPTests/IndexTests.swift @@ -35,10 +35,6 @@ final class IndexTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/MainFilesProviderTests.swift b/Tests/SourceKitLSPTests/MainFilesProviderTests.swift index 779c8c05a..c7184b955 100644 --- a/Tests/SourceKitLSPTests/MainFilesProviderTests.swift +++ b/Tests/SourceKitLSPTests/MainFilesProviderTests.swift @@ -31,10 +31,6 @@ final class MainFilesProviderTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -71,10 +67,6 @@ final class MainFilesProviderTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -123,10 +115,6 @@ final class MainFilesProviderTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -172,10 +160,6 @@ final class MainFilesProviderTests: XCTestCase { "Sources/MyFancyLibrary/MyFancyLibrary.c": "", ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift index 8535ca5cd..a0659d23c 100644 --- a/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PublishDiagnosticsTests.swift @@ -169,10 +169,6 @@ final class PublishDiagnosticsTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift index b22c91d4c..8641f3e86 100644 --- a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift @@ -200,10 +200,6 @@ final class PullDiagnosticsTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -287,10 +283,6 @@ final class PullDiagnosticsTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/RenameTests.swift b/Tests/SourceKitLSPTests/RenameTests.swift index 5934c8d7a..76538e034 100644 --- a/Tests/SourceKitLSPTests/RenameTests.swift +++ b/Tests/SourceKitLSPTests/RenameTests.swift @@ -536,10 +536,6 @@ final class RenameTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -576,10 +572,6 @@ final class RenameTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -627,10 +619,6 @@ final class RenameTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift index 0975525ac..63f055592 100644 --- a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift +++ b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift @@ -66,10 +66,6 @@ final class SwiftInterfaceTests: XCTestCase { "Exec/main.swift": "import MyLibrary", ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ @@ -162,10 +158,6 @@ final class SwiftInterfaceTests: XCTestCase { "Exec/main.swift": "import 1️⃣MyLibrary", ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ diff --git a/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift b/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift index 8a2c3ef2d..9cf2911e4 100644 --- a/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift @@ -19,10 +19,6 @@ import SKTestSupport import XCTest private let packageManifestWithTestTarget = """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [.testTarget(name: "MyLibraryTests")] @@ -844,10 +840,6 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { "Test.swift": "" ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", dependencies: [.package(url: "\(dependencyProject.packageDirectory)", from: "1.0.0")], @@ -925,10 +917,6 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """ ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [.testTarget(name: "MyLibraryTests")] @@ -982,10 +970,6 @@ final class WorkspaceTestDiscoveryTests: XCTestCase { """ ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [.testTarget(name: "MyLibraryTests")] diff --git a/Tests/SourceKitLSPTests/WorkspaceTests.swift b/Tests/SourceKitLSPTests/WorkspaceTests.swift index fadebea3e..f487b6d36 100644 --- a/Tests/SourceKitLSPTests/WorkspaceTests.swift +++ b/Tests/SourceKitLSPTests/WorkspaceTests.swift @@ -568,10 +568,6 @@ final class WorkspaceTests: XCTestCase { """, ], manifest: """ - // swift-tools-version: 5.7 - - import PackageDescription - let package = Package( name: "MyLibrary", targets: [ From 113bd0dbec79e0e53c7cdd81b7b566ded13da6e2 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 10:23:32 -0700 Subject: [PATCH 37/69] Move syntactic XCTest scanner to a separate file The swift-testing scanner is in its own file and the XCTest scanner should be as well, for consistency. rdar://126529507 --- Sources/SourceKitLSP/CMakeLists.txt | 1 + Sources/SourceKitLSP/LanguageService.swift | 16 +++ .../Swift/SyntacticSwiftXCTestScanner.swift | 125 +++++++++++++++++ Sources/SourceKitLSP/TestDiscovery.swift | 129 +----------------- 4 files changed, 143 insertions(+), 128 deletions(-) create mode 100644 Sources/SourceKitLSP/Swift/SyntacticSwiftXCTestScanner.swift diff --git a/Sources/SourceKitLSP/CMakeLists.txt b/Sources/SourceKitLSP/CMakeLists.txt index a76fd4faf..7efb356a8 100644 --- a/Sources/SourceKitLSP/CMakeLists.txt +++ b/Sources/SourceKitLSP/CMakeLists.txt @@ -53,6 +53,7 @@ target_sources(SourceKitLSP PRIVATE Swift/SwiftLanguageService.swift Swift/SwiftTestingScanner.swift Swift/SymbolInfo.swift + Swift/SyntacticSwiftXCTestScanner.swift Swift/SyntacticTestIndex.swift Swift/SyntaxHighlightingToken.swift Swift/SyntaxHighlightingTokenParser.swift diff --git a/Sources/SourceKitLSP/LanguageService.swift b/Sources/SourceKitLSP/LanguageService.swift index 856c929fa..0b0249e5e 100644 --- a/Sources/SourceKitLSP/LanguageService.swift +++ b/Sources/SourceKitLSP/LanguageService.swift @@ -24,6 +24,22 @@ public enum LanguageServerState { case semanticFunctionalityDisabled } +public struct AnnotatedTestItem: Sendable { + /// The test item to be annotated + public var testItem: TestItem + + /// Whether the `TestItem` is an extension. + public var isExtension: Bool + + public init( + testItem: TestItem, + isExtension: Bool + ) { + self.testItem = testItem + self.isExtension = isExtension + } +} + public struct RenameLocation: Sendable { /// How the identifier at a given location is being used. /// diff --git a/Sources/SourceKitLSP/Swift/SyntacticSwiftXCTestScanner.swift b/Sources/SourceKitLSP/Swift/SyntacticSwiftXCTestScanner.swift new file mode 100644 index 000000000..bb5363f62 --- /dev/null +++ b/Sources/SourceKitLSP/Swift/SyntacticSwiftXCTestScanner.swift @@ -0,0 +1,125 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import LanguageServerProtocol +import SwiftSyntax + +/// Scans a source file for `XCTestCase` classes and test methods. +/// +/// The syntax visitor scans from class and extension declarations that could be `XCTestCase` classes or extensions +/// thereof. It then calls into `findTestMethods` to find the actual test methods. +final class SyntacticSwiftXCTestScanner: SyntaxVisitor { + /// The document snapshot of the syntax tree that is being walked. + private var snapshot: DocumentSnapshot + + /// The workspace symbols representing the found `XCTestCase` subclasses and test methods. + private var result: [AnnotatedTestItem] = [] + + private init(snapshot: DocumentSnapshot) { + self.snapshot = snapshot + super.init(viewMode: .fixedUp) + } + + public static func findTestSymbols( + in snapshot: DocumentSnapshot, + syntaxTreeManager: SyntaxTreeManager + ) async -> [AnnotatedTestItem] { + guard snapshot.text.contains("XCTestCase") || snapshot.text.contains("test") else { + // If the file contains tests that can be discovered syntactically, it needs to have a class inheriting from + // `XCTestCase` or a function starting with `test`. + // This is intended to filter out files that obviously do not contain tests. + return [] + } + let syntaxTree = await syntaxTreeManager.syntaxTree(for: snapshot) + let visitor = SyntacticSwiftXCTestScanner(snapshot: snapshot) + visitor.walk(syntaxTree) + return visitor.result + } + + private func findTestMethods(in members: MemberBlockItemListSyntax, containerName: String) -> [TestItem] { + return members.compactMap { (member) -> TestItem? in + guard let function = member.decl.as(FunctionDeclSyntax.self) else { + return nil + } + guard function.name.text.starts(with: "test") else { + return nil + } + guard function.modifiers.map(\.name.tokenKind).allSatisfy({ $0 != .keyword(.static) && $0 != .keyword(.class) }) + else { + // Test methods can't be static. + return nil + } + guard function.signature.returnClause == nil, function.signature.parameterClause.parameters.isEmpty else { + // Test methods can't have a return type or have parameters. + // Technically we are also filtering out functions that have an explicit `Void` return type here but such + // declarations are probably less common than helper functions that start with `test` and have a return type. + return nil + } + let range = snapshot.absolutePositionRange( + of: function.positionAfterSkippingLeadingTrivia.. SyntaxVisitorContinueKind { + guard let inheritedTypes = node.inheritanceClause?.inheritedTypes, let superclass = inheritedTypes.first else { + // The class has no superclass and thus can't inherit from XCTestCase. + // Continue scanning its children in case it has a nested subclass that inherits from XCTestCase. + return .visitChildren + } + let superclassName = superclass.type.as(IdentifierTypeSyntax.self)?.name.text + if superclassName == "NSObject" { + // We know that the class can't be an subclass of `XCTestCase` so don't visit it. + // We can't explicitly check for the `XCTestCase` superclass because the class might inherit from a class that in + // turn inherits from `XCTestCase`. Resolving that inheritance hierarchy would be semantic. + return .visitChildren + } + let testMethods = findTestMethods(in: node.memberBlock.members, containerName: node.name.text) + guard !testMethods.isEmpty || superclassName == "XCTestCase" else { + // Don't report a test class if it doesn't contain any test methods. + return .visitChildren + } + let range = snapshot.absolutePositionRange( + of: node.positionAfterSkippingLeadingTrivia.. SyntaxVisitorContinueKind { + result += findTestMethods(in: node.memberBlock.members, containerName: node.extendedType.trimmedDescription) + .map { AnnotatedTestItem(testItem: $0, isExtension: true) } + return .visitChildren + } +} diff --git a/Sources/SourceKitLSP/TestDiscovery.swift b/Sources/SourceKitLSP/TestDiscovery.swift index 3a5509e8d..15febf978 100644 --- a/Sources/SourceKitLSP/TestDiscovery.swift +++ b/Sources/SourceKitLSP/TestDiscovery.swift @@ -21,22 +21,6 @@ public enum TestStyle { public static let swiftTesting = "swift-testing" } -public struct AnnotatedTestItem: Sendable { - /// The test item to be annotated - public var testItem: TestItem - - /// Whether the `TestItem` is an extension. - public var isExtension: Bool - - public init( - testItem: TestItem, - isExtension: Bool - ) { - self.testItem = testItem - self.isExtension = isExtension - } -} - fileprivate extension SymbolOccurrence { /// Assuming that this is a symbol occurrence returned by the index, return whether it can constitute the definition /// of a test case. @@ -352,117 +336,6 @@ extension SourceKitLSPServer { } } -/// Scans a source file for `XCTestCase` classes and test methods. -/// -/// The syntax visitor scans from class and extension declarations that could be `XCTestCase` classes or extensions -/// thereof. It then calls into `findTestMethods` to find the actual test methods. -final class SyntacticSwiftXCTestScanner: SyntaxVisitor { - /// The document snapshot of the syntax tree that is being walked. - private var snapshot: DocumentSnapshot - - /// The workspace symbols representing the found `XCTestCase` subclasses and test methods. - private var result: [AnnotatedTestItem] = [] - - private init(snapshot: DocumentSnapshot) { - self.snapshot = snapshot - super.init(viewMode: .fixedUp) - } - - public static func findTestSymbols( - in snapshot: DocumentSnapshot, - syntaxTreeManager: SyntaxTreeManager - ) async -> [AnnotatedTestItem] { - guard snapshot.text.contains("XCTestCase") || snapshot.text.contains("test") else { - // If the file contains tests that can be discovered syntactically, it needs to have a class inheriting from - // `XCTestCase` or a function starting with `test`. - // This is intended to filter out files that obviously do not contain tests. - return [] - } - let syntaxTree = await syntaxTreeManager.syntaxTree(for: snapshot) - let visitor = SyntacticSwiftXCTestScanner(snapshot: snapshot) - visitor.walk(syntaxTree) - return visitor.result - } - - private func findTestMethods(in members: MemberBlockItemListSyntax, containerName: String) -> [TestItem] { - return members.compactMap { (member) -> TestItem? in - guard let function = member.decl.as(FunctionDeclSyntax.self) else { - return nil - } - guard function.name.text.starts(with: "test") else { - return nil - } - guard function.modifiers.map(\.name.tokenKind).allSatisfy({ $0 != .keyword(.static) && $0 != .keyword(.class) }) - else { - // Test methods can't be static. - return nil - } - guard function.signature.returnClause == nil, function.signature.parameterClause.parameters.isEmpty else { - // Test methods can't have a return type or have parameters. - // Technically we are also filtering out functions that have an explicit `Void` return type here but such - // declarations are probably less common than helper functions that start with `test` and have a return type. - return nil - } - let range = snapshot.absolutePositionRange( - of: function.positionAfterSkippingLeadingTrivia.. SyntaxVisitorContinueKind { - guard let inheritedTypes = node.inheritanceClause?.inheritedTypes, let superclass = inheritedTypes.first else { - // The class has no superclass and thus can't inherit from XCTestCase. - // Continue scanning its children in case it has a nested subclass that inherits from XCTestCase. - return .visitChildren - } - let superclassName = superclass.type.as(IdentifierTypeSyntax.self)?.name.text - if superclassName == "NSObject" { - // We know that the class can't be an subclass of `XCTestCase` so don't visit it. - // We can't explicitly check for the `XCTestCase` superclass because the class might inherit from a class that in - // turn inherits from `XCTestCase`. Resolving that inheritance hierarchy would be semantic. - return .visitChildren - } - let testMethods = findTestMethods(in: node.memberBlock.members, containerName: node.name.text) - guard !testMethods.isEmpty || superclassName == "XCTestCase" else { - // Don't report a test class if it doesn't contain any test methods. - return .visitChildren - } - let range = snapshot.absolutePositionRange( - of: node.positionAfterSkippingLeadingTrivia.. SyntaxVisitorContinueKind { - result += findTestMethods(in: node.memberBlock.members, containerName: node.extendedType.trimmedDescription) - .map { AnnotatedTestItem(testItem: $0, isExtension: true) } - return .visitChildren - } -} - extension TestItem { /// Use out-of-date semantic information to filter syntactic symbols. /// @@ -506,7 +379,7 @@ extension AnnotatedTestItem { } } -extension Array { +fileprivate extension Array { /// When the test scanners discover tests in extensions they are captured in their own parent `TestItem`, not the /// `TestItem` generated from the class/struct's definition. This is largely because of the syntatic nature of the /// test scanners as they are today, which only know about tests within the context of the current file. Extensions From 2db4259d074cf43d86a74357a6f2cb34e2aeb4d5 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 10:43:46 -0700 Subject: [PATCH 38/69] Mark closures of `CheckedIndex.forEach*` as non-escaping With https://github.com/apple/indexstore-db/pull/198, they no longer need to be marked as escpaing. --- Sources/SemanticIndex/CheckedIndex.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/SemanticIndex/CheckedIndex.swift b/Sources/SemanticIndex/CheckedIndex.swift index c8953ae07..dff9cb004 100644 --- a/Sources/SemanticIndex/CheckedIndex.swift +++ b/Sources/SemanticIndex/CheckedIndex.swift @@ -64,7 +64,7 @@ public final class CheckedIndex { public func forEachSymbolOccurrence( byUSR usr: String, roles: SymbolRole, - _ body: @escaping (SymbolOccurrence) -> Bool + _ body: (SymbolOccurrence) -> Bool ) -> Bool { index.forEachSymbolOccurrence(byUSR: usr, roles: roles) { occurrence in guard self.checker.isUpToDate(occurrence.location) else { @@ -88,7 +88,7 @@ public final class CheckedIndex { anchorEnd: Bool, subsequence: Bool, ignoreCase: Bool, - body: @escaping (SymbolOccurrence) -> Bool + body: (SymbolOccurrence) -> Bool ) -> Bool { index.forEachCanonicalSymbolOccurrence( containing: pattern, From 80c214b00803dbf559db3d03b325de4ac801e021 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 11:53:40 -0700 Subject: [PATCH 39/69] Change `CheckedIndex` to use `DocumentURI` instead of `URL` --- .../SupportTypes/DocumentURI.swift | 4 + Sources/SKTestSupport/SkipUnless.swift | 4 +- Sources/SemanticIndex/CheckedIndex.swift | 92 +++++++++++-------- .../SemanticIndex/SemanticIndexManager.swift | 8 +- .../UpdateIndexStoreTaskDescription.swift | 6 +- Sources/SourceKitLSP/DocumentManager.swift | 6 +- Sources/SourceKitLSP/TestDiscovery.swift | 9 +- 7 files changed, 67 insertions(+), 62 deletions(-) diff --git a/Sources/LanguageServerProtocol/SupportTypes/DocumentURI.swift b/Sources/LanguageServerProtocol/SupportTypes/DocumentURI.swift index dd547de7d..2eea9c173 100644 --- a/Sources/LanguageServerProtocol/SupportTypes/DocumentURI.swift +++ b/Sources/LanguageServerProtocol/SupportTypes/DocumentURI.swift @@ -75,6 +75,10 @@ public struct DocumentURI: Codable, Hashable, Sendable { assert(self.storage.scheme != nil, "Received invalid URI without a scheme '\(self.storage.absoluteString)'") } + public init(filePath: String, isDirectory: Bool) { + self.init(URL(fileURLWithPath: filePath, isDirectory: isDirectory)) + } + public init(from decoder: Decoder) throws { try self.init(string: decoder.singleValueContainer().decode(String.self)) } diff --git a/Sources/SKTestSupport/SkipUnless.swift b/Sources/SKTestSupport/SkipUnless.swift index 9289c6fae..11c5ab913 100644 --- a/Sources/SKTestSupport/SkipUnless.swift +++ b/Sources/SKTestSupport/SkipUnless.swift @@ -70,12 +70,14 @@ public actor SkipUnless { ) } else if toolchainSwiftVersion == requiredSwiftVersion { logger.info("Checking if feature '\(featureName)' is supported") + defer { + logger.info("Done checking if feature '\(featureName)' is supported") + } if try await !featureCheck() { return .featureUnsupported(skipMessage: "Skipping because toolchain doesn't contain \(featureName)") } else { return .featureSupported } - logger.info("Done checking if feature '\(featureName)' is supported") } else { return .featureSupported } diff --git a/Sources/SemanticIndex/CheckedIndex.swift b/Sources/SemanticIndex/CheckedIndex.swift index c8953ae07..559936a55 100644 --- a/Sources/SemanticIndex/CheckedIndex.swift +++ b/Sources/SemanticIndex/CheckedIndex.swift @@ -19,9 +19,9 @@ import LanguageServerProtocol /// /// Protocol is needed because the `SemanticIndex` module is lower-level than the `SourceKitLSP` module. public protocol InMemoryDocumentManager { - /// Returns true if the file at the given URL has a different content in the document manager than on-disk. This is + /// Returns true if the file at the given URI has a different content in the document manager than on-disk. This is /// the case if the user made edits to the file but didn't save them yet. - func fileHasInMemoryModifications(_ url: URL) -> Bool + func fileHasInMemoryModifications(_ uri: DocumentURI) -> Bool } public enum IndexCheckLevel { @@ -105,7 +105,7 @@ public final class CheckedIndex { } public func symbols(inFilePath path: String) -> [Symbol] { - guard self.hasUpToDateUnit(for: URL(fileURLWithPath: path, isDirectory: false)) else { + guard self.hasUpToDateUnit(for: DocumentURI(filePath: path, isDirectory: false)) else { return [] } return index.symbols(inFilePath: path) @@ -121,11 +121,11 @@ public final class CheckedIndex { /// If `crossLanguage` is set to `true`, Swift files that import a header through a module will also be reported. public func mainFilesContainingFile(uri: DocumentURI, crossLanguage: Bool = false) -> [DocumentURI] { return index.mainFilesContainingFile(path: uri.pseudoPath, crossLanguage: crossLanguage).compactMap { - let url = URL(fileURLWithPath: $0) - guard checker.indexHasUpToDateUnit(for: url, mainFile: nil, index: self.index) else { + let uri = DocumentURI(filePath: $0, isDirectory: false) + guard checker.indexHasUpToDateUnit(for: uri, mainFile: nil, index: self.index) else { return nil } - return DocumentURI(url) + return uri } } @@ -141,16 +141,16 @@ public final class CheckedIndex { /// If `mainFile` is passed, then `url` is a header file that won't have a unit associated with it. `mainFile` is /// assumed to be a file that imports `url`. To check that `url` has an up-to-date unit, check that the latest unit /// for `mainFile` is newer than the mtime of the header file at `url`. - public func hasUpToDateUnit(for url: URL, mainFile: URL? = nil) -> Bool { - return checker.indexHasUpToDateUnit(for: url, mainFile: mainFile, index: index) + public func hasUpToDateUnit(for uri: DocumentURI, mainFile: DocumentURI? = nil) -> Bool { + return checker.indexHasUpToDateUnit(for: uri, mainFile: mainFile, index: index) } - /// Returns true if the file at the given URL has a different content in the document manager than on-disk. This is + /// Returns true if the file at the given URI has a different content in the document manager than on-disk. This is /// the case if the user made edits to the file but didn't save them yet. /// /// - Important: This must only be called on a `CheckedIndex` with a `checkLevel` of `inMemoryModifiedFiles` - public func fileHasInMemoryModifications(_ url: URL) -> Bool { - return checker.fileHasInMemoryModifications(url) + public func fileHasInMemoryModifications(_ uri: DocumentURI) -> Bool { + return checker.fileHasInMemoryModifications(uri) } } @@ -212,14 +212,14 @@ private struct IndexOutOfDateChecker { } } - /// Caches whether a file URL has modifications in `documentManager` that haven't been saved to disk yet. - private var fileHasInMemoryModificationsCache: [URL: Bool] = [:] + /// Caches whether a document has modifications in `documentManager` that haven't been saved to disk yet. + private var fileHasInMemoryModificationsCache: [DocumentURI: Bool] = [:] - /// File URLs to modification times that have already been computed. - private var modTimeCache: [URL: ModificationTime] = [:] + /// Document URIs to modification times that have already been computed. + private var modTimeCache: [DocumentURI: ModificationTime] = [:] - /// File URLs to whether they exist on the file system - private var fileExistsCache: [URL: Bool] = [:] + /// Document URIs to whether they exist on the file system + private var fileExistsCache: [DocumentURI: Bool] = [:] init(checkLevel: IndexCheckLevel) { self.checkLevel = checkLevel @@ -230,16 +230,16 @@ private struct IndexOutOfDateChecker { /// Returns `true` if the source file for the given symbol location exists and has not been modified after it has been /// indexed. mutating func isUpToDate(_ symbolLocation: SymbolLocation) -> Bool { - let url = URL(fileURLWithPath: symbolLocation.path, isDirectory: false) + let uri = DocumentURI(filePath: symbolLocation.path, isDirectory: false) switch checkLevel { case .inMemoryModifiedFiles(let documentManager): - if fileHasInMemoryModifications(url, documentManager: documentManager) { + if fileHasInMemoryModifications(uri, documentManager: documentManager) { return false } fallthrough case .modifiedFiles: do { - let sourceFileModificationDate = try modificationDate(of: url) + let sourceFileModificationDate = try modificationDate(of: uri) switch sourceFileModificationDate { case .fileDoesNotExist: return false @@ -251,7 +251,7 @@ private struct IndexOutOfDateChecker { return true } case .deletedFiles: - return fileExists(at: url) + return fileExists(at: uri) } } @@ -262,7 +262,7 @@ private struct IndexOutOfDateChecker { /// If `mainFile` is passed, then `filePath` is a header file that won't have a unit associated with it. `mainFile` is /// assumed to be a file that imports `url`. To check that `url` has an up-to-date unit, check that the latest unit /// for `mainFile` is newer than the mtime of the header file at `url`. - mutating func indexHasUpToDateUnit(for filePath: URL, mainFile: URL?, index: IndexStoreDB) -> Bool { + mutating func indexHasUpToDateUnit(for filePath: DocumentURI, mainFile: DocumentURI?, index: IndexStoreDB) -> Bool { switch checkLevel { case .inMemoryModifiedFiles(let documentManager): if fileHasInMemoryModifications(filePath, documentManager: documentManager) { @@ -273,7 +273,9 @@ private struct IndexOutOfDateChecker { // If there are no in-memory modifications check if there are on-disk modifications. fallthrough case .modifiedFiles: - guard let lastUnitDate = index.dateOfLatestUnitFor(filePath: (mainFile ?? filePath).path) else { + guard let fileURL = (mainFile ?? filePath).fileURL, + let lastUnitDate = index.dateOfLatestUnitFor(filePath: fileURL.path) + else { return false } do { @@ -300,23 +302,25 @@ private struct IndexOutOfDateChecker { /// `documentManager` must always be the same between calls to `hasFileInMemoryModifications` since it is not part of /// the cache key. This is fine because we always assume the `documentManager` to come from the associated value of /// `CheckLevel.imMemoryModifiedFiles`, which is constant. - private mutating func fileHasInMemoryModifications(_ url: URL, documentManager: InMemoryDocumentManager) -> Bool { - if let cached = fileHasInMemoryModificationsCache[url] { + private mutating func fileHasInMemoryModifications(_ uri: DocumentURI, documentManager: InMemoryDocumentManager) + -> Bool + { + if let cached = fileHasInMemoryModificationsCache[uri] { return cached } - let hasInMemoryModifications = documentManager.fileHasInMemoryModifications(url) - fileHasInMemoryModificationsCache[url] = hasInMemoryModifications + let hasInMemoryModifications = documentManager.fileHasInMemoryModifications(uri) + fileHasInMemoryModificationsCache[uri] = hasInMemoryModifications return hasInMemoryModifications } - /// Returns true if the file at the given URL has a different content in the document manager than on-disk. This is + /// Returns true if the file at the given URI has a different content in the document manager than on-disk. This is /// the case if the user made edits to the file but didn't save them yet. /// /// - Important: This must only be called on an `IndexOutOfDateChecker` with a `checkLevel` of `inMemoryModifiedFiles` - mutating func fileHasInMemoryModifications(_ url: URL) -> Bool { + mutating func fileHasInMemoryModifications(_ uri: DocumentURI) -> Bool { switch checkLevel { case .inMemoryModifiedFiles(let documentManager): - return fileHasInMemoryModifications(url, documentManager: documentManager) + return fileHasInMemoryModifications(uri, documentManager: documentManager) case .modifiedFiles, .deletedFiles: logger.fault( "fileHasInMemoryModifications(at:) must only be called on an `IndexOutOfDateChecker` with check level .inMemoryModifiedFiles" @@ -325,9 +329,12 @@ private struct IndexOutOfDateChecker { } } - private func modificationDateUncached(of url: URL) throws -> ModificationTime { + private func modificationDateUncached(of uri: DocumentURI) throws -> ModificationTime { do { - let attributes = try FileManager.default.attributesOfItem(atPath: url.resolvingSymlinksInPath().path) + guard let fileURL = uri.fileURL else { + return .fileDoesNotExist + } + let attributes = try FileManager.default.attributesOfItem(atPath: fileURL.resolvingSymlinksInPath().path) guard let modificationDate = attributes[FileAttributeKey.modificationDate] as? Date else { throw Error.fileAttributesDontHaveModificationDate } @@ -337,21 +344,26 @@ private struct IndexOutOfDateChecker { } } - private mutating func modificationDate(of url: URL) throws -> ModificationTime { - if let cached = modTimeCache[url] { + private mutating func modificationDate(of uri: DocumentURI) throws -> ModificationTime { + if let cached = modTimeCache[uri] { return cached } - let modTime = try modificationDateUncached(of: url) - modTimeCache[url] = modTime + let modTime = try modificationDateUncached(of: uri) + modTimeCache[uri] = modTime return modTime } - private mutating func fileExists(at url: URL) -> Bool { - if let cached = fileExistsCache[url] { + private mutating func fileExists(at uri: DocumentURI) -> Bool { + if let cached = fileExistsCache[uri] { return cached } - let fileExists = FileManager.default.fileExists(atPath: url.path) - fileExistsCache[url] = fileExists + let fileExists = + if let fileUrl = uri.fileURL { + FileManager.default.fileExists(atPath: fileUrl.path) + } else { + false + } + fileExistsCache[uri] = fileExists return fileExists } } diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index 50bebba9e..23d2e7f11 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -260,13 +260,7 @@ public final actor SemanticIndexManager { await testHooks.buildGraphGenerationDidFinish?() let index = index.checked(for: .modifiedFiles) let filesToIndex = await self.buildSystemManager.sourceFiles().lazy.map(\.uri) - .filter { uri in - guard let url = uri.fileURL else { - // The URI is not a file, so there's nothing we can index. - return false - } - return !index.hasUpToDateUnit(for: url) - } + .filter { !index.hasUpToDateUnit(for: $0) } await scheduleBackgroundIndex(files: filesToIndex) generateBuildGraphTask = nil } diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift index 54d7dbeaa..66f544be6 100644 --- a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -206,11 +206,7 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { // If we know that the file is up-to-date without having ot hit the index, do that because it's fastest. return } - guard let sourceFileUrl = file.sourceFile.fileURL else { - // The URI is not a file, so there's nothing we can index. - return - } - guard !index.checked(for: .modifiedFiles).hasUpToDateUnit(for: sourceFileUrl, mainFile: file.mainFile.fileURL) + guard !index.checked(for: .modifiedFiles).hasUpToDateUnit(for: file.sourceFile, mainFile: file.mainFile) else { logger.debug("Not indexing \(file.forLogging) because index has an up-to-date unit") // We consider a file's index up-to-date if we have any up-to-date unit. Changing build settings does not diff --git a/Sources/SourceKitLSP/DocumentManager.swift b/Sources/SourceKitLSP/DocumentManager.swift index 6ade8e8b1..97cfdc624 100644 --- a/Sources/SourceKitLSP/DocumentManager.swift +++ b/Sources/SourceKitLSP/DocumentManager.swift @@ -191,12 +191,12 @@ public final class DocumentManager: InMemoryDocumentManager, Sendable { } } - public func fileHasInMemoryModifications(_ url: URL) -> Bool { - guard let document = try? latestSnapshot(DocumentURI(url)) else { + public func fileHasInMemoryModifications(_ uri: DocumentURI) -> Bool { + guard let document = try? latestSnapshot(uri), let fileURL = uri.fileURL else { return false } - guard let onDiskFileContents = try? String(contentsOf: url, encoding: .utf8) else { + guard let onDiskFileContents = try? String(contentsOf: fileURL, encoding: .utf8) else { // If we can't read the file on disk, it can't match any on-disk state, so it's in-memory state return true } diff --git a/Sources/SourceKitLSP/TestDiscovery.swift b/Sources/SourceKitLSP/TestDiscovery.swift index 3a5509e8d..069a0f40d 100644 --- a/Sources/SourceKitLSP/TestDiscovery.swift +++ b/Sources/SourceKitLSP/TestDiscovery.swift @@ -204,12 +204,9 @@ extension SourceKitLSPServer { let index = workspace.index(checkedFor: .inMemoryModifiedFiles(documentManager)) let filesWithInMemoryState = documentManager.documents.keys.filter { uri in - guard let url = uri.fileURL else { - return true - } // Use the index to check for in-memory modifications so we can re-use its cache. If no index exits, ask the // document manager directly. - return index?.fileHasInMemoryModifications(url) ?? documentManager.fileHasInMemoryModifications(url) + return index?.fileHasInMemoryModifications(uri) ?? documentManager.fileHasInMemoryModifications(uri) } let testsFromFilesWithInMemoryState = await filesWithInMemoryState.concurrentMap { (uri) -> [AnnotatedTestItem] in @@ -254,7 +251,7 @@ extension SourceKitLSPServer { // up-to-date. Include the tests from `testsFromFilesWithInMemoryState`. return nil } - if let fileUrl = testItem.location.uri.fileURL, index?.hasUpToDateUnit(for: fileUrl) ?? false { + if index?.hasUpToDateUnit(for: testItem.location.uri) ?? false { // We don't have a test for this file in the semantic index but an up-to-date unit file. This means that the // index is up-to-date and has more knowledge that identifies a `TestItem` as not actually being a test, eg. // because it starts with `test` but doesn't appear in a class inheriting from `XCTestCase`. @@ -341,7 +338,7 @@ extension SourceKitLSPServer { } ) + syntacticSwiftTestingTests } - if let fileURL = mainFileUri.fileURL, index.hasUpToDateUnit(for: fileURL) { + if index.hasUpToDateUnit(for: mainFileUri) { // The semantic index is up-to-date and doesn't contain any tests. We don't need to do a syntactic fallback for // XCTest. We do still need to return swift-testing tests which don't have a semantic index. return syntacticSwiftTestingTests From f84cfecbf27910cf900af7b3fe005fc93bca71b1 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 12:20:08 -0700 Subject: [PATCH 40/69] Make `CompilationDatabase` use `DocumentURI` instead of `URL` --- Sources/SKCore/CompilationDatabase.swift | 53 +++++++++++-------- .../CompilationDatabaseBuildSystem.swift | 19 +++---- .../CompilationDatabaseTests.swift | 8 +-- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/Sources/SKCore/CompilationDatabase.swift b/Sources/SKCore/CompilationDatabase.swift index 63800ffb3..cdd71e20c 100644 --- a/Sources/SKCore/CompilationDatabase.swift +++ b/Sources/SKCore/CompilationDatabase.swift @@ -12,6 +12,7 @@ import Foundation import LSPLogging +import LanguageServerProtocol import SKSupport import struct TSCBasic.AbsolutePath @@ -47,15 +48,15 @@ public struct CompilationDatabaseCompileCommand: Equatable { extension CompilationDatabase.Command { - /// The `URL` for this file. If `filename` is relative and `directory` is + /// The `DocumentURI` for this file. If `filename` is relative and `directory` is /// absolute, returns the concatenation. However, if both paths are relative, /// it falls back to `filename`, which is more likely to be the identifier /// that a caller will be looking for. - public var url: URL { + public var uri: DocumentURI { if filename.hasPrefix("/") || !directory.hasPrefix("/") { - return URL(fileURLWithPath: filename) + return DocumentURI(filePath: filename, isDirectory: false) } else { - return URL(fileURLWithPath: directory).appendingPathComponent(filename, isDirectory: false) + return DocumentURI(URL(fileURLWithPath: directory).appendingPathComponent(filename, isDirectory: false)) } } } @@ -65,7 +66,7 @@ extension CompilationDatabase.Command { /// See https://clang.llvm.org/docs/JSONCompilationDatabase.html public protocol CompilationDatabase { typealias Command = CompilationDatabaseCompileCommand - subscript(_ path: URL) -> [Command] { get } + subscript(_ uri: DocumentURI) -> [Command] { get } var allCommands: AnySequence { get } } @@ -110,13 +111,13 @@ public func tryLoadCompilationDatabase( /// /// See https://clang.llvm.org/docs/JSONCompilationDatabase.html under Alternatives public struct FixedCompilationDatabase: CompilationDatabase, Equatable { - public var allCommands: AnySequence { AnySequence([]) } + public var allCommands: AnySequence { AnySequence([]) } private let fixedArgs: [String] private let directory: String - public subscript(path: URL) -> [Command] { - [Command(directory: directory, filename: path.path, commandLine: fixedArgs + [path.path])] + public subscript(path: DocumentURI) -> [CompilationDatabaseCompileCommand] { + [Command(directory: directory, filename: path.pseudoPath, commandLine: fixedArgs + [path.pseudoPath])] } } @@ -168,32 +169,38 @@ extension FixedCompilationDatabase { /// /// See https://clang.llvm.org/docs/JSONCompilationDatabase.html public struct JSONCompilationDatabase: CompilationDatabase, Equatable { - var pathToCommands: [URL: [Int]] = [:] - var commands: [Command] = [] + var pathToCommands: [DocumentURI: [Int]] = [:] + var commands: [CompilationDatabaseCompileCommand] = [] - public init(_ commands: [Command] = []) { - commands.forEach { try! add($0) } + public init(_ commands: [CompilationDatabaseCompileCommand] = []) { + for command in commands { + add(command) + } } - public subscript(_ url: URL) -> [Command] { - if let indices = pathToCommands[url] { + public subscript(_ uri: DocumentURI) -> [CompilationDatabaseCompileCommand] { + if let indices = pathToCommands[uri] { return indices.map { commands[$0] } } - if let indices = pathToCommands[url.resolvingSymlinksInPath()] { + if let fileURL = uri.fileURL, let indices = pathToCommands[DocumentURI(fileURL.resolvingSymlinksInPath())] { return indices.map { commands[$0] } } return [] } - public var allCommands: AnySequence { AnySequence(commands) } + public var allCommands: AnySequence { AnySequence(commands) } - public mutating func add(_ command: Command) throws { - let url = command.url - pathToCommands[url, default: []].append(commands.count) + public mutating func add(_ command: CompilationDatabaseCompileCommand) { + let uri = command.uri + pathToCommands[uri, default: []].append(commands.count) - let canonical = URL(fileURLWithPath: try resolveSymlinks(AbsolutePath(validating: url.path)).pathString) - if canonical != url { - pathToCommands[canonical, default: []].append(commands.count) + if let fileURL = uri.fileURL, + let symlinksResolved = try? resolveSymlinks(AbsolutePath(validating: fileURL.path)) + { + let canonical = DocumentURI(filePath: symlinksResolved.pathString, isDirectory: false) + if canonical != uri { + pathToCommands[canonical, default: []].append(commands.count) + } } commands.append(command) @@ -204,7 +211,7 @@ extension JSONCompilationDatabase: Codable { public init(from decoder: Decoder) throws { var container = try decoder.unkeyedContainer() while !container.isAtEnd { - try self.add(try container.decode(Command.self)) + self.add(try container.decode(Command.self)) } } diff --git a/Sources/SKCore/CompilationDatabaseBuildSystem.swift b/Sources/SKCore/CompilationDatabaseBuildSystem.swift index d601bd099..7dc4556d2 100644 --- a/Sources/SKCore/CompilationDatabaseBuildSystem.swift +++ b/Sources/SKCore/CompilationDatabaseBuildSystem.swift @@ -106,12 +106,8 @@ extension CompilationDatabaseBuildSystem: BuildSystem { in buildTarget: ConfiguredTarget, language: Language ) async -> FileBuildSettings? { - guard let url = document.fileURL else { - // We can't determine build settings for non-file URIs. - return nil - } - guard let db = database(for: url), - let cmd = db[url].first + guard let db = database(for: document), + let cmd = db[document].first else { return nil } return FileBuildSettings( compilerArguments: Array(cmd.commandLine.dropFirst()), @@ -153,8 +149,8 @@ extension CompilationDatabaseBuildSystem: BuildSystem { self.watchedFiles.remove(uri) } - private func database(for url: URL) -> CompilationDatabase? { - if let path = try? AbsolutePath(validating: url.path) { + private func database(for uri: DocumentURI) -> CompilationDatabase? { + if let url = uri.fileURL, let path = try? AbsolutePath(validating: url.path) { return database(for: path) } return compdb @@ -212,10 +208,7 @@ extension CompilationDatabaseBuildSystem: BuildSystem { } public func fileHandlingCapability(for uri: DocumentURI) -> FileHandlingCapability { - guard let fileUrl = uri.fileURL else { - return .unhandled - } - if database(for: fileUrl) != nil { + if database(for: uri) != nil { return .handled } else { return .unhandled @@ -227,7 +220,7 @@ extension CompilationDatabaseBuildSystem: BuildSystem { return [] } return compdb.allCommands.map { - SourceFileInfo(uri: DocumentURI($0.url), isPartOfRootProject: true, mayContainTests: true) + SourceFileInfo(uri: $0.uri, isPartOfRootProject: true, mayContainTests: true) } } diff --git a/Tests/SKCoreTests/CompilationDatabaseTests.swift b/Tests/SKCoreTests/CompilationDatabaseTests.swift index 934c1aaa7..27ddddeb5 100644 --- a/Tests/SKCoreTests/CompilationDatabaseTests.swift +++ b/Tests/SKCoreTests/CompilationDatabaseTests.swift @@ -157,9 +157,9 @@ final class CompilationDatabaseTests: XCTestCase { let db = JSONCompilationDatabase([cmd1, cmd2, cmd3]) - XCTAssertEqual(db[URL(fileURLWithPath: "b")], [cmd1]) - XCTAssertEqual(db[URL(fileURLWithPath: "/c/b")], [cmd2]) - XCTAssertEqual(db[URL(fileURLWithPath: "/b")], [cmd3]) + XCTAssertEqual(db[DocumentURI(filePath: "b", isDirectory: false)], [cmd1]) + XCTAssertEqual(db[DocumentURI(filePath: "/c/b", isDirectory: false)], [cmd2]) + XCTAssertEqual(db[DocumentURI(filePath: "/b", isDirectory: false)], [cmd3]) } func testJSONCompilationDatabaseFromDirectory() throws { @@ -255,7 +255,7 @@ final class CompilationDatabaseTests: XCTestCase { XCTAssertNotNil(db) XCTAssertEqual( - db![URL(fileURLWithPath: "/a/b")], + db![DocumentURI(filePath: "/a/b", isDirectory: false)], [ CompilationDatabase.Command( directory: try AbsolutePath(validating: "/a").pathString, From 532a37bb33e3d57372d1f20f1674fc7faa11a28f Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 16:52:53 -0700 Subject: [PATCH 41/69] Show container name in call hierarchy item name instead of detail field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The container name isn’t displayed very prominently and it’s easy to miss. I think the call hierarchy looks nicer when we prepend the container name with language-specific logic. rdar://128396648 --- Sources/SourceKitLSP/SourceKitLSPServer.swift | 34 ++++++- .../CallHierarchyTests.swift | 94 ++++++++++++++++++- 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index e883345c7..16276c85c 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -1851,11 +1851,28 @@ extension SourceKitLSPServer { containerName: String?, location: Location ) -> CallHierarchyItem { - CallHierarchyItem( - name: symbol.name, + let name: String + if let containerName { + switch symbol.language { + case .objc where symbol.kind == .instanceMethod || symbol.kind == .instanceProperty: + name = "-[\(containerName) \(symbol.name)]" + case .objc where symbol.kind == .classMethod || symbol.kind == .classProperty: + name = "+[\(containerName) \(symbol.name)]" + case .cxx, .c, .objc: + // C shouldn't have container names for call hierarchy and Objective-C should be covered above. + // Fall back to using the C++ notation using `::`. + name = "\(containerName)::\(symbol.name)" + case .swift: + name = "\(containerName).\(symbol.name)" + } + } else { + name = symbol.name + } + return CallHierarchyItem( + name: name, kind: symbol.kind.asLspSymbolKind(), tags: nil, - detail: containerName, + detail: nil, uri: location.uri, range: location.range, selectionRange: location.range, @@ -2289,8 +2306,15 @@ extension IndexSymbolKind { return .struct case .parameter: return .typeParameter - - default: + case .module, .namespace: + return .namespace + case .field: + return .property + case .constructor: + return .constructor + case .destructor: + return .null + case .commentTag, .concept, .extension, .macro, .namespaceAlias, .typealias, .union, .unknown, .using: return .null } } diff --git a/Tests/SourceKitLSPTests/CallHierarchyTests.swift b/Tests/SourceKitLSPTests/CallHierarchyTests.swift index bafd55af6..4a90f1d61 100644 --- a/Tests/SourceKitLSPTests/CallHierarchyTests.swift +++ b/Tests/SourceKitLSPTests/CallHierarchyTests.swift @@ -212,10 +212,9 @@ final class CallHierarchyTests: XCTestCase { result, [ CallHierarchyItem( - name: "foo", + name: "FilePathIndex::foo", kind: .method, tags: nil, - detail: "FilePathIndex", uri: try project.uri(for: "lib.cpp"), range: try Range(project.position(of: "2️⃣", in: "lib.cpp")), selectionRange: try Range(project.position(of: "2️⃣", in: "lib.cpp")), @@ -583,10 +582,9 @@ final class CallHierarchyTests: XCTestCase { [ CallHierarchyIncomingCall( from: CallHierarchyItem( - name: "foo()", + name: "MyClass.foo()", kind: .method, tags: nil, - detail: "MyClass", uri: project.fileURI, range: Range(project.positions["1️⃣"]), selectionRange: Range(project.positions["1️⃣"]), @@ -764,4 +762,92 @@ final class CallHierarchyTests: XCTestCase { ] ) } + + func testInitializerInCallHierarchy() async throws { + try await SkipUnless.indexOnlyHasContainedByRelationsToIndexedDecls() + let project = try await IndexedSingleSwiftFileTestProject( + """ + func 1️⃣foo() {} + + struct Bar { + 2️⃣init() { + 3️⃣foo() + } + } + """ + ) + let prepare = try await project.testClient.send( + CallHierarchyPrepareRequest( + textDocument: TextDocumentIdentifier(project.fileURI), + position: project.positions["1️⃣"] + ) + ) + let initialItem = try XCTUnwrap(prepare?.only) + let calls = try await project.testClient.send(CallHierarchyIncomingCallsRequest(item: initialItem)) + XCTAssertEqual( + calls, + [ + CallHierarchyIncomingCall( + from: CallHierarchyItem( + name: "Bar.init()", + kind: .constructor, + tags: nil, + uri: project.fileURI, + range: Range(project.positions["2️⃣"]), + selectionRange: Range(project.positions["2️⃣"]), + data: .dictionary([ + "usr": .string("s:4test3BarVACycfc"), + "uri": .string(project.fileURI.stringValue), + ]) + ), + fromRanges: [Range(project.positions["3️⃣"])] + ) + ] + ) + } + + func testCallHierarchyOfNestedClass() async throws { + try await SkipUnless.indexOnlyHasContainedByRelationsToIndexedDecls() + let project = try await IndexedSingleSwiftFileTestProject( + """ + func 1️⃣foo() {} + + struct Outer { + struct Bar { + 2️⃣init() { + 3️⃣foo() + } + } + } + """ + ) + let prepare = try await project.testClient.send( + CallHierarchyPrepareRequest( + textDocument: TextDocumentIdentifier(project.fileURI), + position: project.positions["1️⃣"] + ) + ) + let initialItem = try XCTUnwrap(prepare?.only) + let calls = try await project.testClient.send(CallHierarchyIncomingCallsRequest(item: initialItem)) + XCTAssertEqual( + calls, + [ + CallHierarchyIncomingCall( + from: CallHierarchyItem( + name: "Bar.init()", + kind: .constructor, + tags: nil, + uri: project.fileURI, + range: Range(project.positions["2️⃣"]), + selectionRange: Range(project.positions["2️⃣"]), + data: .dictionary([ + "usr": .string("s:4test5OuterV3BarVAEycfc"), + "uri": .string(project.fileURI.stringValue), + ]) + ), + fromRanges: [Range(project.positions["3️⃣"])] + ) + ] + ) + } } From 68af33edd0634275168fb7511b94b8617f4b164c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 17:06:11 -0700 Subject: [PATCH 42/69] Show `deinit` in document symbols rdar://128715297 --- .../SourceKitLSP/Swift/DocumentSymbols.swift | 10 +++++++ .../DocumentSymbolTests.swift | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/Sources/SourceKitLSP/Swift/DocumentSymbols.swift b/Sources/SourceKitLSP/Swift/DocumentSymbols.swift index a95946a24..5028d55d9 100644 --- a/Sources/SourceKitLSP/Swift/DocumentSymbols.swift +++ b/Sources/SourceKitLSP/Swift/DocumentSymbols.swift @@ -170,6 +170,16 @@ fileprivate final class DocumentSymbolsFinder: SyntaxAnyVisitor { return .skipChildren } + override func visit(_ node: DeinitializerDeclSyntax) -> SyntaxVisitorContinueKind { + return record( + node: node, + name: node.deinitKeyword.text, + symbolKind: .null, + range: node.rangeWithoutTrivia, + selection: node.deinitKeyword.rangeWithoutTrivia + ) + } + override func visit(_ node: EnumCaseElementSyntax) -> SyntaxVisitorContinueKind { let rangeEnd = if let parameterClause = node.parameterClause { diff --git a/Tests/SourceKitLSPTests/DocumentSymbolTests.swift b/Tests/SourceKitLSPTests/DocumentSymbolTests.swift index 03b853f82..c6f5afc62 100644 --- a/Tests/SourceKitLSPTests/DocumentSymbolTests.swift +++ b/Tests/SourceKitLSPTests/DocumentSymbolTests.swift @@ -729,6 +729,35 @@ final class DocumentSymbolTests: XCTestCase { ] } } + + func testShowDeinit() async throws { + try await assertDocumentSymbols( + """ + 1️⃣class 2️⃣Foo3️⃣ { + 4️⃣deinit5️⃣ { + }6️⃣ + }7️⃣ + """ + ) { positions in + [ + DocumentSymbol( + name: "Foo", + kind: .class, + range: positions["1️⃣"].. Date: Sat, 1 Jun 2024 17:13:52 -0700 Subject: [PATCH 43/69] Add an extension to the LogMessageNotification to add a message to a specific log in the client If the editor has support for this LSP extension, it can create a separate output view for the index log. For example, in VS Code, we could have one output view for the standard LSP logging (which are the messages between VS Code and SourceKit-LSP if verbose logging is enabled in VS Code) and a separate log that shows information about background indexing. rdar://128572032 --- .../Notifications/LogMessageNotification.swift | 9 ++++++++- Sources/SourceKitLSP/SourceKitLSPServer.swift | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Sources/LanguageServerProtocol/Notifications/LogMessageNotification.swift b/Sources/LanguageServerProtocol/Notifications/LogMessageNotification.swift index 17c268034..0e3deebbc 100644 --- a/Sources/LanguageServerProtocol/Notifications/LogMessageNotification.swift +++ b/Sources/LanguageServerProtocol/Notifications/LogMessageNotification.swift @@ -24,8 +24,15 @@ public struct LogMessageNotification: NotificationType, Hashable { /// The contents of the message. public var message: String - public init(type: WindowMessageType, message: String) { + /// If specified, the client should log the message to a log with this name instead of the standard log for this LSP + /// server. + /// + /// **(LSP Extension)** + public var logName: String? + + public init(type: WindowMessageType, message: String, logName: String?) { self.type = type self.message = message + self.logName = logName } } diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index e883345c7..6ab328d97 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -845,7 +845,8 @@ extension SourceKitLSPServer { \(result.taskDescription) finished in \(result.duration) \(result.command) \(result.output) - """ + """, + logName: "SourceKit-LSP: Indexing" ) ) } From ef8b4f5eb5059b58c36b340296d708c68e53ddd8 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 17:26:54 -0700 Subject: [PATCH 44/69] Move debug subcommands of sourcekit-lsp to a `debug` subcommand and unhide them We have a few subcommands on sourcekit-lsp that are intended for debugging sourcekit-lsp and that are thus hidden. This moves them all to a `debug` subcommand (eg. `sourcekit-lsp debug run-sourcekitd-request`) and unhides them. That makes them more discoverable for interested users and also helps developers of sourcekit-lsp remember what they are called. rdar://128443298 --- Sources/Diagnose/CMakeLists.txt | 1 + Sources/Diagnose/DebugCommand.swift | 28 +++++++++++++++++++ Sources/Diagnose/IndexCommand.swift | 3 +- Sources/Diagnose/ReduceCommand.swift | 3 +- Sources/Diagnose/ReduceFrontendCommand.swift | 3 +- .../RunSourcekitdRequestCommand.swift | 3 +- .../Diagnose/SourceKitDRequestExecutor.swift | 1 + Sources/sourcekit-lsp/SourceKitLSP.swift | 5 +--- 8 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 Sources/Diagnose/DebugCommand.swift diff --git a/Sources/Diagnose/CMakeLists.txt b/Sources/Diagnose/CMakeLists.txt index 32caa5515..ca4dcb0f9 100644 --- a/Sources/Diagnose/CMakeLists.txt +++ b/Sources/Diagnose/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(Diagnose STATIC CommandConfiguration+Sendable.swift CommandLineArgumentsReducer.swift + DebugCommand.swift DiagnoseCommand.swift IndexCommand.swift MergeSwiftFiles.swift diff --git a/Sources/Diagnose/DebugCommand.swift b/Sources/Diagnose/DebugCommand.swift new file mode 100644 index 000000000..b6d30e8ab --- /dev/null +++ b/Sources/Diagnose/DebugCommand.swift @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser + +public struct DebugCommand: ParsableCommand { + public static let configuration = CommandConfiguration( + commandName: "debug", + abstract: "Commands to debug sourcekit-lsp. Intended for developers of sourcekit-lsp", + subcommands: [ + IndexCommand.self, + ReduceCommand.self, + ReduceFrontendCommand.self, + RunSourceKitdRequestCommand.self, + ] + ) + + public init() {} +} diff --git a/Sources/Diagnose/IndexCommand.swift b/Sources/Diagnose/IndexCommand.swift index a00872d29..d2b40feb8 100644 --- a/Sources/Diagnose/IndexCommand.swift +++ b/Sources/Diagnose/IndexCommand.swift @@ -55,8 +55,7 @@ private actor IndexLogMessageHandler: MessageHandler { public struct IndexCommand: AsyncParsableCommand { public static let configuration: CommandConfiguration = CommandConfiguration( commandName: "index", - abstract: "Index a project and print all the processes executed for it as well as their outputs", - shouldDisplay: false + abstract: "Index a project and print all the processes executed for it as well as their outputs" ) @Option( diff --git a/Sources/Diagnose/ReduceCommand.swift b/Sources/Diagnose/ReduceCommand.swift index 51bbfe4bc..e8a59cdf0 100644 --- a/Sources/Diagnose/ReduceCommand.swift +++ b/Sources/Diagnose/ReduceCommand.swift @@ -22,8 +22,7 @@ import class TSCUtility.PercentProgressAnimation public struct ReduceCommand: AsyncParsableCommand { public static let configuration: CommandConfiguration = CommandConfiguration( commandName: "reduce", - abstract: "Reduce a single sourcekitd crash", - shouldDisplay: false + abstract: "Reduce a single sourcekitd crash" ) @Option(name: .customLong("request-file"), help: "Path to a sourcekitd request to reduce.") diff --git a/Sources/Diagnose/ReduceFrontendCommand.swift b/Sources/Diagnose/ReduceFrontendCommand.swift index fe57124ae..9906e839c 100644 --- a/Sources/Diagnose/ReduceFrontendCommand.swift +++ b/Sources/Diagnose/ReduceFrontendCommand.swift @@ -22,8 +22,7 @@ import class TSCUtility.PercentProgressAnimation public struct ReduceFrontendCommand: AsyncParsableCommand { public static let configuration: CommandConfiguration = CommandConfiguration( commandName: "reduce-frontend", - abstract: "Reduce a single swift-frontend crash", - shouldDisplay: false + abstract: "Reduce a single swift-frontend crash" ) #if canImport(Darwin) diff --git a/Sources/Diagnose/RunSourcekitdRequestCommand.swift b/Sources/Diagnose/RunSourcekitdRequestCommand.swift index 771fc4777..6cfe921c0 100644 --- a/Sources/Diagnose/RunSourcekitdRequestCommand.swift +++ b/Sources/Diagnose/RunSourcekitdRequestCommand.swift @@ -21,8 +21,7 @@ import struct TSCBasic.AbsolutePath public struct RunSourceKitdRequestCommand: AsyncParsableCommand { public static let configuration = CommandConfiguration( commandName: "run-sourcekitd-request", - abstract: "Run a sourcekitd request and print its result", - shouldDisplay: false + abstract: "Run a sourcekitd request and print its result" ) @Option( diff --git a/Sources/Diagnose/SourceKitDRequestExecutor.swift b/Sources/Diagnose/SourceKitDRequestExecutor.swift index 389549a0d..bcb72add5 100644 --- a/Sources/Diagnose/SourceKitDRequestExecutor.swift +++ b/Sources/Diagnose/SourceKitDRequestExecutor.swift @@ -161,6 +161,7 @@ public class OutOfProcessSourceKitRequestExecutor: SourceKitRequestExecutor { let process = Process( arguments: [ ProcessInfo.processInfo.arguments[0], + "debug", "run-sourcekitd-request", "--sourcekitd", sourcekitd.path, diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index 0cece17c0..133c837b8 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -105,10 +105,7 @@ struct SourceKitLSP: AsyncParsableCommand { abstract: "Language Server Protocol implementation for Swift and C-based languages", subcommands: [ DiagnoseCommand.self, - IndexCommand.self, - ReduceCommand.self, - ReduceFrontendCommand.self, - RunSourceKitdRequestCommand.self, + DebugCommand.self, ] ) From c4e6e85ce8723c8bf637721224991fb182627849 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 22:39:30 -0700 Subject: [PATCH 45/69] =?UTF-8?q?Show=20=E2=80=9CPreparing=20targets?= =?UTF-8?q?=E2=80=9D=20as=20index=20status=20if=20we=20only=20have=20prepa?= =?UTF-8?q?ration=20tasks=20and=20no=20index=20tasks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I was wondering for a while why we were showing “Indexing 0 / 0” as the index progress. I think it’s if we only have a preparation task but no index tasks running. I’m not entirely sure how this happens because preparation should only happen for two reasons: - We are preparing a target so we can index a files -> We should have an active index task - We are preparing a file for editor functionality of the current file -> we should have a `inProgressPrepareForEditorTask` Maybe I’ll understand it once we have this change in. It seems like a worthwhile change in any way. --- Sources/SourceKitLSP/IndexProgressManager.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Sources/SourceKitLSP/IndexProgressManager.swift b/Sources/SourceKitLSP/IndexProgressManager.swift index 17ae4599d..3e9bcb100 100644 --- a/Sources/SourceKitLSP/IndexProgressManager.swift +++ b/Sources/SourceKitLSP/IndexProgressManager.swift @@ -98,7 +98,11 @@ actor IndexProgressManager { // `indexTasksWereScheduled` calls yet but the semantic index managers already track them in their in-progress tasks. // Clip the finished tasks to 0 because showing a negative number there looks stupid. let finishedTasks = max(queuedIndexTasks - indexTasks.count, 0) - message = "\(finishedTasks) / \(queuedIndexTasks)" + if indexTasks.isEmpty { + message = "Preparing targets" + } else { + message = "\(finishedTasks) / \(queuedIndexTasks)" + } if await sourceKitLSPServer.options.experimentalFeatures.contains(.showActivePreparationTasksInProgress) { var inProgressTasks: [String] = [] inProgressTasks += preparationTasks.filter { $0.value == .executing } From 7066784650889e9570643c2675636297959f6107 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sun, 2 Jun 2024 08:12:06 -0700 Subject: [PATCH 46/69] =?UTF-8?q?Don=E2=80=99t=20require=20`--experimental?= =?UTF-8?q?-feature`=20to=20be=20passed=20when=20launching=20`sourcekit-ls?= =?UTF-8?q?p`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/sourcekit-lsp/SourceKitLSP.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index f7aabff37..22531d171 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -205,7 +205,7 @@ struct SourceKitLSP: AsyncParsableCommand { Available features are: \(ExperimentalFeature.allCases.map(\.rawValue).joined(separator: ", ")) """ ) - var experimentalFeatures: [ExperimentalFeature] + var experimentalFeatures: [ExperimentalFeature] = [] func mapOptions() -> SourceKitLSPServer.Options { var serverOptions = SourceKitLSPServer.Options() From b3fab6b19f52638d386c8a1d7aecbea0519c7075 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 22:44:19 -0700 Subject: [PATCH 47/69] Remove `showActivePreparationTasksInProgress` experimental feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `showActivePreparationTasksInProgress` didn’t turn out to be terribly useful and with the streaming index log (https://github.com/apple/sourcekit-lsp/pull/1382), we should have the functionality to view which index tasks are currently running. So, we should remove the feature. rdar://129109830 --- Sources/SourceKitLSP/ExperimentalFeatures.swift | 8 -------- Sources/SourceKitLSP/IndexProgressManager.swift | 15 ++++----------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Sources/SourceKitLSP/ExperimentalFeatures.swift b/Sources/SourceKitLSP/ExperimentalFeatures.swift index ae0a677fd..80aa504f9 100644 --- a/Sources/SourceKitLSP/ExperimentalFeatures.swift +++ b/Sources/SourceKitLSP/ExperimentalFeatures.swift @@ -15,12 +15,4 @@ public enum ExperimentalFeature: String, Codable, Sendable, CaseIterable { /// Enable background indexing. case backgroundIndexing = "background-indexing" - - /// Show the files that are currently being indexed / the targets that are currently being prepared in the work done - /// progress. - /// - /// This is an option because VS Code tries to render a multi-line work done progress into a single line text field in - /// the status bar, which looks broken. But at the same time, it is very useful to get a feeling about what's - /// currently happening indexing-wise. - case showActivePreparationTasksInProgress = "show-active-preparation-tasks-in-progress" } diff --git a/Sources/SourceKitLSP/IndexProgressManager.swift b/Sources/SourceKitLSP/IndexProgressManager.swift index 3e9bcb100..383488c08 100644 --- a/Sources/SourceKitLSP/IndexProgressManager.swift +++ b/Sources/SourceKitLSP/IndexProgressManager.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +import LSPLogging import LanguageServerProtocol import SKCore import SKSupport @@ -100,20 +101,12 @@ actor IndexProgressManager { let finishedTasks = max(queuedIndexTasks - indexTasks.count, 0) if indexTasks.isEmpty { message = "Preparing targets" + if preparationTasks.isEmpty { + logger.fault("Indexer status is 'indexing' but there is no update indexstore or preparation task") + } } else { message = "\(finishedTasks) / \(queuedIndexTasks)" } - if await sourceKitLSPServer.options.experimentalFeatures.contains(.showActivePreparationTasksInProgress) { - var inProgressTasks: [String] = [] - inProgressTasks += preparationTasks.filter { $0.value == .executing } - .map { "- Preparing \($0.key.targetID)" } - .sorted() - inProgressTasks += indexTasks.filter { $0.value == .executing } - .map { "- Indexing \($0.key.fileURL?.lastPathComponent ?? $0.key.pseudoPath)" } - .sorted() - - message += "\n\n" + inProgressTasks.joined(separator: "\n") - } if queuedIndexTasks != 0 { percentage = Int(Double(finishedTasks) / Double(queuedIndexTasks) * 100) } else { From 59672e942e8f956ae45d9c76387e46d5e24baeab Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 08:19:58 -0700 Subject: [PATCH 48/69] Log when a sourcekitd request gets cancelled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I saw a failure in CI where a sourcekitd diagnostic request got cancelled without an LSP cancellation request. I am suspecting that there is some implicit cancellation going on in sourcekitd despite `key.cancel_builds: 0`, which doesn’t make sense to me. Adding some explicit logging to sourcekit-lsp to check if we cancel the request for some reason. --- Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift | 11 ++++++++++- Sources/SourceKitD/SourceKitD.swift | 4 ++++ Tests/SourceKitDTests/SourceKitDRegistryTests.swift | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift index cfa9c46a9..04a5801f5 100644 --- a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift +++ b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift @@ -31,7 +31,6 @@ extension sourcekitd_api_values: @unchecked Sendable {} /// Users of this class should not call the api functions `initialize`, `shutdown`, or /// `set_notification_handler`, which are global state managed internally by this class. public actor DynamicallyLoadedSourceKitD: SourceKitD { - /// The path to the sourcekitd dylib. public let path: AbsolutePath @@ -142,6 +141,16 @@ public actor DynamicallyLoadedSourceKitD: SourceKitD { } } + public nonisolated func logRequestCancellation(request: SKDRequestDictionary) { + // We don't need to log which request has been cancelled because we can associate the cancellation log message with + // the send message via the log + logger.info( + """ + Cancelling sourcekitd request: + \(request.forLogging) + """ + ) + } } struct WeakSKDNotificationHandler: Sendable { diff --git a/Sources/SourceKitD/SourceKitD.swift b/Sources/SourceKitD/SourceKitD.swift index 8aa470541..34bc05ac2 100644 --- a/Sources/SourceKitD/SourceKitD.swift +++ b/Sources/SourceKitD/SourceKitD.swift @@ -59,6 +59,9 @@ public protocol SourceKitD: AnyObject, Sendable { /// This log call is issued during normal operation. It is acceptable for the logger to truncate the log message /// to achieve good performance. func log(response: SKDResponse) + + /// Log that the given request has been cancelled. + func logRequestCancellation(request: SKDRequestDictionary) } public enum SKDError: Error, Equatable { @@ -97,6 +100,7 @@ extension SourceKitD { return handle } cancel: { handle in if let handle { + logRequestCancellation(request: req) api.cancel_request(handle) } } diff --git a/Tests/SourceKitDTests/SourceKitDRegistryTests.swift b/Tests/SourceKitDTests/SourceKitDRegistryTests.swift index 8b43fcff2..508cce551 100644 --- a/Tests/SourceKitDTests/SourceKitDRegistryTests.swift +++ b/Tests/SourceKitDTests/SourceKitDRegistryTests.swift @@ -80,4 +80,5 @@ final class FakeSourceKitD: SourceKitD { public func log(request: SKDRequestDictionary) {} public func log(response: SKDResponse) {} public func log(crashedRequest: SKDRequestDictionary, fileContents: String?) {} + public func logRequestCancellation(request: SKDRequestDictionary) {} } From 63c144226c30449a3bc19ab98770dace5c7a9a7e Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 08:21:26 -0700 Subject: [PATCH 49/69] =?UTF-8?q?Set=20the=20logging=20scope=20for=20notif?= =?UTF-8?q?ication=20handling=20one=20level=20higher=20so=20it=E2=80=99s?= =?UTF-8?q?=20also=20set=20for=20cancel=20notifications?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/SourceKitLSP/SourceKitLSPServer.swift | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index 0c67b93f4..aa77b7fe2 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -532,26 +532,25 @@ private nonisolated(unsafe) var notificationIDForLogging = AtomicUInt32(initialV extension SourceKitLSPServer: MessageHandler { public nonisolated func handle(_ params: some NotificationType) { - if let params = params as? CancelRequestNotification { - // Request cancellation needs to be able to overtake any other message we - // are currently handling. Ordering is not important here. We thus don't - // need to execute it on `messageHandlingQueue`. - self.cancelRequest(params) - } - let notificationID = notificationIDForLogging.fetchAndIncrement() + withLoggingScope("notification-\(notificationID % 100)") { + if let params = params as? CancelRequestNotification { + // Request cancellation needs to be able to overtake any other message we + // are currently handling. Ordering is not important here. We thus don't + // need to execute it on `messageHandlingQueue`. + self.cancelRequest(params) + } - let signposter = Logger(subsystem: LoggingScope.subsystem, category: "message-handling") - .makeSignposter() - let signpostID = signposter.makeSignpostID() - let state = signposter.beginInterval("Notification", id: signpostID, "\(type(of: params))") - messageHandlingQueue.async(metadata: MessageHandlingDependencyTracker(params)) { - signposter.emitEvent("Start handling", id: signpostID) + let signposter = Logger(subsystem: LoggingScope.subsystem, category: "message-handling") + .makeSignposter() + let signpostID = signposter.makeSignpostID() + let state = signposter.beginInterval("Notification", id: signpostID, "\(type(of: params))") + messageHandlingQueue.async(metadata: MessageHandlingDependencyTracker(params)) { + signposter.emitEvent("Start handling", id: signpostID) - // Only use the last two digits of the notification ID for the logging scope to avoid creating too many scopes. - // See comment in `withLoggingScope`. - // The last 2 digits should be sufficient to differentiate between multiple concurrently running notifications. - await withLoggingScope("notification-\(notificationID % 100)") { + // Only use the last two digits of the notification ID for the logging scope to avoid creating too many scopes. + // See comment in `withLoggingScope`. + // The last 2 digits should be sufficient to differentiate between multiple concurrently running notifications. await self.handleImpl(params) signposter.endInterval("Notification", state, "Done") } @@ -2406,7 +2405,7 @@ fileprivate extension RequestID { var numericValue: Int { switch self { case .number(let number): return number - case .string(let string): return Int(string) ?? string.hashValue + case .string(let string): return Int(string) ?? abs(string.hashValue) } } } From f91b63f5f1ace71953a351e7c0d2784a083191e5 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 08:22:44 -0700 Subject: [PATCH 50/69] Set experimental features from the command line in `SourceKitLSPServer.Options` I forgot to actually set the experimental features form the command line in the `SourceKitLSPServer.Options` that get used. --- Sources/sourcekit-lsp/SourceKitLSP.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index 22531d171..d0f317355 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -224,6 +224,7 @@ struct SourceKitLSP: AsyncParsableCommand { serverOptions.indexOptions.indexPrefixMappings = indexPrefixMappings serverOptions.completionOptions.maxResults = completionMaxResults serverOptions.generatedInterfacesPath = generatedInterfacesPath + serverOptions.experimentalFeatures = experimentalFeatures return serverOptions } From c639d2b25a146748a118a276bf3528bafda4ed97 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 09:18:04 -0700 Subject: [PATCH 51/69] Classify deinitializers as constructors in document symbols LSP doesn't have a destructor kind. constructor is the closest match and also what clangd for destructors. --- Sources/SourceKitLSP/Swift/DocumentSymbols.swift | 3 ++- Tests/SourceKitLSPTests/DocumentSymbolTests.swift | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/SourceKitLSP/Swift/DocumentSymbols.swift b/Sources/SourceKitLSP/Swift/DocumentSymbols.swift index 5028d55d9..87e4664c8 100644 --- a/Sources/SourceKitLSP/Swift/DocumentSymbols.swift +++ b/Sources/SourceKitLSP/Swift/DocumentSymbols.swift @@ -171,10 +171,11 @@ fileprivate final class DocumentSymbolsFinder: SyntaxAnyVisitor { } override func visit(_ node: DeinitializerDeclSyntax) -> SyntaxVisitorContinueKind { + // LSP doesn't have a destructor kind. constructor is the closest match and also what clangd for destructors. return record( node: node, name: node.deinitKeyword.text, - symbolKind: .null, + symbolKind: .constructor, range: node.rangeWithoutTrivia, selection: node.deinitKeyword.rangeWithoutTrivia ) diff --git a/Tests/SourceKitLSPTests/DocumentSymbolTests.swift b/Tests/SourceKitLSPTests/DocumentSymbolTests.swift index c6f5afc62..500ef1d75 100644 --- a/Tests/SourceKitLSPTests/DocumentSymbolTests.swift +++ b/Tests/SourceKitLSPTests/DocumentSymbolTests.swift @@ -748,7 +748,7 @@ final class DocumentSymbolTests: XCTestCase { children: [ DocumentSymbol( name: "deinit", - kind: .null, + kind: .constructor, range: positions["4️⃣"].. Date: Sun, 2 Jun 2024 08:38:22 -0700 Subject: [PATCH 52/69] Share module cache between test projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This improves serial test execution time of eg. `DocumentTestDiscoveryTests` from 36s to 22s because we don’t need to re-build the XCTest module from its interface when using an open source toolchain. This also uncovered that we weren‘t passing the build setup flags to the prepare command. rdar://126493151 --- .../SwiftPMBuildSystem.swift | 24 +++++++++ .../IndexedSingleSwiftFileTestProject.swift | 5 ++ .../SKTestSupport/SwiftPMTestProject.swift | 7 ++- .../TestSourceKitLSPClient.swift | 19 +------ Sources/SKTestSupport/Utils.swift | 12 +++++ .../BackgroundIndexingTests.swift | 49 +++++++++++++++++++ .../SwiftInterfaceTests.swift | 7 +-- Utilities/build-script-helper.py | 21 ++++---- 8 files changed, 111 insertions(+), 33 deletions(-) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index c3a24b451..f789be325 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -574,6 +574,30 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { "--disable-index-store", "--target", target.targetID, ] + if self.toolsBuildParameters.configuration != self.destinationBuildParameters.configuration { + logger.fault( + """ + Preparation is assuming that tools and destination are built using the same configuration, \ + got tools: \(String(describing: self.toolsBuildParameters.configuration), privacy: .public), \ + destination: \(String(describing: self.destinationBuildParameters.configuration), privacy: .public) + """ + ) + } + arguments += ["-c", self.destinationBuildParameters.configuration.rawValue] + if self.toolsBuildParameters.flags != self.destinationBuildParameters.flags { + logger.fault( + """ + Preparation is assuming that tools and destination are built using the same build flags, \ + got tools: \(String(describing: self.toolsBuildParameters.flags)), \ + destination: \(String(describing: self.destinationBuildParameters.configuration)) + """ + ) + } + arguments += self.destinationBuildParameters.flags.cCompilerFlags.flatMap { ["-Xcc", $0] } + arguments += self.destinationBuildParameters.flags.cxxCompilerFlags.flatMap { ["-Xcxx", $0] } + arguments += self.destinationBuildParameters.flags.swiftCompilerFlags.flatMap { ["-Xswiftc", $0] } + arguments += self.destinationBuildParameters.flags.linkerFlags.flatMap { ["-Xlinker", $0] } + arguments += self.destinationBuildParameters.flags.xcbuildFlags?.flatMap { ["-Xxcbuild", $0] } ?? [] if await swiftBuildSupportsPrepareForIndexing { arguments.append("--experimental-prepare-for-indexing") } diff --git a/Sources/SKTestSupport/IndexedSingleSwiftFileTestProject.swift b/Sources/SKTestSupport/IndexedSingleSwiftFileTestProject.swift index 9a2d3b833..25299cc09 100644 --- a/Sources/SKTestSupport/IndexedSingleSwiftFileTestProject.swift +++ b/Sources/SKTestSupport/IndexedSingleSwiftFileTestProject.swift @@ -61,6 +61,11 @@ public struct IndexedSingleSwiftFileTestProject { "-index-store-path", indexURL.path, "-typecheck", ] + if let globalModuleCache { + compilerArguments += [ + "-module-cache-path", globalModuleCache.path, + ] + } if !indexSystemModules { compilerArguments.append("-index-ignore-system-modules") } diff --git a/Sources/SKTestSupport/SwiftPMTestProject.swift b/Sources/SKTestSupport/SwiftPMTestProject.swift index ba9dae445..2c186b4ee 100644 --- a/Sources/SKTestSupport/SwiftPMTestProject.swift +++ b/Sources/SKTestSupport/SwiftPMTestProject.swift @@ -209,7 +209,7 @@ public class SwiftPMTestProject: MultiFileTestProject { guard let swift = await ToolchainRegistry.forTesting.default?.swift?.asURL else { throw Error.swiftNotFound } - let arguments = [ + var arguments = [ swift.path, "build", "--package-path", path.path, @@ -217,6 +217,11 @@ public class SwiftPMTestProject: MultiFileTestProject { "-Xswiftc", "-index-ignore-system-modules", "-Xcc", "-index-ignore-system-symbols", ] + if let globalModuleCache { + arguments += [ + "-Xswiftc", "-module-cache-path", "-Xswiftc", globalModuleCache.path, + ] + } var environment = ProcessEnv.block // FIXME: SwiftPM does not index-while-building on non-Darwin platforms for C-family files (rdar://117744039). // Force-enable index-while-building with the environment variable. diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index eedf51d85..0b33438ce 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -44,12 +44,6 @@ public final class TestSourceKitLSPClient: MessageHandler { /// `nonisolated(unsafe)` is fine because `nextRequestID` is atomic. private nonisolated(unsafe) var nextRequestID = AtomicUInt32(initialValue: 0) - /// If the server is not using the global module cache, the path of the local - /// module cache. - /// - /// This module cache will be deleted when the test server is destroyed. - private let moduleCache: URL? - /// The server that handles the requests. public let server: SourceKitLSPServer @@ -102,7 +96,6 @@ public final class TestSourceKitLSPClient: MessageHandler { /// needed. public init( serverOptions: SourceKitLSPServer.Options = .testDefault, - useGlobalModuleCache: Bool = true, initialize: Bool = true, initializationOptions: LSPAny? = nil, capabilities: ClientCapabilities = ClientCapabilities(), @@ -112,14 +105,9 @@ public final class TestSourceKitLSPClient: MessageHandler { preInitialization: ((TestSourceKitLSPClient) -> Void)? = nil, cleanUp: @Sendable @escaping () -> Void = {} ) async throws { - if !useGlobalModuleCache { - moduleCache = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(UUID().uuidString) - } else { - moduleCache = nil - } var serverOptions = serverOptions - if let moduleCache { - serverOptions.buildSetup.flags.swiftCompilerFlags += ["-module-cache-path", moduleCache.path] + if let globalModuleCache { + serverOptions.buildSetup.flags.swiftCompilerFlags += ["-module-cache-path", globalModuleCache.path] } if enableBackgroundIndexing { serverOptions.experimentalFeatures.append(.backgroundIndexing) @@ -191,9 +179,6 @@ public final class TestSourceKitLSPClient: MessageHandler { sema.wait() self.send(ExitNotification()) - if let moduleCache { - try? FileManager.default.removeItem(at: moduleCache) - } cleanUp() } diff --git a/Sources/SKTestSupport/Utils.swift b/Sources/SKTestSupport/Utils.swift index 8d7b1fd00..9c0b2d67a 100644 --- a/Sources/SKTestSupport/Utils.swift +++ b/Sources/SKTestSupport/Utils.swift @@ -112,3 +112,15 @@ fileprivate extension URL { #endif } } + +var globalModuleCache: URL? = { + if let customModuleCache = ProcessInfo.processInfo.environment["SOURCEKIT_LSP_TEST_MODULE_CACHE"] { + if customModuleCache.isEmpty { + return nil + } + return URL(fileURLWithPath: customModuleCache) + } + return FileManager.default.temporaryDirectory.realpath + .appendingPathComponent("sourcekit-lsp-test-scratch") + .appendingPathComponent("shared-module-cache") +}() diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index 6e5ed554e..d907bc669 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -928,4 +928,53 @@ final class BackgroundIndexingTests: XCTestCase { satisfying: { $0.message.contains("Preparing") } ) } + + func testUseBuildFlagsDuringPreparation() async throws { + var serverOptions = SourceKitLSPServer.Options.testDefault + serverOptions.buildSetup.flags.swiftCompilerFlags += ["-D", "MY_FLAG"] + let project = try await SwiftPMTestProject( + files: [ + "Lib/Lib.swift": """ + #if MY_FLAG + public func foo() -> Int { 1 } + #endif + """, + "Client/Client.swift": """ + import Lib + + func test() -> String { + return foo() + } + """, + ], + manifest: """ + let package = Package( + name: "MyLibrary", + targets: [ + .target(name: "Lib"), + .target(name: "Client", dependencies: ["Lib"]), + ] + ) + """, + serverOptions: serverOptions, + enableBackgroundIndexing: true + ) + + // Check that we get an error about the return type of `foo` (`Int`) not being convertible to the return type of + // `test` (`String`), which indicates that `Lib` had `foo` and was thus compiled with `-D MY_FLAG` + let (uri, _) = try project.openDocument("Client.swift") + let diagnostics = try await project.testClient.send( + DocumentDiagnosticsRequest(textDocument: TextDocumentIdentifier(uri)) + ) + guard case .full(let diagnostics) = diagnostics else { + XCTFail("Expected full diagnostics report") + return + } + XCTAssert( + diagnostics.items.contains(where: { + $0.message == "Cannot convert return expression of type 'Int' to return type 'String'" + }), + "Did not get expected diagnostic: \(diagnostics)" + ) + } } diff --git a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift index 63f055592..6d3bf90ff 100644 --- a/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift +++ b/Tests/SourceKitLSPTests/SwiftInterfaceTests.swift @@ -22,12 +22,7 @@ import XCTest final class SwiftInterfaceTests: XCTestCase { func testSystemModuleInterface() async throws { - // This is the only test that references modules from the SDK (Foundation). - // `testSystemModuleInterface` has been flaky for a long while and a - // hypothesis is that it was failing because of a malformed global module - // cache that might still be present from previous CI runs. If we use a - // local module cache, we define away that source of bugs. - let testClient = try await TestSourceKitLSPClient(useGlobalModuleCache: false) + let testClient = try await TestSourceKitLSPClient() let url = URL(fileURLWithPath: "/\(UUID())/a.swift") let uri = DocumentURI(url) diff --git a/Utilities/build-script-helper.py b/Utilities/build-script-helper.py index 2d50aafba..b4e0ba59c 100755 --- a/Utilities/build-script-helper.py +++ b/Utilities/build-script-helper.py @@ -7,6 +7,7 @@ import shutil import subprocess import sys +import tempfile from typing import Dict, List @@ -228,15 +229,17 @@ def run_tests(swift_exec: str, args: argparse.Namespace) -> None: '--test-product', 'SourceKitLSPPackageTests' ] + swiftpm_args - # Try running tests in parallel. If that fails, run tests in serial to get capture more readable output. - try: - check_call(cmd + ['--parallel'], additional_env=additional_env, verbose=args.verbose) - except: - print('--- Running tests in parallel failed. Re-running tests serially to capture more actionable output.') - sys.stdout.flush() - check_call(cmd, additional_env=additional_env, verbose=args.verbose) - # Return with non-zero exit code even if serial test execution succeeds. - raise SystemExit(1) + with tempfile.TemporaryDirectory() as test_module_cache: + additional_env['SOURCEKIT_LSP_TEST_MODULE_CACHE'] = f"{test_module_cache}/module-cache" + # Try running tests in parallel. If that fails, run tests in serial to get capture more readable output. + try: + check_call(cmd + ['--parallel'], additional_env=additional_env, verbose=args.verbose) + except: + print('--- Running tests in parallel failed. Re-running tests serially to capture more actionable output.') + sys.stdout.flush() + check_call(cmd, additional_env=additional_env, verbose=args.verbose) + # Return with non-zero exit code even if serial test execution succeeds. + raise SystemExit(1) def install_binary(exe: str, source_dir: str, install_dir: str, verbose: bool) -> None: From b2b383614c9b528953913b2686dbfb42875b638c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 09:41:27 -0700 Subject: [PATCH 53/69] Fix a typo --- Tests/SourceKitLSPTests/BackgroundIndexingTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index 6e5ed554e..3d6d3d572 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -869,7 +869,7 @@ final class BackgroundIndexingTests: XCTestCase { func testImportPreparedModuleWithFunctionBodiesSkipped() async throws { try await SkipUnless.sourcekitdSupportsRename() // This test case was crashing the indexing compiler invocation for Client if Lib was built for index preparation - // (using `-enable-library-evolution -experimental-skip-all-function-bodies -experimental-lazy-typecheck`) but x + // (using `-enable-library-evolution -experimental-skip-all-function-bodies -experimental-lazy-typecheck`) but the // Client was not indexed with `-experimental-allow-module-with-compiler-errors`. rdar://129071600 let project = try await SwiftPMTestProject( files: [ From c7bf59e2ee3a45128f319dc09356ceed7f805087 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 11:15:27 -0700 Subject: [PATCH 54/69] Require `ThreadSafeBox.T` to be Sendable Otherwise, I think `ThreadSafeBox` might still have data races. This also requires us to make `TestSourceKitLSPClient.RequestHandler` sendable. rdar://128572489 --- Sources/SKSupport/AsyncUtils.swift | 2 +- Sources/SKSupport/ThreadSafeBox.swift | 7 ++++++- Sources/SKTestSupport/TestSourceKitLSPClient.swift | 4 ++-- Sources/SourceKitD/SourceKitD.swift | 6 ++++++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Sources/SKSupport/AsyncUtils.swift b/Sources/SKSupport/AsyncUtils.swift index 216463fb1..0cfb2a94c 100644 --- a/Sources/SKSupport/AsyncUtils.swift +++ b/Sources/SKSupport/AsyncUtils.swift @@ -98,7 +98,7 @@ public extension Task where Failure == Never { /// /// If the task executing `withCancellableCheckedThrowingContinuation` gets /// cancelled, `cancel` is invoked with the handle that `operation` provided. -public func withCancellableCheckedThrowingContinuation( +public func withCancellableCheckedThrowingContinuation( _ operation: (_ continuation: CheckedContinuation) -> Handle, cancel: @Sendable (Handle) -> Void ) async throws -> Result { diff --git a/Sources/SKSupport/ThreadSafeBox.swift b/Sources/SKSupport/ThreadSafeBox.swift index 5f56b8c66..a4f825eaf 100644 --- a/Sources/SKSupport/ThreadSafeBox.swift +++ b/Sources/SKSupport/ThreadSafeBox.swift @@ -24,7 +24,7 @@ extension NSLock { /// A thread safe container that contains a value of type `T`. /// /// - Note: Unchecked sendable conformance because value is guarded by a lock. -public class ThreadSafeBox: @unchecked Sendable { +public class ThreadSafeBox: @unchecked Sendable { /// Lock guarding `_value`. private let lock = NSLock() @@ -41,6 +41,11 @@ public class ThreadSafeBox: @unchecked Sendable { _value = newValue } } + _modify { + lock.lock() + defer { lock.unlock() } + yield &_value + } } public init(initialValue: T) { diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index 83915ad10..299d5ff37 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -34,7 +34,7 @@ extension SourceKitLSPServer.Options { /// that the server sends to the client. public final class TestSourceKitLSPClient: MessageHandler { /// A function that takes a request and returns the request's response. - public typealias RequestHandler = (Request) -> Request.Response + public typealias RequestHandler = @Sendable (Request) -> Request.Response /// The ID that should be assigned to the next request sent to the `server`. /// `nonisolated(unsafe)` is fine because `nextRequestID` is atomic. @@ -72,7 +72,7 @@ public final class TestSourceKitLSPClient: MessageHandler { /// /// `isOneShort` if the request handler should only serve a single request and should be removed from /// `requestHandlers` after it has been called. - private nonisolated(unsafe) var requestHandlers: ThreadSafeBox<[(requestHandler: Any, isOneShot: Bool)]> = + private nonisolated(unsafe) var requestHandlers: ThreadSafeBox<[(requestHandler: Sendable, isOneShot: Bool)]> = ThreadSafeBox(initialValue: []) /// A closure that is called when the `TestSourceKitLSPClient` is destructed. diff --git a/Sources/SourceKitD/SourceKitD.swift b/Sources/SourceKitD/SourceKitD.swift index 8aa470541..a58b1b3a2 100644 --- a/Sources/SourceKitD/SourceKitD.swift +++ b/Sources/SourceKitD/SourceKitD.swift @@ -15,6 +15,12 @@ import Dispatch import Foundation import SKSupport +#if compiler(>=6) +extension sourcekitd_api_request_handle_t: @retroactive @unchecked Sendable {} +#else +extension sourcekitd_api_request_handle_t: @unchecked Sendable {} +#endif + /// Access to sourcekitd API, taking care of initialization, shutdown, and notification handler /// multiplexing. /// From 3c5d8b91192d5ae035d91c90227621d48679d479 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 12:40:31 -0700 Subject: [PATCH 55/69] Change `SwiftPMBuildSystem` to use `DocumentURI` instead of `URL` --- .../SwiftPMBuildSystem.swift | 68 ++++---- Sources/SourceKitLSP/CreateBuildSystem.swift | 8 +- .../SwiftPMBuildSystemTests.swift | 146 +++++++++++++----- 3 files changed, 152 insertions(+), 70 deletions(-) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index 19bf8a197..236b9b225 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -146,8 +146,8 @@ public actor SwiftPMBuildSystem { } } - var fileToTarget: [AbsolutePath: SwiftBuildTarget] = [:] - var sourceDirToTarget: [AbsolutePath: SwiftBuildTarget] = [:] + var fileToTarget: [DocumentURI: SwiftBuildTarget] = [:] + var sourceDirToTarget: [DocumentURI: SwiftBuildTarget] = [:] /// Maps configured targets ids to their SwiftPM build target as well as an index in their topological sorting. /// @@ -286,15 +286,18 @@ public actor SwiftPMBuildSystem { /// - reloadPackageStatusCallback: Will be informed when `reloadPackage` starts and ends executing. /// - Returns: nil if `workspacePath` is not part of a package or there is an error. public init?( - url: URL, + uri: DocumentURI, toolchainRegistry: ToolchainRegistry, buildSetup: BuildSetup, isForIndexBuild: Bool, reloadPackageStatusCallback: @escaping (ReloadPackageStatus) async -> Void ) async { + guard let fileURL = uri.fileURL else { + return nil + } do { try await self.init( - workspacePath: try TSCAbsolutePath(validating: url.path), + workspacePath: try TSCAbsolutePath(validating: fileURL.path), toolchainRegistry: toolchainRegistry, fileSystem: localFileSystem, buildSetup: buildSetup, @@ -304,7 +307,7 @@ public actor SwiftPMBuildSystem { } catch Error.noManifest { return nil } catch { - logger.error("failed to create SwiftPMWorkspace at \(url.path): \(error.forLogging)") + logger.error("failed to create SwiftPMWorkspace at \(uri.forLogging): \(error.forLogging)") return nil } } @@ -351,13 +354,13 @@ extension SwiftPMBuildSystem { } ) - self.fileToTarget = [AbsolutePath: SwiftBuildTarget]( + self.fileToTarget = [DocumentURI: SwiftBuildTarget]( modulesGraph.allTargets.flatMap { target in return target.sources.paths.compactMap { guard let buildTarget = buildDescription.getBuildTarget(for: target, in: modulesGraph) else { return nil } - return (key: $0, value: buildTarget) + return (key: DocumentURI($0.asURL), value: buildTarget) } }, uniquingKeysWith: { td, _ in @@ -366,12 +369,12 @@ extension SwiftPMBuildSystem { } ) - self.sourceDirToTarget = [AbsolutePath: SwiftBuildTarget]( - modulesGraph.allTargets.compactMap { (target) -> (AbsolutePath, SwiftBuildTarget)? in + self.sourceDirToTarget = [DocumentURI: SwiftBuildTarget]( + modulesGraph.allTargets.compactMap { (target) -> (DocumentURI, SwiftBuildTarget)? in guard let buildTarget = buildDescription.getBuildTarget(for: target, in: modulesGraph) else { return nil } - return (key: target.sources.root, value: buildTarget) + return (key: DocumentURI(target.sources.root.asURL), value: buildTarget) }, uniquingKeysWith: { td, _ in // FIXME: is there a preferred target? @@ -390,6 +393,13 @@ extension SwiftPMBuildSystem { } } +fileprivate struct NonFileURIError: Error, CustomStringConvertible { + let uri: DocumentURI + var description: String { + "Trying to get build settings for non-file URI: \(uri)" + } +} + extension SwiftPMBuildSystem: SKCore.BuildSystem { public nonisolated var supportsPreparation: Bool { true } @@ -410,8 +420,11 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { /// Return the compiler arguments for the given source file within a target, making any necessary adjustments to /// account for differences in the SwiftPM versions being linked into SwiftPM and being installed in the toolchain. - private func compilerArguments(for file: URL, in buildTarget: any SwiftBuildTarget) async throws -> [String] { - let compileArguments = try buildTarget.compileArguments(for: file) + private func compilerArguments(for file: DocumentURI, in buildTarget: any SwiftBuildTarget) async throws -> [String] { + guard let fileURL = file.fileURL else { + throw NonFileURIError(uri: file) + } + let compileArguments = try buildTarget.compileArguments(for: fileURL) #if compiler(>=6.1) #warning("When we drop support for Swift 5.10 we no longer need to adjust compiler arguments for the Modules move") @@ -449,9 +462,10 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { return nil } - if !buildTarget.sources.contains(url), + if !buildTarget.sources.lazy.map(DocumentURI.init).contains(uri), let substituteFile = buildTarget.sources.sorted(by: { $0.path < $1.path }).first { + logger.info("Getting compiler arguments for \(url) using substitute file \(substituteFile)") // If `url` is not part of the target's source, it's most likely a header file. Fake compiler arguments for it // from a substitute file within the target. // Even if the file is not a header, this should give reasonable results: Say, there was a new `.cpp` file in a @@ -460,13 +474,13 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { // getting its compiler arguments and then patching up the compiler arguments by replacing the substitute file // with the `.cpp` file. return FileBuildSettings( - compilerArguments: try await compilerArguments(for: substituteFile, in: buildTarget), + compilerArguments: try await compilerArguments(for: DocumentURI(substituteFile), in: buildTarget), workingDirectory: workspacePath.pathString ).patching(newFile: try resolveSymlinks(path).pathString, originalFile: substituteFile.absoluteString) } return FileBuildSettings( - compilerArguments: try await compilerArguments(for: url, in: buildTarget), + compilerArguments: try await compilerArguments(for: uri, in: buildTarget), workingDirectory: workspacePath.pathString ) } @@ -483,7 +497,7 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { return [] } - if let target = try? buildTarget(for: path) { + if let target = buildTarget(for: uri) { return [ConfiguredTarget(target)] } @@ -626,13 +640,15 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { } /// Returns the resolved target description for the given file, if one is known. - private func buildTarget(for file: AbsolutePath) throws -> SwiftBuildTarget? { + private func buildTarget(for file: DocumentURI) -> SwiftBuildTarget? { if let td = fileToTarget[file] { return td } - let realpath = try resolveSymlinks(file) - if realpath != file, let td = fileToTarget[realpath] { + if let fileURL = file.fileURL, + let realpath = try? resolveSymlinks(AbsolutePath(validating: fileURL.path)), + let td = fileToTarget[DocumentURI(realpath.asURL)] + { fileToTarget[file] = td return td } @@ -675,11 +691,7 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { // If a Swift file within a target is updated, reload all the other files within the target since they might be // referring to a function in the updated file. for event in events { - guard let url = event.uri.fileURL, - url.pathExtension == "swift", - let absolutePath = try? AbsolutePath(validating: url.path), - let target = fileToTarget[absolutePath] - else { + guard event.uri.fileURL?.pathExtension == "swift", let target = fileToTarget[event.uri] else { continue } filesWithUpdatedDependencies.formUnion(target.sources.map { DocumentURI($0) }) @@ -695,7 +707,7 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { // If we have background indexing enabled, this is not necessary because we call `fileDependenciesUpdated` when // preparation of a target finishes. if !isForIndexBuild, events.contains(where: { $0.uri.fileURL?.pathExtension == "swiftmodule" }) { - filesWithUpdatedDependencies.formUnion(self.fileToTarget.keys.map { DocumentURI($0.asURL) }) + filesWithUpdatedDependencies.formUnion(self.fileToTarget.keys) } await self.fileDependenciesUpdatedDebouncer.scheduleCall(filesWithUpdatedDependencies) } @@ -708,11 +720,11 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { } public func sourceFiles() -> [SourceFileInfo] { - return fileToTarget.compactMap { (path, target) -> SourceFileInfo? in + return fileToTarget.compactMap { (uri, target) -> SourceFileInfo? in // We should only set mayContainTests to `true` for files from test targets // (https://github.com/apple/sourcekit-lsp/issues/1174). return SourceFileInfo( - uri: DocumentURI(path.asURL), + uri: uri, isPartOfRootProject: target.isPartOfRootPackage, mayContainTests: true ) @@ -753,7 +765,7 @@ extension SwiftPMBuildSystem { func impl(_ path: AbsolutePath) throws -> ConfiguredTarget? { var dir = path.parentDirectory while !dir.isRoot { - if let buildTarget = sourceDirToTarget[dir] { + if let buildTarget = sourceDirToTarget[DocumentURI(dir.asURL)] { return ConfiguredTarget(buildTarget) } dir = dir.parentDirectory diff --git a/Sources/SourceKitLSP/CreateBuildSystem.swift b/Sources/SourceKitLSP/CreateBuildSystem.swift index 9eb963419..73b92f08a 100644 --- a/Sources/SourceKitLSP/CreateBuildSystem.swift +++ b/Sources/SourceKitLSP/CreateBuildSystem.swift @@ -33,9 +33,9 @@ func createBuildSystem( ) return nil } - func createSwiftPMBuildSystem(rootUrl: URL) async -> SwiftPMBuildSystem? { + func createSwiftPMBuildSystem(rootUri: DocumentURI) async -> SwiftPMBuildSystem? { return await SwiftPMBuildSystem( - url: rootUrl, + uri: rootUri, toolchainRegistry: toolchainRegistry, buildSetup: options.buildSetup, isForIndexBuild: options.indexOptions.enableBackgroundIndexing, @@ -58,14 +58,14 @@ func createBuildSystem( switch options.buildSetup.defaultWorkspaceType { case .buildServer: await createBuildServerBuildSystem(rootPath: rootPath) case .compilationDatabase: createCompilationDatabaseBuildSystem(rootPath: rootPath) - case .swiftPM: await createSwiftPMBuildSystem(rootUrl: rootUrl) + case .swiftPM: await createSwiftPMBuildSystem(rootUri: rootUri) case nil: nil } if let defaultBuildSystem { return defaultBuildSystem } else if let buildServer = await createBuildServerBuildSystem(rootPath: rootPath) { return buildServer - } else if let swiftpm = await createSwiftPMBuildSystem(rootUrl: rootUrl) { + } else if let swiftpm = await createSwiftPMBuildSystem(rootUri: rootUri) { return swiftpm } else if let compdb = createCompilationDatabaseBuildSystem(rootPath: rootPath) { return compdb diff --git a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift index c1a426152..89f0f5b8d 100644 --- a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift +++ b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift @@ -97,8 +97,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + let package = Package( + name: "a", + targets: [.target(name: "lib")] + ) """, ] ) @@ -126,8 +128,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + let package = Package( + name: "a", + targets: [.target(name: "lib")] + ) """, ] ) @@ -179,6 +183,61 @@ final class SwiftPMBuildSystemTests: XCTestCase { } } + func testCompilerArgumentsForFileThatContainsPlusCharacterURLEncoded() async throws { + try await withTestScratchDir { tempDir in + try localFileSystem.createFiles( + root: tempDir, + files: [ + "pkg/Sources/lib/a.swift": "", + "pkg/Sources/lib/a+something.swift": "", + "pkg/Package.swift": """ + // swift-tools-version:4.2 + import PackageDescription + let package = Package( + name: "a", + targets: [.target(name: "lib")] + ) + """, + ] + ) + let packageRoot = try resolveSymlinks(tempDir.appending(component: "pkg")) + let tr = ToolchainRegistry.forTesting + let swiftpmBuildSystem = try await SwiftPMBuildSystem( + workspacePath: packageRoot, + toolchainRegistry: tr, + fileSystem: localFileSystem, + buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, + isForIndexBuild: false + ) + try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) + + let aPlusSomething = packageRoot.appending(components: "Sources", "lib", "a+something.swift") + let hostTriple = await swiftpmBuildSystem.destinationBuildParameters.triple + let build = buildPath(root: packageRoot, platform: hostTriple.platformBuildPathComponent) + + assertEqual(await swiftpmBuildSystem.buildPath, build) + assertNotNil(await swiftpmBuildSystem.indexStorePath) + let arguments = try await unwrap( + swiftpmBuildSystem.buildSettings( + for: DocumentURI(URL(string: "file://\(aPlusSomething.asURL.path.replacing("+", with: "%2B"))")!), + language: .swift + ) + ) + .compilerArguments + + // Check that we have both source files in the compiler arguments, which means that we didn't compute the compiler + // arguments for a+something.swift using substitute arguments from a.swift. + XCTAssert( + arguments.contains(aPlusSomething.pathString), + "Compiler arguments do not contain a+something.swift: \(arguments)" + ) + XCTAssert( + arguments.contains(packageRoot.appending(components: "Sources", "lib", "a.swift").pathString), + "Compiler arguments do not contain a.swift: \(arguments)" + ) + } + } + func testBuildSetup() async throws { let fs = localFileSystem try await withTestScratchDir { tempDir in @@ -189,8 +248,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + let package = Package( + name: "a", + targets: [.target(name: "lib")] + ) """, ] ) @@ -237,8 +298,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + let package = Package( + name: "a", + targets: [.target(name: "lib")] + ) """, ] ) @@ -273,8 +336,9 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + let package = Package(name: "a", + targets: [.target(name: "lib")] + ) """, ] ) @@ -316,12 +380,14 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], + let package = Package( + name: "a", targets: [ .target(name: "libA", dependencies: ["libB", "libC"]), - .target(name: "libB", dependencies: []), - .target(name: "libC", dependencies: []), - ]) + .target(name: "libB"), + .target(name: "libC"), + ] + ) """, ] ) @@ -383,10 +449,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [ - .target(name: "libA", dependencies: []), - ]) + let package = Package( + name: "a", + targets: [.target(name: "libA")] + ) """, ] ) @@ -426,9 +492,11 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])], - cxxLanguageStandard: .cxx14) + let package = Package( + name: "a", + targets: [.target(name: "lib")], + cxxLanguageStandard: .cxx14 + ) """, ] ) @@ -509,8 +577,8 @@ final class SwiftPMBuildSystemTests: XCTestCase { import PackageDescription let package = Package(name: "a", platforms: [.macOS(.v10_13)], - products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + targets: [.target(name: "lib")] + ) """, ] ) @@ -552,8 +620,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg_real/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + let package = Package( + name: "a", + targets: [.target(name: "lib")] + ) """, ] ) @@ -613,9 +683,11 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg_real/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])], - cxxLanguageStandard: .cxx14) + let package = Package( + name: "a", + targets: [.target(name: "lib")], + cxxLanguageStandard: .cxx14 + ) """, ] ) @@ -662,12 +734,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:5.3 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [ - .target( - name: "lib", - dependencies: [], - resources: [.copy("a.txt")])]) + let package = Package( + name: "a", + targets: [.target(name: "lib", resources: [.copy("a.txt")])] + ) """, ] ) @@ -705,8 +775,10 @@ final class SwiftPMBuildSystemTests: XCTestCase { "pkg/Package.swift": """ // swift-tools-version:4.2 import PackageDescription - let package = Package(name: "a", products: [], dependencies: [], - targets: [.target(name: "lib", dependencies: [])]) + let package = Package( + name: "a", + targets: [.target(name: "lib")] + ) """, ] ) @@ -737,8 +809,6 @@ final class SwiftPMBuildSystemTests: XCTestCase { import PackageDescription let package = Package( name: "a", - products: [], - dependencies: [], targets: [ .target(name: "lib"), .plugin(name: "MyPlugin", capability: .buildTool) From c03d80d037676678c59ae0ea40c3fd78ae84d6fe Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 12:45:54 -0700 Subject: [PATCH 56/69] Migrate `Rename.swift` to use `DocumentURI` instead of `URL` --- .../IndexStoreDB+MainFilesProvider.swift | 2 +- Sources/SourceKitLSP/Rename.swift | 32 ++++++++++--------- .../SymbolLocation+DocumentURI.swift | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Sources/SourceKitLSP/IndexStoreDB+MainFilesProvider.swift b/Sources/SourceKitLSP/IndexStoreDB+MainFilesProvider.swift index 1d7c0f11b..3ec3e568d 100644 --- a/Sources/SourceKitLSP/IndexStoreDB+MainFilesProvider.swift +++ b/Sources/SourceKitLSP/IndexStoreDB+MainFilesProvider.swift @@ -24,7 +24,7 @@ extension UncheckedIndex { mainFiles = Set( mainFilePaths .filter { FileManager.default.fileExists(atPath: $0) } - .map({ DocumentURI(URL(fileURLWithPath: $0, isDirectory: false)) }) + .map({ DocumentURI(filePath: $0, isDirectory: false) }) ) } else { mainFiles = [] diff --git a/Sources/SourceKitLSP/Rename.swift b/Sources/SourceKitLSP/Rename.swift index 5814a2c78..f87741630 100644 --- a/Sources/SourceKitLSP/Rename.swift +++ b/Sources/SourceKitLSP/Rename.swift @@ -730,24 +730,27 @@ extension SourceKitLSPServer { // If we have a USR + old name, perform an index lookup to find workspace-wide symbols to rename. // First, group all occurrences of that USR by the files they occur in. - var locationsByFile: [URL: [RenameLocation]] = [:] + var locationsByFile: [DocumentURI: [RenameLocation]] = [:] actor LanguageServerTypesCache { let index: UncheckedIndex - var languageServerTypesCache: [URL: LanguageServerType?] = [:] + var languageServerTypesCache: [DocumentURI: LanguageServerType?] = [:] init(index: UncheckedIndex) { self.index = index } - func languageServerType(for url: URL) -> LanguageServerType? { - if let cachedValue = languageServerTypesCache[url] { + func languageServerType(for uri: DocumentURI) -> LanguageServerType? { + if let cachedValue = languageServerTypesCache[uri] { return cachedValue } - let serverType = LanguageServerType( - symbolProvider: index.symbolProvider(for: url.path) - ) - languageServerTypesCache[url] = serverType + let serverType: LanguageServerType? = + if let fileURL = uri.fileURL { + LanguageServerType(symbolProvider: index.symbolProvider(for: fileURL.path)) + } else { + nil + } + languageServerTypesCache[uri] = serverType return serverType } } @@ -757,17 +760,17 @@ extension SourceKitLSPServer { let usrsToRename = overridingAndOverriddenUsrs(of: usr, index: index) let occurrencesToRename = usrsToRename.flatMap { index.occurrences(ofUSR: $0, roles: renameRoles) } for occurrence in occurrencesToRename { - let url = URL(fileURLWithPath: occurrence.location.path) + let uri = occurrence.location.documentUri // Determine whether we should add the location produced by the index to those that will be renamed, or if it has // already been handled by the set provided by the AST. - if changes[DocumentURI(url)] != nil { + if changes[uri] != nil { if occurrence.symbol.usr == usr { // If the language server's rename function already produced AST-based locations for this symbol, no need to // perform an indexed rename for it. continue } - switch await languageServerTypesCache.languageServerType(for: url) { + switch await languageServerTypesCache.languageServerType(for: uri) { case .swift: // sourcekitd only produces AST-based results for the direct calls to this USR. This is because the Swift // AST only has upwards references to superclasses and overridden methods, not the other way round. It is @@ -788,17 +791,16 @@ extension SourceKitLSPServer { utf8Column: occurrence.location.utf8Column, usage: RenameLocation.Usage(roles: occurrence.roles) ) - locationsByFile[url, default: []].append(renameLocation) + locationsByFile[uri, default: []].append(renameLocation) } // Now, call `editsToRename(locations:in:oldName:newName:)` on the language service to convert these ranges into // edits. let urisAndEdits = await locationsByFile - .concurrentMap { (url: URL, renameLocations: [RenameLocation]) -> (DocumentURI, [TextEdit])? in - let uri = DocumentURI(url) + .concurrentMap { (uri: DocumentURI, renameLocations: [RenameLocation]) -> (DocumentURI, [TextEdit])? in let language: Language - switch await languageServerTypesCache.languageServerType(for: url) { + switch await languageServerTypesCache.languageServerType(for: uri) { case .clangd: // Technically, we still don't know the language of the source file but defaulting to C is sufficient to // ensure we get the clang toolchain language server, which is all we care about. diff --git a/Sources/SourceKitLSP/SymbolLocation+DocumentURI.swift b/Sources/SourceKitLSP/SymbolLocation+DocumentURI.swift index 25dc797ac..0496d0744 100644 --- a/Sources/SourceKitLSP/SymbolLocation+DocumentURI.swift +++ b/Sources/SourceKitLSP/SymbolLocation+DocumentURI.swift @@ -15,6 +15,6 @@ import LanguageServerProtocol extension SymbolLocation { var documentUri: DocumentURI { - return DocumentURI(URL(fileURLWithPath: self.path)) + return DocumentURI(filePath: self.path, isDirectory: false) } } From 1ce51ebdc14a10c4a03da07ded264e9b33396aa3 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 1 Jun 2024 12:53:48 -0700 Subject: [PATCH 57/69] A few miscellaneous changes --- Sources/SourceKitLSP/Swift/CursorInfo.swift | 4 ++-- Sources/SourceKitLSP/Swift/SemanticRefactoring.swift | 6 ++---- Sources/SourceKitLSP/Workspace.swift | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Sources/SourceKitLSP/Swift/CursorInfo.swift b/Sources/SourceKitLSP/Swift/CursorInfo.swift index e7be8ff26..c9ead89dd 100644 --- a/Sources/SourceKitLSP/Swift/CursorInfo.swift +++ b/Sources/SourceKitLSP/Swift/CursorInfo.swift @@ -75,7 +75,7 @@ struct CursorInfo { // FIXME: we need to convert the utf8/utf16 column, which may require reading the file! utf16index: column - 1 ) - location = Location(uri: DocumentURI(URL(fileURLWithPath: filepath)), range: Range(position)) + location = Location(uri: DocumentURI(filePath: filepath, isDirectory: false), range: Range(position)) } else { location = nil } @@ -134,7 +134,7 @@ extension SwiftLanguageService { /// USR, and declaration location. This request does minimal processing of the result. /// /// - Parameters: - /// - url: Document URL in which to perform the request. Must be an open document. + /// - url: Document URI in which to perform the request. Must be an open document. /// - range: The position range within the document to lookup the symbol at. /// - completion: Completion block to asynchronously receive the CursorInfo, or error. func cursorInfo( diff --git a/Sources/SourceKitLSP/Swift/SemanticRefactoring.swift b/Sources/SourceKitLSP/Swift/SemanticRefactoring.swift index 95679666b..4a7c60942 100644 --- a/Sources/SourceKitLSP/Swift/SemanticRefactoring.swift +++ b/Sources/SourceKitLSP/Swift/SemanticRefactoring.swift @@ -33,7 +33,7 @@ struct SemanticRefactoring { /// - Parameters: /// - title: The title of the refactoring action. /// - dict: Response dictionary to extract information from. - /// - url: The client URL that triggered the `semantic_refactoring` request. + /// - snapshot: The snapshot that triggered the `semantic_refactoring` request. /// - keys: The sourcekitd key set to use for looking up into `dict`. init?(_ title: String, _ dict: SKDResponseDictionary, _ snapshot: DocumentSnapshot, _ keys: sourcekitd_api_keys) { guard let categorizedEdits: SKDResponseArray = dict[keys.categorizedEdits] else { @@ -108,9 +108,7 @@ extension SwiftLanguageService { /// Wraps the information returned by sourcekitd's `semantic_refactoring` request, such as the necessary edits and placeholder locations. /// /// - Parameters: - /// - url: Document URL in which to perform the request. Must be an open document. - /// - command: The semantic refactor `Command` that triggered this request. - /// - completion: Completion block to asynchronously receive the SemanticRefactoring data, or error. + /// - refactorCommand: The semantic refactor `Command` that triggered this request. func semanticRefactoring( _ refactorCommand: SemanticRefactorCommand ) async throws -> SemanticRefactoring { diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index 9d7e999d3..178f203d4 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -140,7 +140,7 @@ public final class Workspace: Sendable { } } - /// Creates a workspace for a given root `URL`, inferring the `ExternalWorkspace` if possible. + /// Creates a workspace for a given root `DocumentURI`, inferring the `ExternalWorkspace` if possible. /// /// - Parameters: /// - url: The root directory of the workspace, which must be a valid path. From 09ad77ba8d57947350855ec40c8d30ff4b91307d Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 31 May 2024 09:51:15 -0700 Subject: [PATCH 58/69] Instead of sending a message to the index log when an indexing task finishes, stream results as they come in This also means that you can use the index log to view which tasks are currently being executed. Since we only have a single log stream we can write to, I decided to prefix every line in the index log with two colored emojis that an easy visual association of every log line to the task that generated them. --- Sources/CAtomics/include/CAtomics.h | 2 +- Sources/SKCore/BuildServerBuildSystem.swift | 2 +- Sources/SKCore/BuildSystem.swift | 2 +- Sources/SKCore/BuildSystemManager.swift | 4 +- Sources/SKCore/CMakeLists.txt | 2 +- .../CompilationDatabaseBuildSystem.swift | 2 +- Sources/SKCore/IndexProcessResult.swift | 71 ------------------- Sources/SKCore/IndexTaskID.swift | 46 ++++++++++++ Sources/SKSupport/CMakeLists.txt | 1 + Sources/SKSupport/PipeAsStringHandler.swift | 49 +++++++++++++ ...LaunchWithWorkingDirectoryIfPossible.swift | 4 ++ .../SwiftPMBuildSystem.swift | 38 +++++++--- .../TestSourceKitLSPClient.swift | 4 +- .../PreparationTaskDescription.swift | 10 +-- .../SemanticIndex/SemanticIndexManager.swift | 15 ++-- .../UpdateIndexStoreTaskDescription.swift | 34 +++++---- .../Clang/ClangLanguageService.swift | 35 ++------- Sources/SourceKitLSP/SourceKitLSPServer.swift | 25 ++++--- Sources/SourceKitLSP/Workspace.swift | 8 +-- .../SKCoreTests/BuildSystemManagerTests.swift | 2 +- .../BackgroundIndexingTests.swift | 40 ++++++++--- .../SourceKitLSPTests/BuildSystemTests.swift | 4 +- 22 files changed, 226 insertions(+), 174 deletions(-) delete mode 100644 Sources/SKCore/IndexProcessResult.swift create mode 100644 Sources/SKCore/IndexTaskID.swift create mode 100644 Sources/SKSupport/PipeAsStringHandler.swift diff --git a/Sources/CAtomics/include/CAtomics.h b/Sources/CAtomics/include/CAtomics.h index a5d273647..16b797a57 100644 --- a/Sources/CAtomics/include/CAtomics.h +++ b/Sources/CAtomics/include/CAtomics.h @@ -70,7 +70,7 @@ typedef struct { } AtomicUInt32; __attribute__((swift_name("AtomicUInt32.init(initialValue:)"))) -static inline AtomicUInt32 atomic_int_create(uint8_t initialValue) { +static inline AtomicUInt32 atomic_int_create(uint32_t initialValue) { AtomicUInt32 atomic; atomic.value = initialValue; return atomic; diff --git a/Sources/SKCore/BuildServerBuildSystem.swift b/Sources/SKCore/BuildServerBuildSystem.swift index 5f19066ab..5fa36777c 100644 --- a/Sources/SKCore/BuildServerBuildSystem.swift +++ b/Sources/SKCore/BuildServerBuildSystem.swift @@ -293,7 +293,7 @@ extension BuildServerBuildSystem: BuildSystem { public func prepare( targets: [ConfiguredTarget], - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws { throw PrepareNotSupportedError() } diff --git a/Sources/SKCore/BuildSystem.swift b/Sources/SKCore/BuildSystem.swift index 204dbefc3..f409c1ab5 100644 --- a/Sources/SKCore/BuildSystem.swift +++ b/Sources/SKCore/BuildSystem.swift @@ -169,7 +169,7 @@ public protocol BuildSystem: AnyObject, Sendable { /// dependencies. func prepare( targets: [ConfiguredTarget], - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws /// If the build system has knowledge about the language that this document should be compiled in, return it. diff --git a/Sources/SKCore/BuildSystemManager.swift b/Sources/SKCore/BuildSystemManager.swift index 134453ed8..415e98fc6 100644 --- a/Sources/SKCore/BuildSystemManager.swift +++ b/Sources/SKCore/BuildSystemManager.swift @@ -237,9 +237,9 @@ extension BuildSystemManager { public func prepare( targets: [ConfiguredTarget], - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws { - try await buildSystem?.prepare(targets: targets, indexProcessDidProduceResult: indexProcessDidProduceResult) + try await buildSystem?.prepare(targets: targets, logMessageToIndexLog: logMessageToIndexLog) } public func registerForChangeNotifications(for uri: DocumentURI, language: Language) async { diff --git a/Sources/SKCore/CMakeLists.txt b/Sources/SKCore/CMakeLists.txt index 913e1e087..a0653fa64 100644 --- a/Sources/SKCore/CMakeLists.txt +++ b/Sources/SKCore/CMakeLists.txt @@ -10,7 +10,7 @@ add_library(SKCore STATIC Debouncer.swift FallbackBuildSystem.swift FileBuildSettings.swift - IndexProcessResult.swift + IndexTaskID.swift MainFilesProvider.swift PathPrefixMapping.swift SplitShellCommand.swift diff --git a/Sources/SKCore/CompilationDatabaseBuildSystem.swift b/Sources/SKCore/CompilationDatabaseBuildSystem.swift index d601bd099..c906336b0 100644 --- a/Sources/SKCore/CompilationDatabaseBuildSystem.swift +++ b/Sources/SKCore/CompilationDatabaseBuildSystem.swift @@ -129,7 +129,7 @@ extension CompilationDatabaseBuildSystem: BuildSystem { public func prepare( targets: [ConfiguredTarget], - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws { throw PrepareNotSupportedError() } diff --git a/Sources/SKCore/IndexProcessResult.swift b/Sources/SKCore/IndexProcessResult.swift deleted file mode 100644 index 07b71c7c8..000000000 --- a/Sources/SKCore/IndexProcessResult.swift +++ /dev/null @@ -1,71 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import struct TSCBasic.ProcessResult - -/// Result of a process that prepares a target or updates the index store. To be shown in the build log. -/// -/// Abstracted over a `ProcessResult` to facilitate build systems that don't spawn a new process to prepare a target but -/// prepare it from a build graph they have loaded in-process. -public struct IndexProcessResult { - /// A human-readable description of what the process was trying to achieve, like `Preparing MyTarget` - public let taskDescription: String - - /// The command that was run to produce the result. - public let command: String - - /// The output that the process produced. - public let output: String - - /// Whether the process failed. - public let failed: Bool - - /// The duration it took for the process to execute. - public let duration: Duration - - public init(taskDescription: String, command: String, output: String, failed: Bool, duration: Duration) { - self.taskDescription = taskDescription - self.command = command - self.output = output - self.failed = failed - self.duration = duration - } - - public init(taskDescription: String, processResult: ProcessResult, start: ContinuousClock.Instant) { - let stdout = (try? String(bytes: processResult.output.get(), encoding: .utf8)) ?? "" - let stderr = (try? String(bytes: processResult.stderrOutput.get(), encoding: .utf8)) ?? "" - var outputComponents: [String] = [] - if !stdout.isEmpty { - outputComponents.append( - """ - Stdout: - \(stdout) - """ - ) - } - if !stderr.isEmpty { - outputComponents.append( - """ - Stderr: - \(stderr) - """ - ) - } - self.init( - taskDescription: taskDescription, - command: processResult.arguments.joined(separator: " "), - output: outputComponents.joined(separator: "\n\n"), - failed: processResult.exitStatus != .terminated(code: 0), - duration: start.duration(to: .now) - ) - } -} diff --git a/Sources/SKCore/IndexTaskID.swift b/Sources/SKCore/IndexTaskID.swift new file mode 100644 index 000000000..4ca4564b3 --- /dev/null +++ b/Sources/SKCore/IndexTaskID.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct TSCBasic.ProcessResult + +/// The ID of a preparation or update indexstore task. This allows us to log messages from multiple concurrently running +/// indexing tasks to the index log while still being able to differentiate them. +public enum IndexTaskID: Sendable { + case preparation(id: UInt32) + case updateIndexStore(id: UInt32) + + private static func numberToEmojis(_ number: Int, numEmojis: Int) -> String { + let emojis = ["🟥", "🟩", "🟦", "🟧", "⬜️", "🟪", "⬛️", "🟨", "🟫"] + var number = abs(number) + var result = "" + for _ in 0..() + private var buffer = Data() + + /// The closure that actually handles + private let handler: @Sendable (String) -> Void + + public init(handler: @escaping @Sendable (String) -> Void) { + self.handler = handler + } + + private func handleDataFromPipeImpl(_ newData: Data) { + self.buffer += newData + while let newlineIndex = self.buffer.firstIndex(of: UInt8(ascii: "\n")) { + // Output a separate log message for every line in the pipe. + // The reason why we don't output multiple lines in a single log message is that + // a) os_log truncates log messages at about 1000 bytes. The assumption is that a single line is usually less + // than 1000 bytes long but if we merge multiple lines into one message, we might easily exceed this limit. + // b) It might be confusing why sometimes a single log message contains one line while sometimes it contains + // multiple. + handler(String(data: self.buffer[...newlineIndex], encoding: .utf8) ?? "") + buffer = buffer[buffer.index(after: newlineIndex)...] + } + } + + public nonisolated func handleDataFromPipe(_ newData: Data) { + queue.async { + await self.handleDataFromPipeImpl(newData) + } + } +} diff --git a/Sources/SKSupport/Process+LaunchWithWorkingDirectoryIfPossible.swift b/Sources/SKSupport/Process+LaunchWithWorkingDirectoryIfPossible.swift index f57f905a3..45e513058 100644 --- a/Sources/SKSupport/Process+LaunchWithWorkingDirectoryIfPossible.swift +++ b/Sources/SKSupport/Process+LaunchWithWorkingDirectoryIfPossible.swift @@ -26,6 +26,7 @@ extension Process { arguments: [String], environmentBlock: ProcessEnvironmentBlock = ProcessEnv.block, workingDirectory: AbsolutePath?, + outputRedirection: OutputRedirection = .collect, startNewProcessGroup: Bool = true, loggingHandler: LoggingHandler? = .none ) throws -> Process { @@ -35,6 +36,7 @@ extension Process { arguments: arguments, environmentBlock: environmentBlock, workingDirectory: workingDirectory, + outputRedirection: outputRedirection, startNewProcessGroup: startNewProcessGroup, loggingHandler: loggingHandler ) @@ -42,6 +44,7 @@ extension Process { Process( arguments: arguments, environmentBlock: environmentBlock, + outputRedirection: outputRedirection, startNewProcessGroup: startNewProcessGroup, loggingHandler: loggingHandler ) @@ -57,6 +60,7 @@ extension Process { arguments: arguments, environmentBlock: environmentBlock, workingDirectory: nil, + outputRedirection: outputRedirection, startNewProcessGroup: startNewProcessGroup, loggingHandler: loggingHandler ) diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index c3a24b451..6ddab50bf 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -13,7 +13,9 @@ import Basics import Build import BuildServerProtocol +import CAtomics import Dispatch +import Foundation import LSPLogging import LanguageServerProtocol import PackageGraph @@ -71,6 +73,9 @@ fileprivate extension ConfiguredTarget { static let forPackageManifest = ConfiguredTarget(targetID: "", runDestinationID: "") } +/// `nonisolated(unsafe)` is fine because `preparationTaskID` is atomic. +fileprivate nonisolated(unsafe) var preparationTaskID: AtomicUInt32 = AtomicUInt32(initialValue: 0) + /// Swift Package Manager build system and workspace support. /// /// This class implements the `BuildSystem` interface to provide the build settings for a Swift @@ -535,12 +540,12 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { public func prepare( targets: [ConfiguredTarget], - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws { // TODO (indexing): Support preparation of multiple targets at once. // https://github.com/apple/sourcekit-lsp/issues/1262 for target in targets { - try await prepare(singleTarget: target, indexProcessDidProduceResult: indexProcessDidProduceResult) + try await prepare(singleTarget: target, logMessageToIndexLog: logMessageToIndexLog) } let filesInPreparedTargets = targets.flatMap { self.targets[$0]?.buildTarget.sources ?? [] } await fileDependenciesUpdatedDebouncer.scheduleCall(Set(filesInPreparedTargets.map(DocumentURI.init))) @@ -548,7 +553,7 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { private func prepare( singleTarget target: ConfiguredTarget, - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws { if target == .forPackageManifest { // Nothing to prepare for package manifests. @@ -581,15 +586,28 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { return } let start = ContinuousClock.now - let process = try Process.launch(arguments: arguments, workingDirectory: nil) - let result = try await process.waitUntilExitSendingSigIntOnTaskCancellation() - indexProcessDidProduceResult( - IndexProcessResult( - taskDescription: "Preparing \(target.targetID) for \(target.runDestinationID)", - processResult: result, - start: start + + let logID = IndexTaskID.preparation(id: preparationTaskID.fetchAndIncrement()) + logMessageToIndexLog( + logID, + """ + Preparing \(target.targetID) for \(target.runDestinationID) + \(arguments.joined(separator: " ")) + """ + ) + let stdoutHandler = PipeAsStringHandler { logMessageToIndexLog(logID, $0) } + let stderrHandler = PipeAsStringHandler { logMessageToIndexLog(logID, $0) } + + let process = try Process.launch( + arguments: arguments, + workingDirectory: nil, + outputRedirection: .stream( + stdout: { stdoutHandler.handleDataFromPipe(Data($0)) }, + stderr: { stderrHandler.handleDataFromPipe(Data($0)) } ) ) + let result = try await process.waitUntilExitSendingSigIntOnTaskCancellation() + logMessageToIndexLog(logID, "Finished in \(start.duration(to: .now))") switch result.exitStatus.exhaustivelySwitchable { case .terminated(code: 0): break diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index eedf51d85..03cacd18e 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -273,12 +273,12 @@ public final class TestSourceKitLSPClient: MessageHandler { /// Ignores any notifications that are of a different type or that don't satisfy the predicate. public func nextNotification( ofType: ExpectedNotificationType.Type, - satisfying predicate: (ExpectedNotificationType) -> Bool = { _ in true }, + satisfying predicate: (ExpectedNotificationType) throws -> Bool = { _ in true }, timeout: Duration = .seconds(defaultTimeout) ) async throws -> ExpectedNotificationType { while true { let nextNotification = try await nextNotification(timeout: timeout) - if let notification = nextNotification as? ExpectedNotificationType, predicate(notification) { + if let notification = nextNotification as? ExpectedNotificationType, try predicate(notification) { return notification } } diff --git a/Sources/SemanticIndex/PreparationTaskDescription.swift b/Sources/SemanticIndex/PreparationTaskDescription.swift index c0adfeb31..db854e923 100644 --- a/Sources/SemanticIndex/PreparationTaskDescription.swift +++ b/Sources/SemanticIndex/PreparationTaskDescription.swift @@ -37,8 +37,8 @@ public struct PreparationTaskDescription: IndexTaskDescription { private let preparationUpToDateTracker: UpToDateTracker - /// See `SemanticIndexManager.indexProcessDidProduceResult` - private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + /// See `SemanticIndexManager.logMessageToIndexLog`. + private let logMessageToIndexLog: @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void /// Test hooks that should be called when the preparation task finishes. private let testHooks: IndexTestHooks @@ -60,13 +60,13 @@ public struct PreparationTaskDescription: IndexTaskDescription { targetsToPrepare: [ConfiguredTarget], buildSystemManager: BuildSystemManager, preparationUpToDateTracker: UpToDateTracker, - indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void, testHooks: IndexTestHooks ) { self.targetsToPrepare = targetsToPrepare self.buildSystemManager = buildSystemManager self.preparationUpToDateTracker = preparationUpToDateTracker - self.indexProcessDidProduceResult = indexProcessDidProduceResult + self.logMessageToIndexLog = logMessageToIndexLog self.testHooks = testHooks } @@ -105,7 +105,7 @@ public struct PreparationTaskDescription: IndexTaskDescription { do { try await buildSystemManager.prepare( targets: targetsToPrepare, - indexProcessDidProduceResult: indexProcessDidProduceResult + logMessageToIndexLog: logMessageToIndexLog ) } catch { logger.error( diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index dc2b663c0..2cea6efd7 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -160,11 +160,8 @@ public final actor SemanticIndexManager { /// workspaces. private let indexTaskScheduler: TaskScheduler - /// Callback to be called when the process to prepare a target finishes. - /// - /// Allows an index log to be displayed to the user that includes the command line invocations of all index-related - /// process launches, as well as their output. - private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + /// Callback that is called when an indexing task produces output it wants to log to the index log. + private let logMessageToIndexLog: @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void /// Called when files are scheduled to be indexed. /// @@ -206,7 +203,7 @@ public final actor SemanticIndexManager { buildSystemManager: BuildSystemManager, testHooks: IndexTestHooks, indexTaskScheduler: TaskScheduler, - indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void, indexTasksWereScheduled: @escaping @Sendable (Int) -> Void, indexProgressStatusDidChange: @escaping @Sendable () -> Void ) { @@ -214,7 +211,7 @@ public final actor SemanticIndexManager { self.buildSystemManager = buildSystemManager self.testHooks = testHooks self.indexTaskScheduler = indexTaskScheduler - self.indexProcessDidProduceResult = indexProcessDidProduceResult + self.logMessageToIndexLog = logMessageToIndexLog self.indexTasksWereScheduled = indexTasksWereScheduled self.indexProgressStatusDidChange = indexProgressStatusDidChange } @@ -467,7 +464,7 @@ public final actor SemanticIndexManager { targetsToPrepare: targetsToPrepare, buildSystemManager: self.buildSystemManager, preparationUpToDateTracker: preparationUpToDateTracker, - indexProcessDidProduceResult: indexProcessDidProduceResult, + logMessageToIndexLog: logMessageToIndexLog, testHooks: testHooks ) ) @@ -514,7 +511,7 @@ public final actor SemanticIndexManager { buildSystemManager: self.buildSystemManager, index: index, indexStoreUpToDateTracker: indexStoreUpToDateTracker, - indexProcessDidProduceResult: indexProcessDidProduceResult, + logMessageToIndexLog: logMessageToIndexLog, testHooks: testHooks ) ) diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift index 54d7dbeaa..4ee76e59d 100644 --- a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -111,8 +111,8 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { /// case we don't need to index it again. private let index: UncheckedIndex - /// See `SemanticIndexManager.indexProcessDidProduceResult` - private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + /// See `SemanticIndexManager.logMessageToIndexLog`. + private let logMessageToIndexLog: @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void /// Test hooks that should be called when the index task finishes. private let testHooks: IndexTestHooks @@ -139,14 +139,14 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { buildSystemManager: BuildSystemManager, index: UncheckedIndex, indexStoreUpToDateTracker: UpToDateTracker, - indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void, testHooks: IndexTestHooks ) { self.filesToIndex = filesToIndex self.buildSystemManager = buildSystemManager self.index = index self.indexStoreUpToDateTracker = indexStoreUpToDateTracker - self.indexProcessDidProduceResult = indexProcessDidProduceResult + self.logMessageToIndexLog = logMessageToIndexLog self.testHooks = testHooks } @@ -337,9 +337,25 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { defer { signposter.endInterval("Indexing", state) } + let logID = IndexTaskID.updateIndexStore(id: id) + logMessageToIndexLog( + logID, + """ + Indexing \(indexFile.pseudoPath) + \(processArguments.joined(separator: " ")) + """ + ) + + let stdoutHandler = PipeAsStringHandler { logMessageToIndexLog(logID, $0) } + let stderrHandler = PipeAsStringHandler { logMessageToIndexLog(logID, $0) } + let process = try Process.launch( arguments: processArguments, - workingDirectory: workingDirectory + workingDirectory: workingDirectory, + outputRedirection: .stream( + stdout: { stdoutHandler.handleDataFromPipe(Data($0)) }, + stderr: { stderrHandler.handleDataFromPipe(Data($0)) } + ) ) // Time out updating of the index store after 2 minutes. We don't expect any single file compilation to take longer // than 2 minutes in practice, so this indicates that the compiler has entered a loop and we probably won't make any @@ -349,13 +365,7 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { try await process.waitUntilExitSendingSigIntOnTaskCancellation() } - indexProcessDidProduceResult( - IndexProcessResult( - taskDescription: "Indexing \(indexFile.pseudoPath)", - processResult: result, - start: start - ) - ) + logMessageToIndexLog(logID, "Finished in \(start.duration(to: .now))") switch result.exitStatus.exhaustivelySwitchable { case .terminated(code: 0): diff --git a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift index 965d7b01e..b5b8efd08 100644 --- a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift +++ b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift @@ -32,35 +32,6 @@ extension NSLock { } } -/// Gathers data from clangd's stderr pipe. When it has accumulated a full line, writes the the line to the logger. -fileprivate actor ClangdStderrLogForwarder { - /// Queue on which all data from `clangd`’s stderr will be forwarded to `stderr`. This allows us to have a - /// nonisolated `handle` function but ensure that data gets processed in order. - private let queue = AsyncQueue() - private var buffer = Data() - - private func handleImpl(_ newData: Data) { - self.buffer += newData - while let newlineIndex = self.buffer.firstIndex(of: UInt8(ascii: "\n")) { - // Output a separate log message for every line in clangd's stderr. - // The reason why we don't output multiple lines in a single log message is that - // a) os_log truncates log messages at about 1000 bytes. The assumption is that a single line is usually less - // than 1000 bytes long but if we merge multiple lines into one message, we might easily exceed this limit. - // b) It might be confusing why sometimes a single log message contains one line while sometimes it contains - // multiple. - let logger = Logger(subsystem: LoggingScope.subsystem, category: "clangd-stderr") - logger.info("\(String(data: self.buffer[...newlineIndex], encoding: .utf8) ?? "")") - buffer = buffer[buffer.index(after: newlineIndex)...] - } - } - - nonisolated func handle(_ newData: Data) { - queue.async { - await self.handleImpl(newData) - } - } -} - /// A thin wrapper over a connection to a clangd server providing build setting handling. /// /// In addition, it also intercepts notifications and replies from clangd in order to do things @@ -233,14 +204,16 @@ actor ClangLanguageService: LanguageService, MessageHandler { process.standardOutput = clangdToUs process.standardInput = usToClangd - let logForwarder = ClangdStderrLogForwarder() + let logForwarder = PipeAsStringHandler { + Logger(subsystem: LoggingScope.subsystem, category: "clangd-stderr").info("\($0)") + } let stderrHandler = Pipe() stderrHandler.fileHandleForReading.readabilityHandler = { fileHandle in let newData = fileHandle.availableData if newData.count == 0 { stderrHandler.fileHandleForReading.readabilityHandler = nil } else { - logForwarder.handle(newData) + logForwarder.handleDataFromPipe(newData) } } process.standardError = stderrHandler diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index 8c11b4074..e96623e39 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -837,15 +837,18 @@ private extension LanguageServerProtocol.WorkspaceType { } extension SourceKitLSPServer { - nonisolated func indexTaskDidProduceResult(_ result: IndexProcessResult) { + nonisolated func logMessageToIndexLog(taskID: IndexTaskID, message: String) { + var message: Substring = message[...] + while message.last?.isNewline ?? false { + message = message.dropLast(1) + } + let messageWithEmojiLinePrefixes = message.split(separator: "\n", omittingEmptySubsequences: false).map { + "\(taskID.emojiRepresentation) \($0)" + }.joined(separator: "\n") self.sendNotificationToClient( LogMessageNotification( - type: result.failed ? .warning : .info, - message: """ - \(result.taskDescription) finished in \(result.duration) - \(result.command) - \(result.output) - """, + type: .info, + message: messageWithEmojiLinePrefixes, logName: "SourceKit-LSP: Indexing" ) ) @@ -932,8 +935,8 @@ extension SourceKitLSPServer { options: options, indexOptions: self.options.indexOptions, indexTaskScheduler: indexTaskScheduler, - indexProcessDidProduceResult: { [weak self] in - self?.indexTaskDidProduceResult($0) + logMessageToIndexLog: { [weak self] taskID, message in + self?.logMessageToIndexLog(taskID: taskID, message: message) }, indexTasksWereScheduled: { [weak self] count in self?.indexProgressManager.indexTasksWereScheduled(count: count) @@ -1013,8 +1016,8 @@ extension SourceKitLSPServer { index: nil, indexDelegate: nil, indexTaskScheduler: self.indexTaskScheduler, - indexProcessDidProduceResult: { [weak self] in - self?.indexTaskDidProduceResult($0) + logMessageToIndexLog: { [weak self] taskID, message in + self?.logMessageToIndexLog(taskID: taskID, message: message) }, indexTasksWereScheduled: { [weak self] count in self?.indexProgressManager.indexTasksWereScheduled(count: count) diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index 43672de55..848324a24 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -93,7 +93,7 @@ public final class Workspace: Sendable { index uncheckedIndex: UncheckedIndex?, indexDelegate: SourceKitIndexDelegate?, indexTaskScheduler: TaskScheduler, - indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void, indexTasksWereScheduled: @escaping @Sendable (Int) -> Void, indexProgressStatusDidChange: @escaping @Sendable () -> Void ) async { @@ -117,7 +117,7 @@ public final class Workspace: Sendable { buildSystemManager: buildSystemManager, testHooks: options.indexTestHooks, indexTaskScheduler: indexTaskScheduler, - indexProcessDidProduceResult: indexProcessDidProduceResult, + logMessageToIndexLog: logMessageToIndexLog, indexTasksWereScheduled: indexTasksWereScheduled, indexProgressStatusDidChange: indexProgressStatusDidChange ) @@ -155,7 +155,7 @@ public final class Workspace: Sendable { options: SourceKitLSPServer.Options, indexOptions: IndexOptions = IndexOptions(), indexTaskScheduler: TaskScheduler, - indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void, indexTasksWereScheduled: @Sendable @escaping (Int) -> Void, indexProgressStatusDidChange: @Sendable @escaping () -> Void ) async throws { @@ -196,7 +196,7 @@ public final class Workspace: Sendable { index: UncheckedIndex(index), indexDelegate: indexDelegate, indexTaskScheduler: indexTaskScheduler, - indexProcessDidProduceResult: indexProcessDidProduceResult, + logMessageToIndexLog: logMessageToIndexLog, indexTasksWereScheduled: indexTasksWereScheduled, indexProgressStatusDidChange: indexProgressStatusDidChange ) diff --git a/Tests/SKCoreTests/BuildSystemManagerTests.swift b/Tests/SKCoreTests/BuildSystemManagerTests.swift index 10583e945..ab894d5ea 100644 --- a/Tests/SKCoreTests/BuildSystemManagerTests.swift +++ b/Tests/SKCoreTests/BuildSystemManagerTests.swift @@ -471,7 +471,7 @@ class ManualBuildSystem: BuildSystem { public func prepare( targets: [ConfiguredTarget], - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws { throw PrepareNotSupportedError() } diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index 6e5ed554e..81c419017 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -709,22 +709,44 @@ final class BackgroundIndexingTests: XCTestCase { } public func testProduceIndexLog() async throws { + let didReceivePreparationIndexLogMessage = self.expectation(description: "Did receive preparation log message") + let didReceiveIndexingLogMessage = self.expectation(description: "Did receive indexing log message") + let updateIndexStoreTaskDidFinish = self.expectation(description: "Update index store task did finish") + + // Block the index tasks until we have received a log notification to make sure we stream out results as they come + // in and not only when the indexing task has finished + var serverOptions = SourceKitLSPServer.Options.testDefault + serverOptions.indexTestHooks.preparationTaskDidFinish = { _ in + await self.fulfillment(of: [didReceivePreparationIndexLogMessage], timeout: defaultTimeout) + } + serverOptions.indexTestHooks.updateIndexStoreTaskDidFinish = { _ in + await self.fulfillment(of: [didReceiveIndexingLogMessage], timeout: defaultTimeout) + updateIndexStoreTaskDidFinish.fulfill() + } + let project = try await SwiftPMTestProject( files: [ "MyFile.swift": "" ], - enableBackgroundIndexing: true + serverOptions: serverOptions, + enableBackgroundIndexing: true, + pollIndex: false ) - let targetPrepareNotification = try await project.testClient.nextNotification(ofType: LogMessageNotification.self) - XCTAssert( - targetPrepareNotification.message.hasPrefix("Preparing MyLibrary"), - "\(targetPrepareNotification.message) does not have the expected prefix" + _ = try await project.testClient.nextNotification( + ofType: LogMessageNotification.self, + satisfying: { notification in + return notification.message.contains("Preparing MyLibrary") + } ) - let indexFileNotification = try await project.testClient.nextNotification(ofType: LogMessageNotification.self) - XCTAssert( - indexFileNotification.message.hasPrefix("Indexing \(try project.uri(for: "MyFile.swift").pseudoPath)"), - "\(indexFileNotification.message) does not have the expected prefix" + didReceivePreparationIndexLogMessage.fulfill() + _ = try await project.testClient.nextNotification( + ofType: LogMessageNotification.self, + satisfying: { notification in + notification.message.contains("Indexing \(try project.uri(for: "MyFile.swift").pseudoPath)") + } ) + didReceiveIndexingLogMessage.fulfill() + try await fulfillmentOfOrThrow([updateIndexStoreTaskDidFinish]) } func testIndexingHappensInParallel() async throws { diff --git a/Tests/SourceKitLSPTests/BuildSystemTests.swift b/Tests/SourceKitLSPTests/BuildSystemTests.swift index 44c4031e9..c5439ca67 100644 --- a/Tests/SourceKitLSPTests/BuildSystemTests.swift +++ b/Tests/SourceKitLSPTests/BuildSystemTests.swift @@ -63,7 +63,7 @@ actor TestBuildSystem: BuildSystem { public func prepare( targets: [ConfiguredTarget], - indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void ) async throws { throw PrepareNotSupportedError() } @@ -141,7 +141,7 @@ final class BuildSystemTests: XCTestCase { index: nil, indexDelegate: nil, indexTaskScheduler: .forTesting, - indexProcessDidProduceResult: { _ in }, + logMessageToIndexLog: { _, _ in }, indexTasksWereScheduled: { _ in }, indexProgressStatusDidChange: {} ) From f98da773a906da2eb640412be04fa81f212fb398 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 15:19:13 -0700 Subject: [PATCH 59/69] Make passing `--experimental-prepare-for-indexing` to `swift build` an experimental feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’d like to qualify `--experimental-prepare-for-indexing` independently of background indexing using `swift build`. Because of this, I think there is value in using SourceKit-LSP using background indexing but without `--experimental-prepare-for-indexing`. It could also be useful to determine if bugs we may find are due to `--experimental-prepare-for-indexing` or also occur when running plain `swift build` commands. --- Sources/Diagnose/IndexCommand.swift | 2 +- Sources/SKCore/CMakeLists.txt | 1 + .../ExperimentalFeatures.swift | 3 + .../SwiftPMBuildSystem.swift | 81 ++++++------------- .../TestSourceKitLSPClient.swift | 2 +- Sources/SourceKitLSP/CMakeLists.txt | 1 - Sources/SourceKitLSP/CreateBuildSystem.swift | 2 +- .../SourceKitLSPServer+Options.swift | 4 +- Sources/sourcekit-lsp/SourceKitLSP.swift | 2 +- .../SwiftPMBuildSystemTests.swift | 36 ++++----- 10 files changed, 52 insertions(+), 82 deletions(-) rename Sources/{SourceKitLSP => SKCore}/ExperimentalFeatures.swift (82%) diff --git a/Sources/Diagnose/IndexCommand.swift b/Sources/Diagnose/IndexCommand.swift index 757ae5661..ad4c52c14 100644 --- a/Sources/Diagnose/IndexCommand.swift +++ b/Sources/Diagnose/IndexCommand.swift @@ -74,7 +74,7 @@ public struct IndexCommand: AsyncParsableCommand { public func run() async throws { var serverOptions = SourceKitLSPServer.Options() - serverOptions.experimentalFeatures.append(.backgroundIndexing) + serverOptions.experimentalFeatures.insert(.backgroundIndexing) let installPath = if let toolchainOverride, let toolchain = Toolchain(try AbsolutePath(validating: toolchainOverride)) { diff --git a/Sources/SKCore/CMakeLists.txt b/Sources/SKCore/CMakeLists.txt index 913e1e087..7d0d94a7e 100644 --- a/Sources/SKCore/CMakeLists.txt +++ b/Sources/SKCore/CMakeLists.txt @@ -8,6 +8,7 @@ add_library(SKCore STATIC CompilationDatabase.swift CompilationDatabaseBuildSystem.swift Debouncer.swift + ExperimentalFeatures.swift FallbackBuildSystem.swift FileBuildSettings.swift IndexProcessResult.swift diff --git a/Sources/SourceKitLSP/ExperimentalFeatures.swift b/Sources/SKCore/ExperimentalFeatures.swift similarity index 82% rename from Sources/SourceKitLSP/ExperimentalFeatures.swift rename to Sources/SKCore/ExperimentalFeatures.swift index 80aa504f9..75d33d02a 100644 --- a/Sources/SourceKitLSP/ExperimentalFeatures.swift +++ b/Sources/SKCore/ExperimentalFeatures.swift @@ -15,4 +15,7 @@ public enum ExperimentalFeature: String, Codable, Sendable, CaseIterable { /// Enable background indexing. case backgroundIndexing = "background-indexing" + + /// Add `--experimental-prepare-for-indexing` to the `swift build` command run to prepare a target for indexing. + case swiftpmPrepareForIndexing = "swiftpm-prepare-for-indexing" } diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index 2aa8843af..49a994768 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -95,71 +95,35 @@ public actor SwiftPMBuildSystem { } /// Callbacks that should be called if the list of possible test files has changed. - public var testFilesDidChangeCallbacks: [() async -> Void] = [] + private var testFilesDidChangeCallbacks: [() async -> Void] = [] - let workspacePath: TSCAbsolutePath + private let workspacePath: TSCAbsolutePath /// The directory containing `Package.swift`. + @_spi(Testing) public var projectRoot: TSCAbsolutePath - var modulesGraph: ModulesGraph - let workspace: Workspace - public let toolsBuildParameters: BuildParameters - public let destinationBuildParameters: BuildParameters - let fileSystem: FileSystem + private var modulesGraph: ModulesGraph + private let workspace: Workspace + @_spi(Testing) public let toolsBuildParameters: BuildParameters + @_spi(Testing) public let destinationBuildParameters: BuildParameters + private let fileSystem: FileSystem private let toolchainRegistry: ToolchainRegistry private let swiftBuildSupportsPrepareForIndexingTask = SKSupport.ThreadSafeBox?>(initialValue: nil) - #if compiler(>=6.1) - #warning( - "Remove swiftBuildSupportsPrepareForIndexing when we no longer need to support SwiftPM versions that don't have support for `--experimental-prepare-for-indexing`" - ) - #endif - /// Whether `swift build` supports the `--experimental-prepare-for-indexing` flag. - private var swiftBuildSupportsPrepareForIndexing: Bool { - get async { - let task = swiftBuildSupportsPrepareForIndexingTask.withLock { task in - if let task { - return task - } - let newTask = Task { () -> Bool in - guard let swift = await toolchainRegistry.default?.swift else { - return false - } - - do { - let process = Process(args: swift.pathString, "build", "--help-hidden") - try process.launch() - let result = try await process.waitUntilExit() - guard let output = String(bytes: try result.output.get(), encoding: .utf8) else { - return false - } - return output.contains("--experimental-prepare-for-indexing") - } catch { - return false - } - } - task = newTask - return newTask - } - - return await task.value - } - } - - var fileToTarget: [DocumentURI: SwiftBuildTarget] = [:] - var sourceDirToTarget: [DocumentURI: SwiftBuildTarget] = [:] + private var fileToTarget: [DocumentURI: SwiftBuildTarget] = [:] + private var sourceDirToTarget: [DocumentURI: SwiftBuildTarget] = [:] /// Maps configured targets ids to their SwiftPM build target as well as an index in their topological sorting. /// /// Targets with lower index are more low level, ie. targets with higher indices depend on targets with lower indices. - var targets: [ConfiguredTarget: (index: Int, buildTarget: SwiftBuildTarget)] = [:] + private var targets: [ConfiguredTarget: (index: Int, buildTarget: SwiftBuildTarget)] = [:] /// The URIs for which the delegate has registered for change notifications, /// mapped to the language the delegate specified when registering for change notifications. - var watchedFiles: Set = [] + private var watchedFiles: Set = [] /// This callback is informed when `reloadPackage` starts and ends executing. - var reloadPackageStatusCallback: (ReloadPackageStatus) async -> Void + private var reloadPackageStatusCallback: (ReloadPackageStatus) async -> Void /// Debounces calls to `delegate.filesDependenciesUpdated`. /// @@ -169,16 +133,19 @@ public actor SwiftPMBuildSystem { /// `fileDependenciesUpdated` call once for every updated file within the target. /// /// Force-unwrapped optional because initializing it requires access to `self`. - var fileDependenciesUpdatedDebouncer: Debouncer>! = nil + private var fileDependenciesUpdatedDebouncer: Debouncer>! = nil /// A `ObservabilitySystem` from `SwiftPM` that logs. private let observabilitySystem = ObservabilitySystem({ scope, diagnostic in logger.log(level: diagnostic.severity.asLogLevel, "SwiftPM log: \(diagnostic.description)") }) + /// Whether to pass `--experimental-prepare-for-indexing` to `swift build` as part of preparation. + private let experimentalFeatures: Set + /// Whether the `SwiftPMBuildSystem` is pointed at a `.index-build` directory that's independent of the /// user's build. - private let isForIndexBuild: Bool + private var isForIndexBuild: Bool { experimentalFeatures.contains(.backgroundIndexing) } /// Creates a build system using the Swift Package Manager, if this workspace is a package. /// @@ -193,13 +160,13 @@ public actor SwiftPMBuildSystem { toolchainRegistry: ToolchainRegistry, fileSystem: FileSystem = localFileSystem, buildSetup: BuildSetup, - isForIndexBuild: Bool, + experimentalFeatures: Set, reloadPackageStatusCallback: @escaping (ReloadPackageStatus) async -> Void = { _ in } ) async throws { self.workspacePath = workspacePath self.fileSystem = fileSystem self.toolchainRegistry = toolchainRegistry - self.isForIndexBuild = isForIndexBuild + self.experimentalFeatures = experimentalFeatures guard let packageRoot = findPackageDirectory(containing: workspacePath, fileSystem) else { throw Error.noManifest(workspacePath: workspacePath) @@ -218,7 +185,7 @@ public actor SwiftPMBuildSystem { forRootPackage: AbsolutePath(packageRoot), fileSystem: fileSystem ) - if isForIndexBuild { + if experimentalFeatures.contains(.backgroundIndexing) { location.scratchDirectory = AbsolutePath(packageRoot.appending(component: ".index-build")) } else if let scratchDirectory = buildSetup.path { location.scratchDirectory = AbsolutePath(scratchDirectory) @@ -289,7 +256,7 @@ public actor SwiftPMBuildSystem { uri: DocumentURI, toolchainRegistry: ToolchainRegistry, buildSetup: BuildSetup, - isForIndexBuild: Bool, + experimentalFeatures: Set, reloadPackageStatusCallback: @escaping (ReloadPackageStatus) async -> Void ) async { guard let fileURL = uri.fileURL else { @@ -301,7 +268,7 @@ public actor SwiftPMBuildSystem { toolchainRegistry: toolchainRegistry, fileSystem: localFileSystem, buildSetup: buildSetup, - isForIndexBuild: isForIndexBuild, + experimentalFeatures: experimentalFeatures, reloadPackageStatusCallback: reloadPackageStatusCallback ) } catch Error.noManifest { @@ -612,7 +579,7 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { arguments += self.destinationBuildParameters.flags.swiftCompilerFlags.flatMap { ["-Xswiftc", $0] } arguments += self.destinationBuildParameters.flags.linkerFlags.flatMap { ["-Xlinker", $0] } arguments += self.destinationBuildParameters.flags.xcbuildFlags?.flatMap { ["-Xxcbuild", $0] } ?? [] - if await swiftBuildSupportsPrepareForIndexing { + if experimentalFeatures.contains(.swiftpmPrepareForIndexing) { arguments.append("--experimental-prepare-for-indexing") } if Task.isCancelled { diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index 73a91bf6c..908ea61d1 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -110,7 +110,7 @@ public final class TestSourceKitLSPClient: MessageHandler { serverOptions.buildSetup.flags.swiftCompilerFlags += ["-module-cache-path", globalModuleCache.path] } if enableBackgroundIndexing { - serverOptions.experimentalFeatures.append(.backgroundIndexing) + serverOptions.experimentalFeatures.insert(.backgroundIndexing) } var notificationYielder: AsyncStream.Continuation! diff --git a/Sources/SourceKitLSP/CMakeLists.txt b/Sources/SourceKitLSP/CMakeLists.txt index b930b76c3..7efb356a8 100644 --- a/Sources/SourceKitLSP/CMakeLists.txt +++ b/Sources/SourceKitLSP/CMakeLists.txt @@ -4,7 +4,6 @@ add_library(SourceKitLSP STATIC CreateBuildSystem.swift DocumentManager.swift DocumentSnapshot+FromFileContents.swift - ExperimentalFeatures.swift IndexProgressManager.swift IndexStoreDB+MainFilesProvider.swift LanguageServerType.swift diff --git a/Sources/SourceKitLSP/CreateBuildSystem.swift b/Sources/SourceKitLSP/CreateBuildSystem.swift index f673cf486..09a2f4071 100644 --- a/Sources/SourceKitLSP/CreateBuildSystem.swift +++ b/Sources/SourceKitLSP/CreateBuildSystem.swift @@ -38,7 +38,7 @@ func createBuildSystem( uri: rootUri, toolchainRegistry: toolchainRegistry, buildSetup: options.buildSetup, - isForIndexBuild: options.experimentalFeatures.contains(.backgroundIndexing), + experimentalFeatures: options.experimentalFeatures, reloadPackageStatusCallback: reloadPackageStatusCallback ) } diff --git a/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift b/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift index 78dc82d02..c4102a5a3 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift @@ -50,7 +50,7 @@ extension SourceKitLSPServer { public var swiftPublishDiagnosticsDebounceDuration: TimeInterval /// Experimental features that are enabled. - public var experimentalFeatures: [ExperimentalFeature] + public var experimentalFeatures: Set public var indexTestHooks: IndexTestHooks @@ -62,7 +62,7 @@ extension SourceKitLSPServer { completionOptions: SKCompletionOptions = .init(), generatedInterfacesPath: AbsolutePath = defaultDirectoryForGeneratedInterfaces, swiftPublishDiagnosticsDebounceDuration: TimeInterval = 2, /* 2s */ - experimentalFeatures: [ExperimentalFeature] = [], + experimentalFeatures: Set = [], indexTestHooks: IndexTestHooks = IndexTestHooks() ) { self.buildSetup = buildSetup diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index d0f317355..3b89b55b8 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -224,7 +224,7 @@ struct SourceKitLSP: AsyncParsableCommand { serverOptions.indexOptions.indexPrefixMappings = indexPrefixMappings serverOptions.completionOptions.maxResults = completionMaxResults serverOptions.generatedInterfacesPath = generatedInterfacesPath - serverOptions.experimentalFeatures = experimentalFeatures + serverOptions.experimentalFeatures = Set(experimentalFeatures) return serverOptions } diff --git a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift index 89f0f5b8d..f0b8b0cb7 100644 --- a/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift +++ b/Tests/SKSwiftPMWorkspaceTests/SwiftPMBuildSystemTests.swift @@ -15,7 +15,7 @@ import LSPTestSupport import LanguageServerProtocol import PackageModel @_spi(Testing) import SKCore -import SKSwiftPMWorkspace +@_spi(Testing) import SKSwiftPMWorkspace import SKTestSupport import SourceKitLSP import TSCBasic @@ -54,7 +54,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) ) } @@ -81,7 +81,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) await assertThrowsError(try await buildSystem.generateBuildGraph(allowFileSystemWrites: false)) } @@ -111,7 +111,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: ToolchainRegistry(toolchains: []), fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) ) } @@ -142,7 +142,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -207,7 +207,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: localFileSystem, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -270,7 +270,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: config, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -312,7 +312,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -349,7 +349,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -398,7 +398,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -463,7 +463,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -507,7 +507,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -588,7 +588,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: ToolchainRegistry.forTesting, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -640,7 +640,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -708,7 +708,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: ToolchainRegistry.forTesting, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -748,7 +748,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) @@ -789,7 +789,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) assertEqual(await swiftpmBuildSystem.projectRoot, try resolveSymlinks(tempDir.appending(component: "pkg"))) @@ -824,7 +824,7 @@ final class SwiftPMBuildSystemTests: XCTestCase { toolchainRegistry: tr, fileSystem: fs, buildSetup: SourceKitLSPServer.Options.testDefault.buildSetup, - isForIndexBuild: false + experimentalFeatures: [] ) try await swiftpmBuildSystem.generateBuildGraph(allowFileSystemWrites: false) From d04bf848935d86d58bc5fa8d57ff1d70a00ce927 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 17:51:23 -0700 Subject: [PATCH 60/69] =?UTF-8?q?Don=E2=80=99t=20cancel=20in-progress=20di?= =?UTF-8?q?agnostic=20generation=20when=20calling=20`DiagnosticReportManag?= =?UTF-8?q?er.removeItemsFromCache`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was causing a non-deterministic test failure: When target preparation finishes while a diagnostic request is in progress, it will re-open the document, which calls `DiagnosticReportManager.removeItemsFromCache` for that document’s URI. With the old implementation, we would thus cancel the diagnostics sourcekitd request and return a cancelled error to the diagnostics LSP request. While doing this, I also realized that there was a race condition: Document re-opening would happen outside of the SourceKit-LSP message handling queue and could thus run concurrently to any other request. This means that a sourcekitd request could run after `reopenDocument` had closed the document but before it was opened again. Introduce an internal reopen request that can be handled on the main message handling queue and thus doesn’t have this problem --- Sources/LanguageServerProtocol/CMakeLists.txt | 1 + .../ReopenTextDocumentNotifications.swift | 29 ++++++++++++++++++ .../Clang/ClangLanguageService.swift | 2 ++ Sources/SourceKitLSP/LanguageService.swift | 8 +++++ .../MessageHandlingDependencyTracker.swift | 2 ++ Sources/SourceKitLSP/SourceKitLSPServer.swift | 13 ++++++++ .../Swift/DiagnosticReportManager.swift | 4 --- .../Swift/SwiftLanguageService.swift | 30 +++++++++++-------- 8 files changed, 73 insertions(+), 16 deletions(-) create mode 100644 Sources/LanguageServerProtocol/Notifications/ReopenTextDocumentNotifications.swift diff --git a/Sources/LanguageServerProtocol/CMakeLists.txt b/Sources/LanguageServerProtocol/CMakeLists.txt index 03f55e21a..4495c6f34 100644 --- a/Sources/LanguageServerProtocol/CMakeLists.txt +++ b/Sources/LanguageServerProtocol/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(LanguageServerProtocol STATIC Notifications/LogMessageNotification.swift Notifications/LogTraceNotification.swift Notifications/PublishDiagnosticsNotification.swift + Notifications/ReopenTextDocumentNotifications.swift Notifications/SetTraceNotification.swift Notifications/ShowMessageNotification.swift Notifications/TextSynchronizationNotifications.swift diff --git a/Sources/LanguageServerProtocol/Notifications/ReopenTextDocumentNotifications.swift b/Sources/LanguageServerProtocol/Notifications/ReopenTextDocumentNotifications.swift new file mode 100644 index 000000000..fb1f8a8a1 --- /dev/null +++ b/Sources/LanguageServerProtocol/Notifications/ReopenTextDocumentNotifications.swift @@ -0,0 +1,29 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// Re-open the given document, discarding any in-memory state. +/// +/// This notification is designed to be used internally in SourceKit-LSP: When build setting have changed, we re-open +/// the document in sourcekitd to re-build its AST. This needs to be handled via a notification to ensure that no other +/// request for this document is executing at the same time. +/// +/// **(LSP Extension)** +public struct ReopenTextDocumentNotification: NotificationType, Hashable { + public static let method: String = "textDocument/reopen" + + /// The document identifier and initial contents. + public var textDocument: TextDocumentIdentifier + + public init(textDocument: TextDocumentIdentifier) { + self.textDocument = textDocument + } +} diff --git a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift index 965d7b01e..60a0739ef 100644 --- a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift +++ b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift @@ -478,6 +478,8 @@ extension ClangLanguageService { clangd.send(notification) } + func reopenDocument(_ notification: ReopenTextDocumentNotification) {} + public func changeDocument(_ notification: DidChangeTextDocumentNotification) { clangd.send(notification) } diff --git a/Sources/SourceKitLSP/LanguageService.swift b/Sources/SourceKitLSP/LanguageService.swift index 0b0249e5e..61021a324 100644 --- a/Sources/SourceKitLSP/LanguageService.swift +++ b/Sources/SourceKitLSP/LanguageService.swift @@ -118,6 +118,14 @@ public protocol LanguageService: AnyObject, Sendable { /// Sent to close a document on the Language Server. func closeDocument(_ notification: DidCloseTextDocumentNotification) async + + /// Re-open the given document, discarding any in-memory state and forcing an AST to be re-built after build settings + /// have been changed. This needs to be handled via a notification to ensure that no other request for this document + /// is executing at the same time. + /// + /// Only intended for `SwiftLanguageService`. + func reopenDocument(_ notification: ReopenTextDocumentNotification) async + func changeDocument(_ notification: DidChangeTextDocumentNotification) async func willSaveDocument(_ notification: WillSaveTextDocumentNotification) async func didSaveDocument(_ notification: DidSaveTextDocumentNotification) async diff --git a/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift b/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift index 79c09bc5f..cc4fdd4b2 100644 --- a/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift +++ b/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift @@ -126,6 +126,8 @@ enum MessageHandlingDependencyTracker: DependencyTracker { self = .freestanding case is PublishDiagnosticsNotification: self = .freestanding + case let notification as ReopenTextDocumentNotification: + self = .documentUpdate(notification.textDocument.uri) case is SetTraceNotification: self = .globalConfigurationChange case is ShowMessageNotification: diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index ab858e397..dfc936e7a 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -569,6 +569,8 @@ extension SourceKitLSPServer: MessageHandler { await self.openDocument(notification) case let notification as DidCloseTextDocumentNotification: await self.closeDocument(notification) + case let notification as ReopenTextDocumentNotification: + await self.reopenDocument(notification) case let notification as DidChangeTextDocumentNotification: await self.changeDocument(notification) case let notification as DidChangeWorkspaceFoldersNotification: @@ -1300,6 +1302,17 @@ extension SourceKitLSPServer { await self.closeDocument(notification, workspace: workspace) } + func reopenDocument(_ notification: ReopenTextDocumentNotification) async { + let uri = notification.textDocument.uri + guard let workspace = await workspaceForDocument(uri: uri) else { + logger.error( + "received reopen notification for file '\(uri.forLogging)' without a corresponding workspace, ignoring..." + ) + return + } + await workspace.documentService.value[uri]?.reopenDocument(notification) + } + func closeDocument(_ notification: DidCloseTextDocumentNotification, workspace: Workspace) async { // Immediately close the document. We need to be sure to clear our pending work queue in case // the build system still isn't ready. diff --git a/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift b/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift index 81b4f1cf7..c8a12330a 100644 --- a/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift +++ b/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift @@ -86,11 +86,7 @@ actor DiagnosticReportManager { } func removeItemsFromCache(with uri: DocumentURI) async { - let tasksToCancel = reportTaskCache.filter { $0.snapshotID.uri == uri } reportTaskCache.removeAll(where: { $0.snapshotID.uri == uri }) - for task in tasksToCancel { - await task.reportTask.cancel() - } } private func requestReport( diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index e488264e0..fc0e8b1db 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -327,15 +327,24 @@ extension SwiftLanguageService { // MARK: - Build System Integration - private func reopenDocument(_ snapshot: DocumentSnapshot, _ compileCmd: SwiftCompileCommand?) async { + public func reopenDocument(_ notification: ReopenTextDocumentNotification) async { + let snapshot = orLog("Getting snapshot to re-open document") { + try documentManager.latestSnapshot(notification.textDocument.uri) + } + guard let snapshot else { + return + } cancelInFlightPublishDiagnosticsTask(for: snapshot.uri) await diagnosticReportManager.removeItemsFromCache(with: snapshot.uri) let closeReq = closeDocumentSourcekitdRequest(uri: snapshot.uri) - _ = try? await self.sourcekitd.send(closeReq, fileContents: nil) + _ = await orLog("Closing document to re-open it") { try await self.sourcekitd.send(closeReq, fileContents: nil) } - let openReq = openDocumentSourcekitdRequest(snapshot: snapshot, compileCommand: compileCmd) - _ = try? await self.sourcekitd.send(openReq, fileContents: snapshot.text) + let openReq = openDocumentSourcekitdRequest( + snapshot: snapshot, + compileCommand: await buildSettings(for: snapshot.uri) + ) + _ = await orLog("Re-opening document") { try await self.sourcekitd.send(openReq, fileContents: snapshot.text) } if await capabilityRegistry.clientSupportsPullDiagnostics(for: .swift) { await self.refreshDiagnosticsDebouncer.scheduleCall() @@ -345,14 +354,11 @@ extension SwiftLanguageService { } public func documentUpdatedBuildSettings(_ uri: DocumentURI) async { - // We may not have a snapshot if this is called just before `openDocument`. - guard let snapshot = try? self.documentManager.latestSnapshot(uri) else { - return - } - - // Close and re-open the document internally to inform sourcekitd to update the compile - // command. At the moment there's no better way to do this. - await self.reopenDocument(snapshot, await self.buildSettings(for: uri)) + // Close and re-open the document internally to inform sourcekitd to update the compile command. At the moment + // there's no better way to do this. + // Schedule the document re-open in the SourceKit-LSP server. This ensures that the re-open happens exclusively with + // no other request running at the same time. + sourceKitLSPServer?.handle(ReopenTextDocumentNotification(textDocument: TextDocumentIdentifier(uri))) } public func documentDependenciesUpdated(_ uri: DocumentURI) async { From 5b119b705f81f80a73548c5dab874e40e32c1bd8 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sun, 2 Jun 2024 10:16:46 -0700 Subject: [PATCH 61/69] Add tests to rename C++ symbols exposed to Swift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This exposes an issue where we wouldn’t rename a symbol on the clang side f a symbol was only defined in a header and not a C/C++/... file. In that case we failed to determine the language of the header file because there was no unit file that contained the header file. The cleaner solution here is to ask for the symbol provider of each occurrence directly. rdar://127391127 --- .../SKTestSupport/SwiftPMTestProject.swift | 2 +- Sources/SemanticIndex/CheckedIndex.swift | 4 - Sources/SourceKitLSP/Rename.swift | 70 ++++----- .../CrossLanguageRenameTests.swift | 146 ++++++++++++++++++ 4 files changed, 175 insertions(+), 47 deletions(-) diff --git a/Sources/SKTestSupport/SwiftPMTestProject.swift b/Sources/SKTestSupport/SwiftPMTestProject.swift index 2c186b4ee..18de4f72f 100644 --- a/Sources/SKTestSupport/SwiftPMTestProject.swift +++ b/Sources/SKTestSupport/SwiftPMTestProject.swift @@ -177,7 +177,7 @@ public class SwiftPMTestProject: MultiFileTestProject { if !manifest.contains("swift-tools-version") { // Tests specify a shorthand package manifest that doesn't contain the tools version boilerplate. manifest = """ - // swift-tools-version: 5.7 + // swift-tools-version: 5.10 import PackageDescription diff --git a/Sources/SemanticIndex/CheckedIndex.swift b/Sources/SemanticIndex/CheckedIndex.swift index 55cf4d460..76d115eda 100644 --- a/Sources/SemanticIndex/CheckedIndex.swift +++ b/Sources/SemanticIndex/CheckedIndex.swift @@ -176,10 +176,6 @@ public struct UncheckedIndex: Sendable { return CheckedIndex(index: underlyingIndexStoreDB, checkLevel: checkLevel) } - public func symbolProvider(for sourceFilePath: String) -> SymbolProviderKind? { - return underlyingIndexStoreDB.symbolProvider(for: sourceFilePath) - } - /// Wait for IndexStoreDB to be updated based on new unit files written to disk. public func pollForUnitChangesAndWait() { self.underlyingIndexStoreDB.pollForUnitChangesAndWait() diff --git a/Sources/SourceKitLSP/Rename.swift b/Sources/SourceKitLSP/Rename.swift index f87741630..91df9c773 100644 --- a/Sources/SourceKitLSP/Rename.swift +++ b/Sources/SourceKitLSP/Rename.swift @@ -500,7 +500,7 @@ extension SourceKitLSPServer { ) async -> (swiftLanguageService: SwiftLanguageService, snapshot: DocumentSnapshot, location: SymbolLocation)? { var reference: SymbolOccurrence? = nil index.forEachSymbolOccurrence(byUSR: usr, roles: renameRoles) { - if index.unchecked.symbolProvider(for: $0.location.path) == .swift { + if $0.symbolProvider == .swift { reference = $0 // We have found a reference from Swift. Stop iteration. return false @@ -631,7 +631,7 @@ extension SourceKitLSPServer { // If we terminate early by returning `false` from the closure, `forEachSymbolOccurrence` returns `true`, // indicating that we have found a reference from clang. let hasReferenceFromClang = !index.forEachSymbolOccurrence(byUSR: usr, roles: renameRoles) { - return index.unchecked.symbolProvider(for: $0.location.path) != .clang + return $0.symbolProvider != .clang } let clangName: String? if hasReferenceFromClang { @@ -730,32 +730,7 @@ extension SourceKitLSPServer { // If we have a USR + old name, perform an index lookup to find workspace-wide symbols to rename. // First, group all occurrences of that USR by the files they occur in. - var locationsByFile: [DocumentURI: [RenameLocation]] = [:] - - actor LanguageServerTypesCache { - let index: UncheckedIndex - var languageServerTypesCache: [DocumentURI: LanguageServerType?] = [:] - - init(index: UncheckedIndex) { - self.index = index - } - - func languageServerType(for uri: DocumentURI) -> LanguageServerType? { - if let cachedValue = languageServerTypesCache[uri] { - return cachedValue - } - let serverType: LanguageServerType? = - if let fileURL = uri.fileURL { - LanguageServerType(symbolProvider: index.symbolProvider(for: fileURL.path)) - } else { - nil - } - languageServerTypesCache[uri] = serverType - return serverType - } - } - - let languageServerTypesCache = LanguageServerTypesCache(index: index.unchecked) + var locationsByFile: [DocumentURI: (renameLocations: [RenameLocation], symbolProvider: SymbolProviderKind)] = [:] let usrsToRename = overridingAndOverriddenUsrs(of: usr, index: index) let occurrencesToRename = usrsToRename.flatMap { index.occurrences(ofUSR: $0, roles: renameRoles) } @@ -770,19 +745,16 @@ extension SourceKitLSPServer { // perform an indexed rename for it. continue } - switch await languageServerTypesCache.languageServerType(for: uri) { + switch occurrence.symbolProvider { case .swift: // sourcekitd only produces AST-based results for the direct calls to this USR. This is because the Swift // AST only has upwards references to superclasses and overridden methods, not the other way round. It is // thus not possible to (easily) compute an up-down closure like described in `overridingAndOverriddenUsrs`. // We thus need to perform an indexed rename for other, related USRs. break - case .clangd: + case .clang: // clangd produces AST-based results for the entire class hierarchy, so nothing to do. continue - case nil: - // Unknown symbol provider - continue } } @@ -791,25 +763,39 @@ extension SourceKitLSPServer { utf8Column: occurrence.location.utf8Column, usage: RenameLocation.Usage(roles: occurrence.roles) ) - locationsByFile[uri, default: []].append(renameLocation) + if let existingLocations = locationsByFile[uri] { + if existingLocations.symbolProvider != occurrence.symbolProvider { + logger.fault( + """ + Found mismatching symbol providers for \(uri.forLogging): \ + \(String(describing: existingLocations.symbolProvider), privacy: .public) vs \ + \(String(describing: occurrence.symbolProvider), privacy: .public) + """ + ) + } + locationsByFile[uri] = (existingLocations.renameLocations + [renameLocation], occurrence.symbolProvider) + } else { + locationsByFile[uri] = ([renameLocation], occurrence.symbolProvider) + } } // Now, call `editsToRename(locations:in:oldName:newName:)` on the language service to convert these ranges into // edits. let urisAndEdits = await locationsByFile - .concurrentMap { (uri: DocumentURI, renameLocations: [RenameLocation]) -> (DocumentURI, [TextEdit])? in + .concurrentMap { + ( + uri: DocumentURI, + value: (renameLocations: [RenameLocation], symbolProvider: SymbolProviderKind) + ) -> (DocumentURI, [TextEdit])? in let language: Language - switch await languageServerTypesCache.languageServerType(for: uri) { - case .clangd: + switch value.symbolProvider { + case .clang: // Technically, we still don't know the language of the source file but defaulting to C is sufficient to // ensure we get the clang toolchain language server, which is all we care about. language = .c case .swift: language = .swift - case nil: - logger.error("Failed to determine symbol provider for \(uri.forLogging)") - return nil } // Create a document snapshot to operate on. If the document is open, load it from the document manager, // otherwise conjure one from the file on disk. We need the file in memory to perform UTF-8 to UTF-16 column @@ -825,13 +811,13 @@ extension SourceKitLSPServer { var edits: [TextEdit] = await orLog("Getting edits for rename location") { return try await languageService.editsToRename( - locations: renameLocations, + locations: value.renameLocations, in: snapshot, oldName: oldName, newName: newName ) } ?? [] - for location in renameLocations where location.usage == .definition { + for location in value.renameLocations where location.usage == .definition { edits += await languageService.editsToRenameParametersInFunctionBody( snapshot: snapshot, renameLocation: location, diff --git a/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift b/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift index e49225f32..ef6279ecc 100644 --- a/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift +++ b/Tests/SourceKitLSPTests/CrossLanguageRenameTests.swift @@ -23,6 +23,16 @@ private let libAlibBPackageManifest = """ ) """ +private let libAlibBCxxInteropPackageManifest = """ + let package = Package( + name: "MyLibrary", + targets: [ + .target(name: "LibA"), + .target(name: "LibB", dependencies: ["LibA"], swiftSettings: [.interoperabilityMode(.Cxx)]), + ] + ) + """ + final class CrossLanguageRenameTests: XCTestCase { func testZeroArgCFunction() async throws { try await SkipUnless.clangdSupportsIndexBasedRename() @@ -733,4 +743,140 @@ final class CrossLanguageRenameTests: XCTestCase { manifest: libAlibBPackageManifest ) } + + func testRenameCxxClassExposedToSwift() async throws { + try await SkipUnless.clangdSupportsIndexBasedRename() + try await assertMultiFileRename( + files: [ + "LibA/include/LibA.h": """ + struct 1️⃣Foo {}; + """, + "LibA/LibA.cpp": "", + "LibB/LibB.swift": """ + import LibA + + func test(foo: 2️⃣Foo) {} + """, + ], + headerFileLanguage: .cpp, + newName: "Bar", + expectedPrepareRenamePlaceholder: "Foo", + expected: [ + "LibA/include/LibA.h": """ + struct Bar {}; + """, + "LibA/LibA.cpp": "", + "LibB/LibB.swift": """ + import LibA + + func test(foo: Bar) {} + """, + ], + manifest: libAlibBCxxInteropPackageManifest + ) + } + + func testRenameCxxMethodExposedToSwift() async throws { + try await SkipUnless.clangdSupportsIndexBasedRename() + try await assertMultiFileRename( + files: [ + "LibA/include/LibA.h": """ + struct Foo { + int 1️⃣doStuff() const; + }; + """, + "LibA/LibA.cpp": """ + #include "LibA.h" + + int Foo::2️⃣doStuff() const { + return 42; + } + """, + "LibB/LibB.swift": """ + import LibA + + func test(foo: Foo) { + foo.3️⃣doStuff() + } + """, + ], + headerFileLanguage: .cpp, + newName: "doNewStuff", + expectedPrepareRenamePlaceholder: "doStuff", + expected: [ + "LibA/include/LibA.h": """ + struct Foo { + int doNewStuff() const; + }; + """, + "LibA/LibA.cpp": """ + #include "LibA.h" + + int Foo::doNewStuff() const { + return 42; + } + """, + "LibB/LibB.swift": """ + import LibA + + func test(foo: Foo) { + foo.doNewStuff() + } + """, + ], + manifest: libAlibBCxxInteropPackageManifest + ) + } + + func testRenameSwiftMethodExposedToSwift() async throws { + try await SkipUnless.clangdSupportsIndexBasedRename() + try await assertMultiFileRename( + files: [ + "LibA/include/LibA.h": """ + struct Foo { + int 1️⃣doStuff() const; + }; + """, + "LibA/LibA.cpp": """ + #include "LibA.h" + + int Foo::2️⃣doStuff() const { + return 42; + } + """, + "LibB/LibB.swift": """ + import LibA + + func test(foo: Foo) { + foo.3️⃣doStuff() + } + """, + ], + headerFileLanguage: .cpp, + newName: "doNewStuff", + expectedPrepareRenamePlaceholder: "doStuff", + expected: [ + "LibA/include/LibA.h": """ + struct Foo { + int doNewStuff() const; + }; + """, + "LibA/LibA.cpp": """ + #include "LibA.h" + + int Foo::doNewStuff() const { + return 42; + } + """, + "LibB/LibB.swift": """ + import LibA + + func test(foo: Foo) { + foo.doNewStuff() + } + """, + ], + manifest: libAlibBCxxInteropPackageManifest + ) + } } From 32852afd1f43169bafc4ddf381077576127e252c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 23:15:30 -0700 Subject: [PATCH 62/69] Demote two log messages issued on the `error` level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just two cases where we were logging an error that wasn’t strictly necessary and was just spamming the log at the high log levels. --- Sources/SourceKitLSP/SourceKitLSPServer.swift | 2 +- Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index ee0362945..e20aaafca 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -1764,7 +1764,7 @@ extension SourceKitLSPServer { // `SwiftLanguageService` will always respond with `unsupported method`. Thus, only log such a failure instead of // returning it to the client. if indexBasedResponse.isEmpty { - return await orLog("Fallback definition request") { + return await orLog("Fallback definition request", level: .info) { return try await languageService.definition(req) } } diff --git a/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift b/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift index 961dc23ea..a52809c6a 100644 --- a/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift +++ b/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift @@ -178,6 +178,11 @@ actor SyntacticTestIndex { guard !removedFiles.contains(uri) else { return } + guard FileManager.default.fileExists(atPath: url.path) else { + // File no longer exists. Probably deleted since we scheduled it for indexing. Nothing to worry about. + logger.info("Not indexing \(uri.forLogging) for tests because it does not exist") + return + } guard let fileModificationDate = try? FileManager.default.attributesOfItem(atPath: url.path)[.modificationDate] as? Date From b479b2e8744fc6474c9035fe27bce5c45854d058 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 11:56:45 -0700 Subject: [PATCH 63/69] Create a `SwiftExtensions` module This allows us to share common Swift utility functions between SourceKit-LSP and LSPLogging. --- Package.swift | 18 ++++++++++++++++++ Sources/CMakeLists.txt | 1 + Sources/Diagnose/CMakeLists.txt | 1 + Sources/Diagnose/IndexCommand.swift | 1 + .../LSPTestSupport/TestJSONRPCConnection.swift | 1 + Sources/SKCore/BuildServerBuildSystem.swift | 1 + Sources/SKCore/CMakeLists.txt | 1 + Sources/SKCore/TaskScheduler.swift | 1 + Sources/SKCore/Toolchain.swift | 1 + Sources/SKSupport/CMakeLists.txt | 8 +------- Sources/SKSupport/Connection+Send.swift | 1 + Sources/SKSupport/PipeAsStringHandler.swift | 1 + Sources/SKSwiftPMWorkspace/CMakeLists.txt | 1 + .../SwiftPMBuildSystem.swift | 5 ++++- .../SKTestSupport/TestSourceKitLSPClient.swift | 1 + Sources/SemanticIndex/CMakeLists.txt | 1 + .../UpdateIndexStoreTaskDescription.swift | 1 + Sources/SourceKitD/CMakeLists.txt | 1 + .../DynamicallyLoadedSourceKitD.swift | 1 + Sources/SourceKitD/SourceKitD.swift | 1 + Sources/SourceKitLSP/CMakeLists.txt | 1 + .../Clang/ClangLanguageService.swift | 1 + .../SourceKitLSP/IndexProgressManager.swift | 1 + .../MessageHandlingDependencyTracker.swift | 1 + .../SourceKitLSP/SourceKitIndexDelegate.swift | 1 + Sources/SourceKitLSP/SourceKitLSPServer.swift | 1 + .../Swift/CodeCompletionSession.swift | 1 + .../Swift/DiagnosticReportManager.swift | 1 + .../Swift/SwiftLanguageService.swift | 1 + .../Swift/SyntacticTestIndex.swift | 1 + .../SourceKitLSP/WorkDoneProgressManager.swift | 1 + .../SourceKitLSP/WorkDoneProgressState.swift | 1 + Sources/SourceKitLSP/Workspace.swift | 1 + .../AsyncQueue.swift | 0 .../AsyncUtils.swift | 0 Sources/SwiftExtensions/CMakeLists.txt | 14 ++++++++++++++ .../Collection+Only.swift | 0 .../Collection+PartitionIntoBatches.swift | 0 .../Sequence+AsyncMap.swift | 0 .../Task+WithPriorityChangedHandler.swift | 0 .../ThreadSafeBox.swift | 0 Tests/SKSupportTests/AsyncUtilsTests.swift | 1 + Tests/SourceKitDTests/CrashRecoveryTests.swift | 1 + 43 files changed, 69 insertions(+), 8 deletions(-) rename Sources/{SKSupport => SwiftExtensions}/AsyncQueue.swift (100%) rename Sources/{SKSupport => SwiftExtensions}/AsyncUtils.swift (100%) create mode 100644 Sources/SwiftExtensions/CMakeLists.txt rename Sources/{SKSupport => SwiftExtensions}/Collection+Only.swift (100%) rename Sources/{SKSupport => SwiftExtensions}/Collection+PartitionIntoBatches.swift (100%) rename Sources/{SKSupport => SwiftExtensions}/Sequence+AsyncMap.swift (100%) rename Sources/{SKSupport => SwiftExtensions}/Task+WithPriorityChangedHandler.swift (100%) rename Sources/{SKSupport => SwiftExtensions}/ThreadSafeBox.swift (100%) diff --git a/Package.swift b/Package.swift index 6300efcbf..c9f932632 100644 --- a/Package.swift +++ b/Package.swift @@ -80,6 +80,7 @@ let package = Package( "SKSupport", "SourceKitD", "SourceKitLSP", + "SwiftExtensions", .product(name: "ArgumentParser", package: "swift-argument-parser"), .product(name: "SwiftIDEUtils", package: "swift-syntax"), .product(name: "SwiftSyntax", package: "swift-syntax"), @@ -183,6 +184,7 @@ let package = Package( "LanguageServerProtocol", "LanguageServerProtocolJSONRPC", "SKSupport", + "SwiftExtensions", ] ), @@ -195,6 +197,7 @@ let package = Package( "LanguageServerProtocol", "LSPLogging", "SKCore", + "SwiftExtensions", .product(name: "IndexStoreDB", package: "indexstore-db"), ], exclude: ["CMakeLists.txt"] @@ -221,6 +224,7 @@ let package = Package( "LSPLogging", "SKSupport", "SourceKitD", + "SwiftExtensions", .product(name: "SwiftPMDataModel-auto", package: "swift-package-manager"), .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), ], @@ -246,6 +250,7 @@ let package = Package( .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), "LanguageServerProtocol", "LSPLogging", + "SwiftExtensions", ], exclude: ["CMakeLists.txt"], swiftSettings: [.enableExperimentalFeature("StrictConcurrency")] @@ -257,6 +262,7 @@ let package = Package( "LSPTestSupport", "SKSupport", "SKTestSupport", + "SwiftExtensions", ] ), @@ -269,6 +275,7 @@ let package = Package( "LanguageServerProtocol", "LSPLogging", "SKCore", + "SwiftExtensions", .product(name: "SwiftPM-auto", package: "swift-package-manager"), .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), ], @@ -302,6 +309,7 @@ let package = Package( "LSPLogging", "SKCore", "SourceKitLSP", + "SwiftExtensions", .product(name: "ISDBTestSupport", package: "indexstore-db"), .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), ], @@ -319,6 +327,7 @@ let package = Package( "Csourcekitd", "LSPLogging", "SKSupport", + "SwiftExtensions", .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), ], exclude: ["CMakeLists.txt", "sourcekitd_uids.swift.gyb"], @@ -331,6 +340,7 @@ let package = Package( "SourceKitD", "SKCore", "SKTestSupport", + "SwiftExtensions", ] ), @@ -349,6 +359,7 @@ let package = Package( "SKSupport", "SKSwiftPMWorkspace", "SourceKitD", + "SwiftExtensions", .product(name: "IndexStoreDB", package: "indexstore-db"), .product(name: "SwiftBasicFormat", package: "swift-syntax"), .product(name: "SwiftDiagnostics", package: "swift-syntax"), @@ -387,6 +398,13 @@ let package = Package( .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), ] ), + + // MARK: SwiftExtensions + + .target( + name: "SwiftExtensions", + exclude: ["CMakeLists.txt"] + ), ] ) diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 2c729e5f0..48a2bd141 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -13,3 +13,4 @@ add_subdirectory(SKSwiftPMWorkspace) add_subdirectory(SourceKitLSP) add_subdirectory(SourceKitD) add_subdirectory(sourcekit-lsp) +add_subdirectory(SwiftExtensions) diff --git a/Sources/Diagnose/CMakeLists.txt b/Sources/Diagnose/CMakeLists.txt index ca4dcb0f9..edfe1d8c5 100644 --- a/Sources/Diagnose/CMakeLists.txt +++ b/Sources/Diagnose/CMakeLists.txt @@ -29,6 +29,7 @@ target_link_libraries(Diagnose PUBLIC LSPLogging SKCore SourceKitD + SwiftExtensions ArgumentParser SwiftSyntax::SwiftIDEUtils SwiftSyntax::SwiftSyntax diff --git a/Sources/Diagnose/IndexCommand.swift b/Sources/Diagnose/IndexCommand.swift index ad4c52c14..cc0fc5cf7 100644 --- a/Sources/Diagnose/IndexCommand.swift +++ b/Sources/Diagnose/IndexCommand.swift @@ -17,6 +17,7 @@ import LanguageServerProtocol import SKCore import SKSupport import SourceKitLSP +import SwiftExtensions import struct TSCBasic.AbsolutePath import class TSCBasic.Process diff --git a/Sources/LSPTestSupport/TestJSONRPCConnection.swift b/Sources/LSPTestSupport/TestJSONRPCConnection.swift index d175f4afe..768f1caf3 100644 --- a/Sources/LSPTestSupport/TestJSONRPCConnection.swift +++ b/Sources/LSPTestSupport/TestJSONRPCConnection.swift @@ -14,6 +14,7 @@ import InProcessClient import LanguageServerProtocol import LanguageServerProtocolJSONRPC import SKSupport +import SwiftExtensions import XCTest import class Foundation.Pipe diff --git a/Sources/SKCore/BuildServerBuildSystem.swift b/Sources/SKCore/BuildServerBuildSystem.swift index 5fa36777c..a52fd5301 100644 --- a/Sources/SKCore/BuildServerBuildSystem.swift +++ b/Sources/SKCore/BuildServerBuildSystem.swift @@ -16,6 +16,7 @@ import LSPLogging import LanguageServerProtocol import LanguageServerProtocolJSONRPC import SKSupport +import SwiftExtensions import struct TSCBasic.AbsolutePath import protocol TSCBasic.FileSystem diff --git a/Sources/SKCore/CMakeLists.txt b/Sources/SKCore/CMakeLists.txt index efd95b4ca..0190b23b6 100644 --- a/Sources/SKCore/CMakeLists.txt +++ b/Sources/SKCore/CMakeLists.txt @@ -29,5 +29,6 @@ target_link_libraries(SKCore PUBLIC LSPLogging SKSupport SourceKitD + SwiftExtensions PackageModel TSCBasic) diff --git a/Sources/SKCore/TaskScheduler.swift b/Sources/SKCore/TaskScheduler.swift index 2bae9c161..fbe8e947a 100644 --- a/Sources/SKCore/TaskScheduler.swift +++ b/Sources/SKCore/TaskScheduler.swift @@ -14,6 +14,7 @@ import CAtomics import Foundation import LSPLogging import SKSupport +import SwiftExtensions /// See comment on ``TaskDescriptionProtocol/dependencies(to:taskPriority:)`` public enum TaskDependencyAction { diff --git a/Sources/SKCore/Toolchain.swift b/Sources/SKCore/Toolchain.swift index 8b5edb612..dcf108991 100644 --- a/Sources/SKCore/Toolchain.swift +++ b/Sources/SKCore/Toolchain.swift @@ -14,6 +14,7 @@ import LSPLogging import LanguageServerProtocol import RegexBuilder import SKSupport +import SwiftExtensions import enum PackageLoading.Platform import struct TSCBasic.AbsolutePath diff --git a/Sources/SKSupport/CMakeLists.txt b/Sources/SKSupport/CMakeLists.txt index 561d8d9db..b5c7d8d19 100644 --- a/Sources/SKSupport/CMakeLists.txt +++ b/Sources/SKSupport/CMakeLists.txt @@ -1,11 +1,7 @@ add_library(SKSupport STATIC - AsyncQueue.swift - AsyncUtils.swift BuildConfiguration.swift ByteString.swift - Collection+Only.swift - Collection+PartitionIntoBatches.swift Connection+Send.swift dlopen.swift DocumentURI+CustomLogStringConvertible.swift @@ -16,10 +12,7 @@ add_library(SKSupport STATIC Process+WaitUntilExitWithCancellation.swift Random.swift Result.swift - Sequence+AsyncMap.swift SwitchableProcessResultExitStatus.swift - Task+WithPriorityChangedHandler.swift - ThreadSafeBox.swift WorkspaceType.swift ) set_target_properties(SKSupport PROPERTIES @@ -27,5 +20,6 @@ set_target_properties(SKSupport PROPERTIES target_link_libraries(SKSupport PRIVATE LanguageServerProtocol LSPLogging + SwiftExtensions TSCBasic $<$>:Foundation>) diff --git a/Sources/SKSupport/Connection+Send.swift b/Sources/SKSupport/Connection+Send.swift index f7fc6926e..a7bc4aca3 100644 --- a/Sources/SKSupport/Connection+Send.swift +++ b/Sources/SKSupport/Connection+Send.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import LanguageServerProtocol +import SwiftExtensions extension Connection { /// Send the given request to the connection and await its result. diff --git a/Sources/SKSupport/PipeAsStringHandler.swift b/Sources/SKSupport/PipeAsStringHandler.swift index 08669e0f0..53c96defa 100644 --- a/Sources/SKSupport/PipeAsStringHandler.swift +++ b/Sources/SKSupport/PipeAsStringHandler.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import Foundation +import SwiftExtensions /// Gathers data from a stdout or stderr pipe. When it has accumulated a full line, calls the handler to handle the /// string. diff --git a/Sources/SKSwiftPMWorkspace/CMakeLists.txt b/Sources/SKSwiftPMWorkspace/CMakeLists.txt index a89cc2e93..277f83819 100644 --- a/Sources/SKSwiftPMWorkspace/CMakeLists.txt +++ b/Sources/SKSwiftPMWorkspace/CMakeLists.txt @@ -8,6 +8,7 @@ target_link_libraries(SKSwiftPMWorkspace PRIVATE LanguageServerProtocol LSPLogging SKCore + SwiftExtensions TSCBasic) target_link_libraries(SKSwiftPMWorkspace PUBLIC Build diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index efa959cbd..6f3812fe9 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -25,6 +25,7 @@ import SKCore import SKSupport import SourceControl import SourceKitLSPAPI +import SwiftExtensions import Workspace import struct Basics.AbsolutePath @@ -113,7 +114,9 @@ public actor SwiftPMBuildSystem { private let fileSystem: FileSystem private let toolchainRegistry: ToolchainRegistry - private let swiftBuildSupportsPrepareForIndexingTask = SKSupport.ThreadSafeBox?>(initialValue: nil) + private let swiftBuildSupportsPrepareForIndexingTask = SwiftExtensions.ThreadSafeBox?>( + initialValue: nil + ) private var fileToTarget: [DocumentURI: SwiftBuildTarget] = [:] private var sourceDirToTarget: [DocumentURI: SwiftBuildTarget] = [:] diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index 92641990f..cd4e38f1c 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -19,6 +19,7 @@ import LanguageServerProtocolJSONRPC @_spi(Testing) import SKCore import SKSupport import SourceKitLSP +import SwiftExtensions import SwiftSyntax import XCTest diff --git a/Sources/SemanticIndex/CMakeLists.txt b/Sources/SemanticIndex/CMakeLists.txt index e3efc74ae..4623f2614 100644 --- a/Sources/SemanticIndex/CMakeLists.txt +++ b/Sources/SemanticIndex/CMakeLists.txt @@ -14,5 +14,6 @@ set_target_properties(SemanticIndex PROPERTIES target_link_libraries(SemanticIndex PRIVATE LSPLogging SKCore + SwiftExtensions IndexStoreDB $<$>:Foundation>) diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift index 8944da4ba..27f8c0974 100644 --- a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -16,6 +16,7 @@ import LSPLogging import LanguageServerProtocol import SKCore import SKSupport +import SwiftExtensions import struct TSCBasic.AbsolutePath import class TSCBasic.Process diff --git a/Sources/SourceKitD/CMakeLists.txt b/Sources/SourceKitD/CMakeLists.txt index c21e14ab2..21abf549b 100644 --- a/Sources/SourceKitD/CMakeLists.txt +++ b/Sources/SourceKitD/CMakeLists.txt @@ -17,5 +17,6 @@ target_link_libraries(SourceKitD PUBLIC target_link_libraries(SourceKitD PRIVATE LSPLogging SKSupport + SwiftExtensions TSCBasic $<$>:Foundation>) diff --git a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift index 04a5801f5..895a84eca 100644 --- a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift +++ b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift @@ -13,6 +13,7 @@ import Foundation import LSPLogging import SKSupport +import SwiftExtensions import struct TSCBasic.AbsolutePath diff --git a/Sources/SourceKitD/SourceKitD.swift b/Sources/SourceKitD/SourceKitD.swift index 76a81a06e..9884e553d 100644 --- a/Sources/SourceKitD/SourceKitD.swift +++ b/Sources/SourceKitD/SourceKitD.swift @@ -14,6 +14,7 @@ import Dispatch import Foundation import SKSupport +import SwiftExtensions #if compiler(>=6) extension sourcekitd_api_request_handle_t: @retroactive @unchecked Sendable {} diff --git a/Sources/SourceKitLSP/CMakeLists.txt b/Sources/SourceKitLSP/CMakeLists.txt index 7efb356a8..063072dfc 100644 --- a/Sources/SourceKitLSP/CMakeLists.txt +++ b/Sources/SourceKitLSP/CMakeLists.txt @@ -74,6 +74,7 @@ target_link_libraries(SourceKitLSP PUBLIC SKSupport SKSwiftPMWorkspace SourceKitD + SwiftExtensions IndexStoreDB SwiftSyntax::SwiftBasicFormat SwiftSyntax::SwiftDiagnostics diff --git a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift index b5b8efd08..352cd16fc 100644 --- a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift +++ b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift @@ -16,6 +16,7 @@ import LanguageServerProtocol import LanguageServerProtocolJSONRPC import SKCore import SKSupport +import SwiftExtensions import struct TSCBasic.AbsolutePath diff --git a/Sources/SourceKitLSP/IndexProgressManager.swift b/Sources/SourceKitLSP/IndexProgressManager.swift index 383488c08..a8e45a8e6 100644 --- a/Sources/SourceKitLSP/IndexProgressManager.swift +++ b/Sources/SourceKitLSP/IndexProgressManager.swift @@ -15,6 +15,7 @@ import LanguageServerProtocol import SKCore import SKSupport import SemanticIndex +import SwiftExtensions /// Listens for index status updates from `SemanticIndexManagers`. From that information, it manages a /// `WorkDoneProgress` that communicates the index progress to the editor. diff --git a/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift b/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift index 79c09bc5f..2ace1f4c3 100644 --- a/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift +++ b/Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift @@ -13,6 +13,7 @@ import LSPLogging import LanguageServerProtocol import SKSupport +import SwiftExtensions /// A lightweight way of describing tasks that are created from handling LSP /// requests or notifications for the purpose of dependency tracking. diff --git a/Sources/SourceKitLSP/SourceKitIndexDelegate.swift b/Sources/SourceKitLSP/SourceKitIndexDelegate.swift index 8168534ee..bd8c13118 100644 --- a/Sources/SourceKitLSP/SourceKitIndexDelegate.swift +++ b/Sources/SourceKitLSP/SourceKitIndexDelegate.swift @@ -15,6 +15,7 @@ import IndexStoreDB import LSPLogging import SKCore import SKSupport +import SwiftExtensions /// `IndexDelegate` for the SourceKit workspace. /// diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index ee0362945..815faa573 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -23,6 +23,7 @@ import SKSupport import SKSwiftPMWorkspace import SemanticIndex import SourceKitD +import SwiftExtensions import struct PackageModel.BuildFlags import struct TSCBasic.AbsolutePath diff --git a/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift b/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift index baae728bb..c8aa55928 100644 --- a/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift +++ b/Sources/SourceKitLSP/Swift/CodeCompletionSession.swift @@ -15,6 +15,7 @@ import LSPLogging import LanguageServerProtocol import SKSupport import SourceKitD +import SwiftExtensions import SwiftParser @_spi(SourceKitLSP) import SwiftRefactor import SwiftSyntax diff --git a/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift b/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift index 81b4f1cf7..444691f8a 100644 --- a/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift +++ b/Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift @@ -14,6 +14,7 @@ import LSPLogging import LanguageServerProtocol import SKSupport import SourceKitD +import SwiftExtensions import SwiftParserDiagnostics actor DiagnosticReportManager { diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index e488264e0..a96fac481 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -19,6 +19,7 @@ import SKCore import SKSupport import SemanticIndex import SourceKitD +import SwiftExtensions import SwiftParser import SwiftParserDiagnostics import SwiftSyntax diff --git a/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift b/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift index 961dc23ea..b99a24536 100644 --- a/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift +++ b/Sources/SourceKitLSP/Swift/SyntacticTestIndex.swift @@ -14,6 +14,7 @@ import Foundation import LSPLogging import LanguageServerProtocol import SKSupport +import SwiftExtensions /// Task metadata for `SyntacticTestIndexer.indexingQueue` fileprivate enum TaskMetadata: DependencyTracker, Equatable { diff --git a/Sources/SourceKitLSP/WorkDoneProgressManager.swift b/Sources/SourceKitLSP/WorkDoneProgressManager.swift index bd07a5cd1..aba8df0d1 100644 --- a/Sources/SourceKitLSP/WorkDoneProgressManager.swift +++ b/Sources/SourceKitLSP/WorkDoneProgressManager.swift @@ -13,6 +13,7 @@ import Foundation import LanguageServerProtocol import SKSupport +import SwiftExtensions /// Represents a single `WorkDoneProgress` task that gets communicated with the client. /// diff --git a/Sources/SourceKitLSP/WorkDoneProgressState.swift b/Sources/SourceKitLSP/WorkDoneProgressState.swift index 2a2076a58..8f655a60c 100644 --- a/Sources/SourceKitLSP/WorkDoneProgressState.swift +++ b/Sources/SourceKitLSP/WorkDoneProgressState.swift @@ -13,6 +13,7 @@ import LSPLogging import LanguageServerProtocol import SKSupport +import SwiftExtensions /// Keeps track of the state to send work done progress updates to the client final actor WorkDoneProgressState { diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index 60257d82c..31cf8233b 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -16,6 +16,7 @@ import LanguageServerProtocol import SKCore import SKSupport import SemanticIndex +import SwiftExtensions import struct TSCBasic.AbsolutePath import struct TSCBasic.RelativePath diff --git a/Sources/SKSupport/AsyncQueue.swift b/Sources/SwiftExtensions/AsyncQueue.swift similarity index 100% rename from Sources/SKSupport/AsyncQueue.swift rename to Sources/SwiftExtensions/AsyncQueue.swift diff --git a/Sources/SKSupport/AsyncUtils.swift b/Sources/SwiftExtensions/AsyncUtils.swift similarity index 100% rename from Sources/SKSupport/AsyncUtils.swift rename to Sources/SwiftExtensions/AsyncUtils.swift diff --git a/Sources/SwiftExtensions/CMakeLists.txt b/Sources/SwiftExtensions/CMakeLists.txt new file mode 100644 index 000000000..8fe3b3223 --- /dev/null +++ b/Sources/SwiftExtensions/CMakeLists.txt @@ -0,0 +1,14 @@ + +add_library(SwiftExtensions STATIC + AsyncQueue.swift + AsyncUtils.swift + Collection+Only.swift + Collection+PartitionIntoBatches.swift + Sequence+AsyncMap.swift + Task+WithPriorityChangedHandler.swift + ThreadSafeBox.swift +) +set_target_properties(SwiftExtensions PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) +target_link_libraries(SwiftExtensions PRIVATE + $<$>:Foundation>) diff --git a/Sources/SKSupport/Collection+Only.swift b/Sources/SwiftExtensions/Collection+Only.swift similarity index 100% rename from Sources/SKSupport/Collection+Only.swift rename to Sources/SwiftExtensions/Collection+Only.swift diff --git a/Sources/SKSupport/Collection+PartitionIntoBatches.swift b/Sources/SwiftExtensions/Collection+PartitionIntoBatches.swift similarity index 100% rename from Sources/SKSupport/Collection+PartitionIntoBatches.swift rename to Sources/SwiftExtensions/Collection+PartitionIntoBatches.swift diff --git a/Sources/SKSupport/Sequence+AsyncMap.swift b/Sources/SwiftExtensions/Sequence+AsyncMap.swift similarity index 100% rename from Sources/SKSupport/Sequence+AsyncMap.swift rename to Sources/SwiftExtensions/Sequence+AsyncMap.swift diff --git a/Sources/SKSupport/Task+WithPriorityChangedHandler.swift b/Sources/SwiftExtensions/Task+WithPriorityChangedHandler.swift similarity index 100% rename from Sources/SKSupport/Task+WithPriorityChangedHandler.swift rename to Sources/SwiftExtensions/Task+WithPriorityChangedHandler.swift diff --git a/Sources/SKSupport/ThreadSafeBox.swift b/Sources/SwiftExtensions/ThreadSafeBox.swift similarity index 100% rename from Sources/SKSupport/ThreadSafeBox.swift rename to Sources/SwiftExtensions/ThreadSafeBox.swift diff --git a/Tests/SKSupportTests/AsyncUtilsTests.swift b/Tests/SKSupportTests/AsyncUtilsTests.swift index dd95aadfe..338fe9120 100644 --- a/Tests/SKSupportTests/AsyncUtilsTests.swift +++ b/Tests/SKSupportTests/AsyncUtilsTests.swift @@ -12,6 +12,7 @@ import LSPTestSupport import SKSupport +import SwiftExtensions import XCTest final class AsyncUtilsTests: XCTestCase { diff --git a/Tests/SourceKitDTests/CrashRecoveryTests.swift b/Tests/SourceKitDTests/CrashRecoveryTests.swift index 339387b54..d644314e5 100644 --- a/Tests/SourceKitDTests/CrashRecoveryTests.swift +++ b/Tests/SourceKitDTests/CrashRecoveryTests.swift @@ -18,6 +18,7 @@ import SKSupport import SKTestSupport import SourceKitD import SourceKitLSP +import SwiftExtensions import XCTest import enum PackageLoading.Platform From 1553c9c17a8bef7f439b299c3d6e83665dcf7c19 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 23:09:36 -0700 Subject: [PATCH 64/69] =?UTF-8?q?Don=E2=80=99t=20index=20files=20with=20fa?= =?UTF-8?q?llback=20build=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fallback build settings don’t even have an indexstore path set, so we would never pick up any index data generated from them. Also, indexing with fallback args has some other problems: - If it did generate a unit file, we would consider the file’s index up-to-date even if the compiler arguments change, which basically means that we wouldn’t get any up-to-date-index even when we do get build settings for the file. - It’s unlikely that the index from a single file with fallback arguments will be very useful as it can’t tie into the rest of the project. --- .../UpdateIndexStoreTaskDescription.swift | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift index 8944da4ba..2d728bef3 100644 --- a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -225,6 +225,17 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { logger.error("Not indexing \(file.forLogging) because it has no compiler arguments") return } + if buildSettings.isFallback { + // Fallback build settings don’t even have an indexstore path set, so they can't generate index data that we would + // pick up. Also, indexing with fallback args has some other problems: + // - If it did generate a unit file, we would consider the file’s index up-to-date even if the compiler arguments + // change, which means that we wouldn't get any up-to-date-index even when we have build settings for the file. + // - It's unlikely that the index from a single file with fallback arguments will be very useful as it can't tie + // into the rest of the project. + // So, don't index the file. + logger.error("Not indexing \(file.forLogging) because it has fallback compiler arguments") + return + } guard let toolchain = await buildSystemManager.toolchain(for: file.mainFile, language) else { logger.error( "Not updating index store for \(file.forLogging) because no toolchain could be determined for the document" From ece8634be59580b83497de6d21a112dce7179de4 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 4 Jun 2024 07:58:16 -0700 Subject: [PATCH 65/69] Keep track of recently finished requests This is only used so we don't log an error when receiving a `CancelRequestNotification` for a request that has just returned a response. --- Sources/SourceKitLSP/SourceKitLSPServer.swift | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index ee0362945..2daa00d58 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -176,10 +176,22 @@ public actor SourceKitLSPServer { /// Used to cancel the tasks if the client requests cancellation. private var inProgressRequests: [RequestID: Task<(), Never>] = [:] + /// Up to 10 request IDs that have recently finished. + /// + /// This is only used so we don't log an error when receiving a `CancelRequestNotification` for a request that has + /// just returned a response. + private var recentlyFinishedRequests: [RequestID] = [] + /// - Note: Needed so we can set an in-progress request from a different /// isolation context. private func setInProgressRequest(for id: RequestID, task: Task<(), Never>?) { self.inProgressRequests[id] = task + if task == nil { + recentlyFinishedRequests.append(id) + while recentlyFinishedRequests.count > 10 { + recentlyFinishedRequests.removeFirst() + } + } } var onExit: () -> Void @@ -1191,16 +1203,15 @@ extension SourceKitLSPServer { // Since the request is very cheap to execute and stops other requests // from performing more work, we execute it with a high priority. cancellationMessageHandlingQueue.async(priority: .high) { - guard let task = await self.inProgressRequests[notification.id] else { + if let task = await self.inProgressRequests[notification.id] { + task.cancel() + return + } + if await !self.recentlyFinishedRequests.contains(notification.id) { logger.error( - """ - Cannot cancel request \(notification.id, privacy: .public) because it hasn't been scheduled for execution \ - yet or because the request already returned a response - """ + "Cannot cancel request \(notification.id, privacy: .public) because it hasn't been scheduled for execution yet" ) - return } - task.cancel() } } From cf482529026af64b14da7d8e85f1241a33f19942 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 4 Jun 2024 08:11:30 -0700 Subject: [PATCH 66/69] =?UTF-8?q?Log=20when=20the=2010=20year=20wait=20tha?= =?UTF-8?q?t=E2=80=99s=20parking=20the=20main=20thread=20expires=20for=20s?= =?UTF-8?q?ome=20reason?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not sure if this ever happens but if it does, we should log it. Also, just start waiting again for another 10 years. --- Sources/sourcekit-lsp/SourceKitLSP.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index 3b89b55b8..b322c2bcb 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -274,7 +274,10 @@ struct SourceKitLSP: AsyncParsableCommand { // Park the main function by sleeping for 10 years. // All request handling is done on other threads and sourcekit-lsp exits by calling `_Exit` when it receives a // shutdown notification. - try await Task.sleep(for: .seconds(60 * 60 * 24 * 365 * 10)) + while true { + try? await Task.sleep(for: .seconds(60 * 60 * 24 * 365 * 10)) + logger.fault("10 year wait that's parking the main thread expired. Waiting again.") + } } } From d8067fb4390e4569fa4941c51bf325179c4c359b Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 4 Jun 2024 11:19:15 -0700 Subject: [PATCH 67/69] Add some more overview documentation documents --- Documentation/Environment Variables.md | 19 ++ Documentation/LSP Extensions.md | 433 +++++++++++++++++++++++++ Documentation/Overview.md | 34 ++ Documentation/Testing.md | 19 ++ 4 files changed, 505 insertions(+) create mode 100644 Documentation/Environment Variables.md create mode 100644 Documentation/LSP Extensions.md create mode 100644 Documentation/Overview.md create mode 100644 Documentation/Testing.md diff --git a/Documentation/Environment Variables.md b/Documentation/Environment Variables.md new file mode 100644 index 000000000..6b85ff6f1 --- /dev/null +++ b/Documentation/Environment Variables.md @@ -0,0 +1,19 @@ +# Environment variables + +The following environment variables can be used to control some behavior in SourceKit-LSP + +## Build time + +- `SOURCEKITLSP_FORCE_NON_DARWIN_LOGGER`: Use the `NonDarwinLogger` to log to stderr, even when building SourceKit-LSP on macOS. This is useful when running tests using `swift test` because it writes the log messages to stderr, which is displayed during the `swift test` invocation. +- `SOURCEKIT_LSP_CI_INSTALL`: Modifies rpaths in a way that’s necessary to build SourceKit-LSP to be included in a distributed toolchain. Should not be used locally. +- `SWIFTCI_USE_LOCAL_DEPS`: Assume that all of SourceKit-LSP’s dependencies are checked out next to it and use those instead of cloning the repositories. Primarily intended for CI environments that check out related branches. + +## Runtime + +- `SOURCEKITLSP_LOG_LEVEL`: When using `NonDarwinLogger`, specify the level at which messages should be logged. Defaults to `default`. Use `debug` to increase to the highest log level. +- `SOURCEKITLSP_LOG_PRIVACY_LEVEL`: When using `NonDarwinLogger`, specifies whether information that might contain personal information (essentially source code) should be logged. Defaults to `private`, which logs this information. Set to `public` to redact this information. + +## Testing +- `SKIP_LONG_TESTS`: Skip tests that typically take more than 1s to execute. +- `SOURCEKITLSP_KEEP_TEST_SCRATCH_DIR`: Does not delete the temporary files created during test execution. Allows inspection of the test projects after the test finishes. +- `SOURCEKIT_LSP_TEST_MODULE_CACHE`: Specifies where tests should store their shared module cache. Defaults to writing the module cache to a temporary directory. Intended so that CI systems can clean the module cache directory after running. diff --git a/Documentation/LSP Extensions.md b/Documentation/LSP Extensions.md new file mode 100644 index 000000000..bdfde470c --- /dev/null +++ b/Documentation/LSP Extensions.md @@ -0,0 +1,433 @@ +# LSP Extensions + +SourceKit-LSP extends the LSP protocol in the following ways. + +## `PublishDiagnosticsClientCapabilities` + +Added field (this is an extension from clangd that SourceKit-LSP re-exposes): + +```ts +/** + * Requests that SourceKit-LSP send `Diagnostic.codeActions`. + */ +codeActionsInline?: bool +``` + +## `Diagnostic` + +Added field (this is an extension from clangd that SourceKit-LSP re-exposes): + +```ts +/** + * All the code actions that address this diagnostic. + */ +codeActions: CodeAction[]? +``` + +## `DiagnosticRelatedInformation` + +Added field (this is an extension from clangd that SourceKit-LSP re-exposes): + +```ts +/** + * All the code actions that address the parent diagnostic via this note. + */ +codeActions: CodeAction[]? +``` + +## Semantic token types + +Added case + +```ts +/** + * An identifier that hasn't been further classified + */ +identifier = 'identifier' +``` + +## `WorkspaceFolder` + +Added field: + +```ts +/** + * Build settings that should be used for this workspace. + * + * For arguments that have a single value (like the build configuration), this takes precedence over the global + * options set when launching sourcekit-lsp. For all other options, the values specified in the workspace-specific + * build setup are appended to the global options. + */ +var buildSetup?: WorkspaceBuildSetup +``` + +with + +```ts +/** + * The configuration to build a workspace in. + */ +export enum BuildConfiguration { + case debug = 'debug' + case release = 'release' +} + +/** + * The type of workspace; default workspace type selection logic can be overridden. + */ +export enum WorkspaceType { + buildServer = 'buildServer' + compilationDatabase = 'compilationDatabase' + swiftPM = 'swiftPM' +} + +/// Build settings that should be used for a workspace. +interface WorkspaceBuildSetup { + /** + * The configuration that the workspace should be built in. + */ + buildConfiguration?: BuildConfiguration + + /** + * The default workspace type to use for this workspace. + */ + defaultWorkspaceType?: WorkspaceType + + /** + * The build directory for the workspace. + */ + scratchPath?: DocumentURI + + /** + * Arguments to be passed to any C compiler invocations. + */ + cFlags?: string[] + + /** + * Arguments to be passed to any C++ compiler invocations. + */ + cxxFlags?: string[] + + /** + * Arguments to be passed to any linker invocations. + */ + linkerFlags?: string[] + + /** + * Arguments to be passed to any Swift compiler invocations. + */ + swiftFlags?: string[] +} +``` + +## `textDocument/completion` + +Added field: + +```ts +/** + * Options to control code completion behavior in Swift + */ +sourcekitlspOptions?: SKCompletionOptions +``` + +with + +```ts +interface SKCompletionOptions { + /** + * The maximum number of completion results to return, or `null` for unlimited. + */ + maxResults?: int +} +``` + +## `textDocument/interface` + +New request that request a textual interface of a module to display in the IDE. Used internally within SourceKit-LSP + +- params: `OpenInterfaceParams` +- result: `InterfaceDetails?` + +```ts +export interface OpenInterfaceRequest: TextDocumentRequest, Hashable { + /** + * The document whose compiler arguments should be used to generate the interface. + */ + textDocument: TextDocumentIdentifier + + /** + * The module to generate an index for. + */ + moduleName: string + + /** + * The module group name. + */ + groupName?: string + + /** + * The symbol USR to search for in the generated module interface. + */ + symbolUSR?: string +} + +interface InterfaceDetails { + uri: DocumentURI + position?: Position +} +``` + +## `textDocument/symbolInfo` + +New request for semantic information about the symbol at a given location. + +This request looks up the symbol (if any) at a given text document location and returns +SymbolDetails for that location, including information such as the symbol's USR. The symbolInfo +request is not primarily designed for editors, but instead as an implementation detail of how +one LSP implementation (e.g. SourceKit) gets information from another (e.g. clangd) to use in +performing index queries or otherwise implementing the higher level requests such as definition. + +This request is an extension of the `textDocument/symbolInfo` request defined by clangd. + +- params: `SymbolInfoParams` +- result: `SymbolDetails[]` + + +```ts +export interface SymbolInfoParams { + /** + * The document in which to lookup the symbol location. + */ + textDocument: TextDocumentIdentifier + + /** + * The document location at which to lookup symbol information. + */ + position: Position +} + +interface ModuleInfo { + /** + * The name of the module in which the symbol is defined. + */ + moduleName: string + + /** + * If the symbol is defined within a subgroup of a module, the name of the group. Otherwise `nil`. + */ + groupName?: string +} + +interface SymbolDetails { + /** + * The name of the symbol, if any. + */ + name?: string + + /** + * The name of the containing type for the symbol, if any. + * + * For example, in the following snippet, the `containerName` of `foo()` is `C`. + * + * ```c++ + * class C { + * void foo() {} + * } + */ + containerName?: string + + /** + * The USR of the symbol, if any. + */ + usr?: string + + /** + * Best known declaration or definition location without global knowledge. + * + * For a local or private variable, this is generally the canonical definition location - + * appropriate as a response to a `textDocument/definition` request. For global symbols this is + * the best known location within a single compilation unit. For example, in C++ this might be + * the declaration location from a header as opposed to the definition in some other + * translation unit. + */ + bestLocalDeclaration?: Location + + /** + * The kind of the symbol + */ + kind?: SymbolKind + + /** + * Whether the symbol is a dynamic call for which it isn't known which method will be invoked at runtime. This is + * the case for protocol methods and class functions. + * + * Optional because `clangd` does not return whether a symbol is dynamic. + */ + isDynamic?: bool + + /** + * Whether this symbol is defined in the SDK or standard library. + * + * This property only applies to Swift symbols + */ + isSystem?: bool + + /** + * If the symbol is dynamic, the USRs of the types that might be called. + * + * This is relevant in the following cases + * ```swift + * class A { + * func doThing() {} + * } + * class B: A {} + * class C: B { + * override func doThing() {} + * } + * class D: A { + * override func doThing() {} + * } + * func test(value: B) { + * value.doThing() + * } + * ``` + * + * The USR of the called function in `value.doThing` is `A.doThing` (or its + * mangled form) but it can never call `D.doThing`. In this case, the + * receiver USR would be `B`, indicating that only overrides of subtypes in + * `B` may be called dynamically. + */ + receiverUsrs?: string[] + + /** + * If the symbol is defined in a module that doesn't have source information associated with it, the name and group + * and group name that defines this symbol. + * + * This property only applies to Swift symbols. + */ + systemModule?: ModuleInfo +} +``` + +## `textDocument/tests` + +New request that returns symbols for all the test classes and test methods within a file. + +- params: `DocumentTestsParams` +- result: `TestItem[]` + +```ts +interface TestTag { + /** + * ID of the test tag. `TestTag` instances with the same ID are considered to be identical. + */ + id: string +} + +/** + * A test item that can be shown an a client's test explorer or used to identify tests alongside a source file. + * + * A `TestItem` can represent either a test suite or a test itself, since they both have similar capabilities. + * + * This type matches the `TestItem` type in Visual Studio Code to a fair degree. + */ +interface TestItem { + /** + * Identifier for the `TestItem`. + * + * This identifier uniquely identifies the test case or test suite. It can be used to run an individual test (suite). + */ + id: string + + /** + * Display name describing the test. + */ + label: string + + /** + * Optional description that appears next to the label. + */ + description?: string + + /** + * A string that should be used when comparing this item with other items. + * + * When `nil` the `label` is used. + */ + sortText?: string + + /** + * Whether the test is disabled. + */ + disabled: bool + + /** + * The type of test, eg. the testing framework that was used to declare the test. + */ + style: string + + /** + * The location of the test item in the source code. + */ + location: Location + + /** + * The children of this test item. + * + * For a test suite, this may contain the individual test cases or nested suites. + */ + children: TestItem[]] + + /** + * Tags associated with this test item. + */ + tags: TestTag[] +} + +export interface DocumentTestsParams { + /** + * The text document. + */ + textDocument: TextDocumentIdentifier; +} +``` + +## `textDocument/symbolInfo` + +TODO + +## `window/logMessage` + +Added field: + +```ts +/** + * Asks the client to log the message to a log with this name, to organize log messages from different aspects (eg. general logging vs. logs from background indexing). + * + * Clients may ignore this parameter and add the message to the global log + */ +logName?: string +``` + +## `workspace/_pollIndex` + +New request to wait until the index is up-to-date. + +- params: `PollIndexParams` +- result: `void` + +```ts +export interface PollIndexParams {} +``` + +## `workspace/tests` + +New request that returns symbols for all the test classes and test methods within the current workspace. + +- params: `WorkspaceTestsParams` +- result: `TestItem[]` + +```ts +export interface WorkspaceTestsParams {} +``` diff --git a/Documentation/Overview.md b/Documentation/Overview.md new file mode 100644 index 000000000..331cbff5e --- /dev/null +++ b/Documentation/Overview.md @@ -0,0 +1,34 @@ +# Design Overview + +This document gives a general overview of the major design concepts in SourceKit-LSP and how components fit together. Detailed doc comments can usually be found on the individual types. + +## Message handling + +LSP Messages (ie. LSP requests or notifications) sent to SourceKit-LSP via stdin are handled using `JSONRPCConnection`, which decodes the JSON in the message to a type from `LanguageServerProtocol`. We need to guarantee that LSP messages are generally handled in-order. Re-ordering eg. `textDocument/didChange` notifications would be a major issue since it results in out-of-sync document contents. Since Swift concurrency does not make any ordering guarantees (Tasks and async calls can be executed in any order), the `Connection` types are serial and don’t use Swift Concurrency features. + +After the message has been decoded, the `SourceKitLSPServer` handles it on its `messageHandlingQueue`. `SourceKitLSPServer` is capable of interpreting the semantic meaning of requests and is able to decide whether requests can be executed in parallel, modelled by `MessageHandlingDependencyTracker`. For example, edits to different documents don’t have any dependencies and can be handled concurrently or out-of-order. Similarly requests to the same document that aren’t modifying it can be handled concurrently. + +## Language services + +To provide language-specific functionality `SourceKitLSPServer` delegates to the types that implement `LanguageService`. This is either `SwiftLanguageService`, which implements language functionality for Swift files based on sourcekitd and swift-syntax, or `ClangLanguageService`, which launches a `clangd` process to provide functionality for C, C++, Objective-C and Objective-C++. + +For requests that require knowledge about the project’s index, like call hierarchy or rename of global symbols, `SourceKitLSPServer` enriches the information returned from the language service with information from the [index](http://github.com/apple/indexstore-db/). + +### sourcekitd + +[sourcekitd](https://github.com/apple/swift/tree/main/tools/SourceKit) is implemented in the Swift compiler’s repository and uses the Swift compiler’s understanding of Swift code to provide semantic functionality. On macOS, sourcekitd is run as an XPC service and all other platforms sourcekitd is run in the sourcekit-lsp process. This means that on macOS, a crash of sourcekitd can be recovered by re-launching sourcekitd, while on other platforms a crash inside sourcekitd crashes sourcekit-lsp. + +## Build systems + +Basically all semantic functionality requires knowledge about how a file is being built (like module search paths to be able to resolve `import` statements). SourceKit-LSP consults a `BuildSystem` to get this information. Note that the term build system has a somewhat loose definition in SourceKit-LSP. It’s less about building the project and more about providing information that would be used if building the project. Build systems that are currently supported are: +- `SwiftPMBuildSystem`: Reads a `Package.swift` at the project’s root +- `CompilationDatabaseBuildSystem`: Reads a `compile_commands.json` file to provide build settings for files. `compile_commands.json` can eg. be generated by CMake. +- `BuildServerBuildSystem`: Launches an external build server process to provide build settings. + +## Logging + +SourceKit-LSP has fairly extensive logging to help diagnose issues. The way logging works depends on the platform that SourceKit-LSP is running on. + +On macOS, SourceKit-LSP logs to the system log. [CONTRIBUTING.md](../CONTRIBUTING.md#logging) contains some information about how to read the system logs. Since [OSLog](https://developer.apple.com/documentation/os/logging) cannot be wrapped, the decision to log to macOS’s system log is done at build time and cannot be modified at runtime. + +On other platforms, the `NonDarwinLogger` types are used to log messages. These types are API-compatible with OSLog. Log messages are written to stderr by default. When possible, `SourceKitLSP.run` will redirect the log messages to log files in `/var/log/sourcekit-lsp` on launch. diff --git a/Documentation/Testing.md b/Documentation/Testing.md new file mode 100644 index 000000000..3742f4cde --- /dev/null +++ b/Documentation/Testing.md @@ -0,0 +1,19 @@ +# Testing + +Most tests in SourceKit-LSP are integration tests that create a `SourceKitLSPServer` instance in-process, initialize it and send messages to it. The test support modules essentially define four ways of creating test projects. + +### `TestSourceKitLSPClient` + +Launches a `SourceKitLSPServer` in-process. Documents can be opened within it but these documents don't have any representation on the file system. `TestSourceKitLSPClient` has the lowest overhead and is the basis for all the other test projects. Because there are no files on disk, this type cannot test anything that requires cross-file functionality or exercise requests that require an index. + +### `IndexedSingleSwiftFileTestProject` + +Creates a single `.swift` file on disk, indexes it and then opens it using a `TestSourceKitLSPClient`. This is the best choice for tests that require an index but don’t need to exercise any cross-file functionality. + +### `SwiftPMTestProject` + +Creates a SwiftPM project on disk that allows testing of cross-file and cross-module functionality. By default the `SwiftPMTestProject` does not build an index or build any Swift modules, which is often sufficient when testing cross-file functionality within a single module. When cross-module functionality or an index is needed, background indexing can be enabled using `enableBackgroundIndexing: true`, which waits for background indexing to finish before allowing any requests. + +## `MultiFileTestProject` + +This is the most flexible test type that writes arbitrary files to disk. It provides less functionality out-of-the-box but is capable of eg. representing workspaces with multiple SwiftPM projects or projects that have `compile_commands.json`. From 3e11cd6bc893b3f40ac7577671d2b106c3b9fcfa Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 21:18:39 -0700 Subject: [PATCH 68/69] Unify `withLock` implementations Turns out that also most of the `withLock` definitions were never used. --- Sources/SourceKitD/SourceKitDRegistry.swift | 9 -------- .../Clang/ClangLanguageService.swift | 9 -------- Sources/SwiftExtensions/AsyncQueue.swift | 9 -------- Sources/SwiftExtensions/CMakeLists.txt | 1 + Sources/SwiftExtensions/NSLock+WithLock.swift | 21 +++++++++++++++++++ Sources/SwiftExtensions/ThreadSafeBox.swift | 9 -------- 6 files changed, 22 insertions(+), 36 deletions(-) create mode 100644 Sources/SwiftExtensions/NSLock+WithLock.swift diff --git a/Sources/SourceKitD/SourceKitDRegistry.swift b/Sources/SourceKitD/SourceKitDRegistry.swift index 21deca62c..b3494c4dd 100644 --- a/Sources/SourceKitD/SourceKitDRegistry.swift +++ b/Sources/SourceKitD/SourceKitDRegistry.swift @@ -14,15 +14,6 @@ import Foundation import struct TSCBasic.AbsolutePath -extension NSLock { - /// NOTE: Keep in sync with SwiftPM's 'Sources/Basics/NSLock+Extensions.swift' - internal func withLock(_ body: () throws -> T) rethrows -> T { - lock() - defer { unlock() } - return try body() - } -} - /// The set of known SourceKitD instances, uniqued by path. /// /// It is not generally safe to have two instances of SourceKitD for the same libsourcekitd, so diff --git a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift index 18cf90837..ffd834d82 100644 --- a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift +++ b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift @@ -24,15 +24,6 @@ import struct TSCBasic.AbsolutePath import WinSDK #endif -extension NSLock { - /// NOTE: Keep in sync with SwiftPM's 'Sources/Basics/NSLock+Extensions.swift' - fileprivate func withLock(_ body: () throws -> T) rethrows -> T { - lock() - defer { unlock() } - return try body() - } -} - /// A thin wrapper over a connection to a clangd server providing build setting handling. /// /// In addition, it also intercepts notifications and replies from clangd in order to do things diff --git a/Sources/SwiftExtensions/AsyncQueue.swift b/Sources/SwiftExtensions/AsyncQueue.swift index 31d58687d..58125ad08 100644 --- a/Sources/SwiftExtensions/AsyncQueue.swift +++ b/Sources/SwiftExtensions/AsyncQueue.swift @@ -26,15 +26,6 @@ extension Task: AnyTask { } } -fileprivate extension NSLock { - /// NOTE: Keep in sync with SwiftPM's 'Sources/Basics/NSLock+Extensions.swift' - func withLock(_ body: () throws -> T) rethrows -> T { - lock() - defer { unlock() } - return try body() - } -} - /// A type that is able to track dependencies between tasks. public protocol DependencyTracker: Sendable { /// Whether the task described by `self` needs to finish executing before diff --git a/Sources/SwiftExtensions/CMakeLists.txt b/Sources/SwiftExtensions/CMakeLists.txt index 8fe3b3223..7b024d4d1 100644 --- a/Sources/SwiftExtensions/CMakeLists.txt +++ b/Sources/SwiftExtensions/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(SwiftExtensions STATIC AsyncUtils.swift Collection+Only.swift Collection+PartitionIntoBatches.swift + NSLock+WithLock.swift Sequence+AsyncMap.swift Task+WithPriorityChangedHandler.swift ThreadSafeBox.swift diff --git a/Sources/SwiftExtensions/NSLock+WithLock.swift b/Sources/SwiftExtensions/NSLock+WithLock.swift new file mode 100644 index 000000000..9c0d9bd6e --- /dev/null +++ b/Sources/SwiftExtensions/NSLock+WithLock.swift @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +extension NSLock { + func withLock(_ body: () throws -> T) rethrows -> T { + lock() + defer { unlock() } + return try body() + } +} diff --git a/Sources/SwiftExtensions/ThreadSafeBox.swift b/Sources/SwiftExtensions/ThreadSafeBox.swift index a4f825eaf..091d7e287 100644 --- a/Sources/SwiftExtensions/ThreadSafeBox.swift +++ b/Sources/SwiftExtensions/ThreadSafeBox.swift @@ -12,15 +12,6 @@ import Foundation -extension NSLock { - /// NOTE: Keep in sync with SwiftPM's 'Sources/Basics/NSLock+Extensions.swift' - fileprivate func withLock(_ body: () throws -> T) rethrows -> T { - lock() - defer { unlock() } - return try body() - } -} - /// A thread safe container that contains a value of type `T`. /// /// - Note: Unchecked sendable conformance because value is guarded by a lock. From 37e6a8a65a42fdae020e97add0a8be22a9733540 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Mon, 3 Jun 2024 14:30:20 -0700 Subject: [PATCH 69/69] Wait for cancellation to propagate in `testDontReturnEmptyDiagnosticsIfDiagnosticRequestIsCancelled` I saw a few non-deterministic test failures. I think the issue was that handling of the `CancelRequestNotification` is done asynchronously, which left a short window in which the sourcekitd diagnostics request could run and return results instead of being cancelled. Wait for the diagnostic request to actually be cancelled before running it for real. While doing this, also introduce proper sourcekitd test hooks instead of relying on the preparation test hooks, which just got run as a side effect. --- .../RunSourcekitdRequestCommand.swift | 3 ++- .../DynamicallyLoadedSourceKitD.swift | 25 +++++++++++++++---- Sources/SourceKitD/SourceKitD.swift | 14 +++++++---- Sources/SourceKitD/SourceKitDRegistry.swift | 12 +++------ .../SourceKitLSPServer+Options.swift | 6 ++++- .../Swift/SwiftLanguageService.swift | 5 +++- Tests/DiagnoseTests/DiagnoseTests.swift | 3 ++- .../SourceKitDRegistryTests.swift | 1 + Tests/SourceKitDTests/SourceKitDTests.swift | 5 +++- .../PullDiagnosticsTests.swift | 21 ++++++++++++---- 10 files changed, 66 insertions(+), 29 deletions(-) diff --git a/Sources/Diagnose/RunSourcekitdRequestCommand.swift b/Sources/Diagnose/RunSourcekitdRequestCommand.swift index 6cfe921c0..6be683609 100644 --- a/Sources/Diagnose/RunSourcekitdRequestCommand.swift +++ b/Sources/Diagnose/RunSourcekitdRequestCommand.swift @@ -55,7 +55,8 @@ public struct RunSourceKitdRequestCommand: AsyncParsableCommand { throw ExitCode(1) } let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( - dylibPath: try! AbsolutePath(validating: sourcekitdPath) + dylibPath: try! AbsolutePath(validating: sourcekitdPath), + testHooks: SourceKitDTestHooks() ) if let lineColumn = position?.split(separator: ":", maxSplits: 2).map(Int.init), diff --git a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift index 04a5801f5..e08992dd1 100644 --- a/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift +++ b/Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift @@ -25,6 +25,14 @@ extension sourcekitd_api_keys: @unchecked Sendable {} extension sourcekitd_api_requests: @unchecked Sendable {} extension sourcekitd_api_values: @unchecked Sendable {} +public struct SourceKitDTestHooks: Sendable { + public var sourcekitdRequestDidStart: (@Sendable (SKDRequestDictionary) -> Void)? + + public init(sourcekitdRequestDidStart: (@Sendable (SKDRequestDictionary) -> Void)? = nil) { + self.sourcekitdRequestDidStart = sourcekitdRequestDidStart + } +} + /// Wrapper for sourcekitd, taking care of initialization, shutdown, and notification handler /// multiplexing. /// @@ -32,10 +40,12 @@ extension sourcekitd_api_values: @unchecked Sendable {} /// `set_notification_handler`, which are global state managed internally by this class. public actor DynamicallyLoadedSourceKitD: SourceKitD { /// The path to the sourcekitd dylib. - public let path: AbsolutePath + private let path: AbsolutePath /// The handle to the dylib. - let dylib: DLHandle + private let dylib: DLHandle + + public let testHooks: SourceKitDTestHooks /// The sourcekitd API functions. public let api: sourcekitd_api_functions_t @@ -54,18 +64,23 @@ public actor DynamicallyLoadedSourceKitD: SourceKitD { /// List of notification handlers that will be called for each notification. private var notificationHandlers: [WeakSKDNotificationHandler] = [] - public static func getOrCreate(dylibPath: AbsolutePath) async throws -> SourceKitD { + /// If there is already a `sourcekitd` instance from the given return it, otherwise create a new one. + /// + /// `testHooks` are only considered when an instance is being created. If a sourcekitd instance at the given path + /// already exists, its test hooks will be used. + public static func getOrCreate(dylibPath: AbsolutePath, testHooks: SourceKitDTestHooks) async throws -> SourceKitD { try await SourceKitDRegistry.shared - .getOrAdd(dylibPath, create: { try DynamicallyLoadedSourceKitD(dylib: dylibPath) }) + .getOrAdd(dylibPath, create: { try DynamicallyLoadedSourceKitD(dylib: dylibPath, testHooks: testHooks) }) } - init(dylib path: AbsolutePath) throws { + init(dylib path: AbsolutePath, testHooks: SourceKitDTestHooks) throws { self.path = path #if os(Windows) self.dylib = try dlopen(path.pathString, mode: []) #else self.dylib = try dlopen(path.pathString, mode: [.lazy, .local, .first]) #endif + self.testHooks = testHooks self.api = try sourcekitd_api_functions_t(self.dylib) self.keys = sourcekitd_api_keys(api: self.api) self.requests = sourcekitd_api_requests(api: self.api) diff --git a/Sources/SourceKitD/SourceKitD.swift b/Sources/SourceKitD/SourceKitD.swift index 76a81a06e..61096f05a 100644 --- a/Sources/SourceKitD/SourceKitD.swift +++ b/Sources/SourceKitD/SourceKitD.swift @@ -30,6 +30,8 @@ extension sourcekitd_api_request_handle_t: @unchecked Sendable {} /// *Implementors* are expected to handle initialization and shutdown, e.g. during `init` and /// `deinit` or by wrapping an existing sourcekitd session that outlives this object. public protocol SourceKitD: AnyObject, Sendable { + var testHooks: SourceKitDTestHooks { get } + /// The sourcekitd API functions. var api: sourcekitd_api_functions_t { get } @@ -95,18 +97,20 @@ extension SourceKitD { /// - req: The request to send to sourcekitd. /// - fileContents: The contents of the file that the request operates on. If sourcekitd crashes, the file contents /// will be logged. - public func send(_ req: SKDRequestDictionary, fileContents: String?) async throws -> SKDResponseDictionary { - log(request: req) + public func send(_ request: SKDRequestDictionary, fileContents: String?) async throws -> SKDResponseDictionary { + log(request: request) + + testHooks.sourcekitdRequestDidStart?(request) let sourcekitdResponse: SKDResponse = try await withCancellableCheckedThrowingContinuation { continuation in var handle: sourcekitd_api_request_handle_t? = nil - api.send_request(req.dict, &handle) { response in + api.send_request(request.dict, &handle) { response in continuation.resume(returning: SKDResponse(response!, sourcekitd: self)) } return handle } cancel: { handle in if let handle { - logRequestCancellation(request: req) + logRequestCancellation(request: request) api.cancel_request(handle) } } @@ -115,7 +119,7 @@ extension SourceKitD { guard let dict = sourcekitdResponse.value else { if sourcekitdResponse.error == .connectionInterrupted { - log(crashedRequest: req, fileContents: fileContents) + log(crashedRequest: request, fileContents: fileContents) } throw sourcekitdResponse.error! } diff --git a/Sources/SourceKitD/SourceKitDRegistry.swift b/Sources/SourceKitD/SourceKitDRegistry.swift index 21deca62c..a7f37f688 100644 --- a/Sources/SourceKitD/SourceKitDRegistry.swift +++ b/Sources/SourceKitD/SourceKitDRegistry.swift @@ -35,10 +35,10 @@ extension NSLock { public actor SourceKitDRegistry { /// Mapping from path to active SourceKitD instance. - var active: [AbsolutePath: SourceKitD] = [:] + private var active: [AbsolutePath: SourceKitD] = [:] /// Instances that have been unregistered, but may be resurrected if accessed before destruction. - var cemetary: [AbsolutePath: WeakSourceKitD] = [:] + private var cemetary: [AbsolutePath: WeakSourceKitD] = [:] /// Initialize an empty registry. public init() {} @@ -79,14 +79,8 @@ public actor SourceKitDRegistry { } return existing } - - /// Remove all SourceKitD instances, including weak ones. - public func clear() { - active.removeAll() - cemetary.removeAll() - } } -struct WeakSourceKitD { +fileprivate struct WeakSourceKitD { weak var value: SourceKitD? } diff --git a/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift b/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift index c4102a5a3..6d2acfdb9 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer+Options.swift @@ -15,12 +15,12 @@ import LanguageServerProtocol import SKCore import SKSupport import SemanticIndex +import SourceKitD import struct TSCBasic.AbsolutePath import struct TSCBasic.RelativePath extension SourceKitLSPServer { - /// Configuration options for the SourceKitServer. public struct Options: Sendable { /// Additional compiler flags (e.g. `-Xswiftc` for SwiftPM projects) and other build-related @@ -52,6 +52,8 @@ extension SourceKitLSPServer { /// Experimental features that are enabled. public var experimentalFeatures: Set + public var sourcekitdTestHooks: SourceKitDTestHooks + public var indexTestHooks: IndexTestHooks public init( @@ -63,6 +65,7 @@ extension SourceKitLSPServer { generatedInterfacesPath: AbsolutePath = defaultDirectoryForGeneratedInterfaces, swiftPublishDiagnosticsDebounceDuration: TimeInterval = 2, /* 2s */ experimentalFeatures: Set = [], + sourcekitdTestHooks: SourceKitDTestHooks = SourceKitDTestHooks(), indexTestHooks: IndexTestHooks = IndexTestHooks() ) { self.buildSetup = buildSetup @@ -73,6 +76,7 @@ extension SourceKitLSPServer { self.generatedInterfacesPath = generatedInterfacesPath self.swiftPublishDiagnosticsDebounceDuration = swiftPublishDiagnosticsDebounceDuration self.experimentalFeatures = experimentalFeatures + self.sourcekitdTestHooks = sourcekitdTestHooks self.indexTestHooks = indexTestHooks } } diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift index e488264e0..16624c83e 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift @@ -194,7 +194,10 @@ public actor SwiftLanguageService: LanguageService, Sendable { guard let sourcekitd = toolchain.sourcekitd else { return nil } self.sourceKitLSPServer = sourceKitLSPServer self.swiftFormat = toolchain.swiftFormat - self.sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd) + self.sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( + dylibPath: sourcekitd, + testHooks: options.sourcekitdTestHooks + ) self.capabilityRegistry = workspace.capabilityRegistry self.semanticIndexManager = workspace.semanticIndexManager self.serverOptions = options diff --git a/Tests/DiagnoseTests/DiagnoseTests.swift b/Tests/DiagnoseTests/DiagnoseTests.swift index c1daf1d9c..1776d086f 100644 --- a/Tests/DiagnoseTests/DiagnoseTests.swift +++ b/Tests/DiagnoseTests/DiagnoseTests.swift @@ -318,7 +318,8 @@ private class InProcessSourceKitRequestExecutor: SourceKitRequestExecutor { logger.info("Sending request: \(requestString)") let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( - dylibPath: try! AbsolutePath(validating: sourcekitd.path) + dylibPath: try! AbsolutePath(validating: sourcekitd.path), + testHooks: SourceKitDTestHooks() ) let response = try await sourcekitd.run(requestYaml: requestString) diff --git a/Tests/SourceKitDTests/SourceKitDRegistryTests.swift b/Tests/SourceKitDTests/SourceKitDRegistryTests.swift index 508cce551..b5dd07026 100644 --- a/Tests/SourceKitDTests/SourceKitDRegistryTests.swift +++ b/Tests/SourceKitDTests/SourceKitDRegistryTests.swift @@ -63,6 +63,7 @@ private nonisolated(unsafe) var nextToken = AtomicUInt32(initialValue: 0) final class FakeSourceKitD: SourceKitD { let token: UInt32 + var testHooks: SourceKitDTestHooks { SourceKitDTestHooks() } var api: sourcekitd_api_functions_t { fatalError() } var keys: sourcekitd_api_keys { fatalError() } var requests: sourcekitd_api_requests { fatalError() } diff --git a/Tests/SourceKitDTests/SourceKitDTests.swift b/Tests/SourceKitDTests/SourceKitDTests.swift index 0d5a3c5bc..919b95c2f 100644 --- a/Tests/SourceKitDTests/SourceKitDTests.swift +++ b/Tests/SourceKitDTests/SourceKitDTests.swift @@ -28,7 +28,10 @@ import class TSCBasic.Process final class SourceKitDTests: XCTestCase { func testMultipleNotificationHandlers() async throws { let sourcekitdPath = await ToolchainRegistry.forTesting.default!.sourcekitd! - let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitdPath) + let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate( + dylibPath: sourcekitdPath, + testHooks: SourceKitDTestHooks() + ) let keys = sourcekitd.keys let path = DocumentURI(for: .swift).pseudoPath diff --git a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift index 8641f3e86..32a58bb9f 100644 --- a/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift +++ b/Tests/SourceKitLSPTests/PullDiagnosticsTests.swift @@ -310,18 +310,28 @@ final class PullDiagnosticsTests: XCTestCase { } func testDontReturnEmptyDiagnosticsIfDiagnosticRequestIsCancelled() async throws { + let diagnosticSourcekitdRequestDidStart = self.expectation(description: "diagnostic sourcekitd request did start") let diagnosticRequestCancelled = self.expectation(description: "diagnostic request cancelled") var serverOptions = SourceKitLSPServer.Options.testDefault - serverOptions.indexTestHooks.preparationTaskDidStart = { _ in - await self.fulfillment(of: [diagnosticRequestCancelled], timeout: defaultTimeout) + serverOptions.sourcekitdTestHooks.sourcekitdRequestDidStart = { request in + guard request.description.contains("source.request.diagnostics") else { + return + } + diagnosticSourcekitdRequestDidStart.fulfill() + self.wait(for: [diagnosticRequestCancelled], timeout: defaultTimeout) + // Poll until the `CancelRequestNotification` has been propagated to the request handling. + for _ in 0..