diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 102776238..e98a314e6 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -648,18 +648,70 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr #if canImport(_StringProcessing) // Filtering + + // Filters currently come in two flavors: those with a prefix and those + // without. Those without a prefix are treated the same as those with an + // "id:" prefix. + enum FilterPrefix: String, CaseIterable { + case id = "id:" + case tag = "tag:" + } var filters = [Configuration.TestFilter]() - func testFilter(forRegularExpressions regexes: [String]?, label: String, membership: Configuration.TestFilter.Membership) throws -> Configuration.TestFilter { - guard let regexes, !regexes.isEmpty else { - // Return early if empty, even though the `reduce` logic below can handle - // this case, in order to avoid the `#available` guard. - return .unfiltered + func testFilters(forOptionArguments optionArguments: [String]?, label: String, membership: Configuration.TestFilter.Membership) throws -> [Configuration.TestFilter] { + var tagPatterns = [String]() + var idPatterns = [String]() + + // If one of the patterns we encounter is surrounded by backticks, we can + // surmize the user intended to express a Swift raw identifier. In that + // case, we should alert the user that it's not going to match what the + // user expects and strip the backticks for them. + func stripBackticksAndAlertIfEncountered(string: inout String) { + let backtickRegex = /^`[^`]*`$/ + if string.contains(backtickRegex) { + let originalString = string + string = String(string.dropFirst().dropLast()) + try? FileHandle.stderr.write("Backticks aren't a valid part of a Swift symbol. Replacing '\(originalString)' with '\(string)'.\n") + } + } + + // Loop through all the option arguments, separating tags from regex filters + for var optionArg in optionArguments ?? [] { + if let prefix = FilterPrefix.allCases.first(where: { optionArg.hasPrefix($0.rawValue) }) { + // We have encountered a prefix, so trim it off and add the supplied + // argument to the appropriate filter list + optionArg.trimPrefix(prefix.rawValue) + stripBackticksAndAlertIfEncountered(string: &optionArg) + switch prefix { + case .id: idPatterns.append(optionArg) + case .tag: tagPatterns.append(optionArg) + } + } else { + // No prefix was detected, so we treat this as a regex matching a test ID. + stripBackticksAndAlertIfEncountered(string: &optionArg) + idPatterns.append(optionArg) + } + } + + // If we didn't find any tags, the tagFilter should be .unfiltered, + // otherwise we construct it with the provided tags + let tagFilter: Configuration.TestFilter = switch (membership, tagPatterns.isEmpty) { + case (_, true): .unfiltered + case (.including, false): try Configuration.TestFilter(includingTagsMatching: tagPatterns) + case (.excluding, false): try Configuration.TestFilter(excludingTagsMatching: tagPatterns) } - return try Configuration.TestFilter(membership: membership, matchingAnyOf: regexes) + guard !idPatterns.isEmpty else { + // Return early with just the tag filter, otherwise we try to match + // against an empty array of regular expressions which is _not_ + // equivalent to `.unfiltered`. + return [tagFilter] + } + + return [try Configuration.TestFilter(membership: membership, matchingAnyOf: idPatterns), tagFilter] } - filters.append(try testFilter(forRegularExpressions: args.filter, label: "--filter", membership: .including)) - filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding)) + + filters += try testFilters(forOptionArguments: args.filter, label: "--filter", membership: .including) + filters += try testFilters(forOptionArguments: args.skip, label: "--skip", membership: .excluding) configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) } if args.includeHiddenTests == true { diff --git a/Sources/Testing/Running/Configuration.TestFilter.swift b/Sources/Testing/Running/Configuration.TestFilter.swift index 06bdb15df..f2c6493f3 100644 --- a/Sources/Testing/Running/Configuration.TestFilter.swift +++ b/Sources/Testing/Running/Configuration.TestFilter.swift @@ -52,6 +52,13 @@ extension Configuration { case tags(_ tags: Set, anyOf: Bool, membership: Membership) #if canImport(_StringProcessing) + /// The test filter contains a pattern to predicate test tags against. + /// + /// - Parameters: + /// - tagPatterns: The patterns to predicate test tags against + /// - membership: How to interpret the result when predicating tests. + case tagPatterns(_ tagPatterns: [String], membership: Membership) + /// The test filter contains a pattern to predicate test IDs against. /// /// - Parameters: @@ -187,6 +194,32 @@ extension Configuration.TestFilter { public init(excludingAllOf tags: some Collection) { self.init(_kind: .tags(Set(tags), anyOf: false, membership: .excluding)) } + + /// Initialize this instance to include tests with tags matching a pattern. + /// + /// - Parameters: + /// - tagPatterns: The patterns, expressed as a `Regex`-compatible regular + /// expressions, to match test tags against. + public init(includingTagsMatching tagPatterns: [String]) throws { + // See the comment above in init(membership:matchingAnyOf:) to understand why we construct regexes here. + for pattern in tagPatterns { + _ = try Regex(pattern) + } + self.init(_kind: .tagPatterns(tagPatterns, membership: .including)) + } + + /// Initialize this instance to exclude tests with tags matching a pattern. + /// + /// - Parameters: + /// - tagPatterns: The patterns, expressed as a `Regex`-compatible regular + /// expressions, to match test tags against. + public init(excludingTagsMatching tagPatterns: [String]) throws { + // See the comment above in init(membership:matchingAnyOf:) to understand why we construct regexes here. + for pattern in tagPatterns { + _ = try Regex(pattern) + } + self.init(_kind: .tagPatterns(tagPatterns, membership: .excluding)) + } } // MARK: - Operations @@ -252,6 +285,20 @@ extension Configuration.TestFilter.Kind { } return .function(predicate, membership: membership) #if canImport(_StringProcessing) + case let .tagPatterns(tagPatterns, membership): + nonisolated(unsafe) let regexes = try tagPatterns.map(Regex.init) + return .function({ item in + let tagNames = item.tags.map { tag in + switch tag.kind { + case let .staticMember(tagName): tagName + } + } + return tagNames.contains(where: { tagName in + regexes.contains(where: { regex in + tagName.contains(regex) + }) + }) + }, membership: membership) case let .patterns(patterns, membership): nonisolated(unsafe) let regexes = try patterns.map(Regex.init) return .function({ item in @@ -505,7 +552,9 @@ extension Configuration.TestFilter.Kind { false #if canImport(_StringProcessing) case .patterns: - false + false + case .tagPatterns: + true #endif case .tags: true diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 2738473d3..68ab8ce80 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -17,6 +17,12 @@ private func configurationForEntryPoint(withArguments args: [String]) throws -> } #if !SWT_NO_CODABLE + +private extension Tag { + @Tag static var testTag: Self + @Tag static var testTagOther: Self + @Tag static var unrelatedTag: Self +} /// Reads event stream output from the provided file matching event stream /// version `V`. private func decodedEventStreamRecords(fromPath filePath: String) throws -> [ABI.Record] { @@ -114,6 +120,18 @@ struct SwiftPMTests { #expect(!planTests.contains(test2)) } + + @Test("--filter argument with tag: prefix") + func filterByTag() async throws { + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag"]) + let test1 = Test(.tags(.testTag), name: "hello") {} + let test2 = Test(name: "goodbye") {} + let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration) + let planTests = plan.steps.map(\.test) + #expect(planTests.contains(test1)) + #expect(!planTests.contains(test2)) + } + @Test("Multiple --filter arguments") func multipleFilter() async throws { let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "hello", "--filter", "sorry"]) @@ -157,6 +175,94 @@ struct SwiftPMTests { #expect(planTests.contains(test2)) } + @Test("--skip argument with tag: prefix") + func skipByTag() async throws { + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--skip", "tag:testTag"]) + let test1 = Test(.tags(.testTag), name: "hello") {} + let test2 = Test(name: "goodbye") {} + let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration) + let planTests = plan.steps.map(\.test) + #expect(!planTests.contains(test1)) + #expect(planTests.contains(test2)) + } + + @Test("--filter argument with tag: prefix supports regex patterns") + func filterByTagRegex() async throws { + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag.*"]) + let test1 = Test(.tags(.testTag), name: "hello") {} + let test2 = Test(.tags(.testTagOther), name: "hi") {} + let test3 = Test(.tags(.unrelatedTag), name: "goodbye") {} + let test4 = Test(name: "untagged") {} + let plan = await Runner.Plan(tests: [test1, test2, test3, test4], configuration: configuration) + let planTests = plan.steps.map(\.test) + #expect(planTests.contains(test1)) + #expect(planTests.contains(test2)) + #expect(!planTests.contains(test3)) + #expect(!planTests.contains(test4)) + } + + @Test("--filter tag: argument strips backticks around tag names") + func filterByTagStripsBackticks() async throws { + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:`testTag`"]) + let test1 = Test(.tags(.testTag), name: "hello") {} + let test2 = Test(name: "goodbye") {} + let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration) + let planTests = plan.steps.map(\.test) + #expect(planTests.contains(test1)) + #expect(!planTests.contains(test2)) + } + + @Test("--filter combining tag: and id: patterns AND's them together") + func mixedPrefixedAndUnprefixedFilters() async throws { + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag", "--filter", "hello"]) + let test1 = Test(.tags(.testTag), name: "hello") {} + let test2 = Test(.tags(.testTag), name: "goodbye") {} + let test3 = Test(name: "hello") {} + let test4 = Test(name: "goodbye") {} + let plan = await Runner.Plan(tests: [test1, test2, test3, test4], configuration: configuration) + let planTests = plan.steps.map(\.test) + #expect(planTests.contains(test1)) + #expect(!planTests.contains(test2)) + #expect(!planTests.contains(test3)) + #expect(!planTests.contains(test4)) + } + + @Test("--filter argument with explicit id: prefix") + func filterByExplicitIdPrefix() async throws { + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "id:hello"]) + let test1 = Test(name: "hello") {} + let test2 = Test(name: "goodbye") {} + let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration) + let planTests = plan.steps.map(\.test) + #expect(planTests.contains(test1)) + #expect(!planTests.contains(test2)) + } + + @Test("--filter tag: combined with --skip id: in the same execution") + func filterByTagAndSkipById() async throws { + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag", "--skip", "id:goodbye"]) + let test1 = Test(.tags(.testTag), name: "hello") {} + let test2 = Test(.tags(.testTag), name: "goodbye") {} + let test3 = Test(.tags(.unrelatedTag), name: "hello") {} + let test4 = Test(name: "untagged") {} + let plan = await Runner.Plan(tests: [test1, test2, test3, test4], configuration: configuration) + let planTests = plan.steps.map(\.test) + #expect(planTests.contains(test1)) + #expect(!planTests.contains(test2)) + #expect(!planTests.contains(test3)) + #expect(!planTests.contains(test4)) + } + + @Test("--filter or --skip tag: argument with bad regex") + func filterByTagWithBadRegex() throws { + #expect(throws: (any Error).self) { + _ = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:("]) + } + #expect(throws: (any Error).self) { + _ = try configurationForEntryPoint(withArguments: ["PATH", "--skip", "tag:)"]) + } + } + @Test("--filter or --skip argument as last argument") func filterOrSkipAsLast() async throws { _ = try configurationForEntryPoint(withArguments: ["PATH", "--filter"])