Add ability to filter/skip by tags using a special tag: prefix

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -648,18 +648,70 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
 
 #if canImport(_StringProcessing)
   // Filtering
+
+  // Filters currently come in two flavors: those with a prefix and those
+  // without. Those without a prefix are treated the same as those with an
+  // "id:" prefix.
+  enum FilterPrefix: String, CaseIterable {
+    case id = "id:"
+    case tag = "tag:"
+  }
   var filters = [Configuration.TestFilter]()
-  func testFilter(forRegularExpressions regexes: [String]?, label: String, membership: Configuration.TestFilter.Membership) throws -> Configuration.TestFilter {
-    guard let regexes, !regexes.isEmpty else {
-      // Return early if empty, even though the `reduce` logic below can handle
-      // this case, in order to avoid the `#available` guard.
-      return .unfiltered
+  func testFilters(forOptionArguments optionArguments: [String]?, label: String, membership: Configuration.TestFilter.Membership) throws -> [Configuration.TestFilter] {
+    var tagPatterns = [String]()
+    var idPatterns = [String]()
+
+    // If one of the patterns we encounter is surrounded by backticks, we can
+    // surmize the user intended to express a Swift raw identifier. In that
+    // case, we should alert the user that it's not going to match what the
+    // user expects and strip the backticks for them.
+    func stripBackticksAndAlertIfEncountered(string: inout String) {
+      let backtickRegex = /^`[^`]*`$/
+      if string.contains(backtickRegex) {
+        let originalString = string
+        string = String(string.dropFirst().dropLast())
+        try? FileHandle.stderr.write("Backticks aren't a valid part of a Swift symbol. Replacing '\(originalString)' with '\(string)'.\n")
+      }
+    }
+
+    // Loop through all the option arguments, separating tags from regex filters
+    for var optionArg in optionArguments ?? [] {
+      if let prefix = FilterPrefix.allCases.first(where: { optionArg.hasPrefix($0.rawValue) }) {
+        // We have encountered a prefix, so trim it off and add the supplied
+        // argument to the appropriate filter list
+        optionArg.trimPrefix(prefix.rawValue)
+        stripBackticksAndAlertIfEncountered(string: &optionArg)
+        switch prefix {
+          case .id: idPatterns.append(optionArg)
+          case .tag: tagPatterns.append(optionArg)
+        }
+      } else {
+        // No prefix was detected, so we treat this as a regex matching a test ID.
+        stripBackticksAndAlertIfEncountered(string: &optionArg)
+        idPatterns.append(optionArg)
+      }
+    }
+
+    // If we didn't find any tags, the tagFilter should be .unfiltered,
+    // otherwise we construct it with the provided tags
+    let tagFilter: Configuration.TestFilter = switch (membership, tagPatterns.isEmpty) {
+      case (_, true): .unfiltered
+      case (.including, false): try Configuration.TestFilter(includingTagsMatching: tagPatterns)
+      case (.excluding, false): try Configuration.TestFilter(excludingTagsMatching: tagPatterns)
     }
 
-    return try Configuration.TestFilter(membership: membership, matchingAnyOf: regexes)
+    guard !idPatterns.isEmpty else {
+      // Return early with just the tag filter, otherwise we try to match
+      // against an empty array of regular expressions which is _not_
+      // equivalent to `.unfiltered`.
+      return [tagFilter]
+    }
+
+    return [try Configuration.TestFilter(membership: membership, matchingAnyOf: idPatterns), tagFilter]
   }
-  filters.append(try testFilter(forRegularExpressions: args.filter, label: "--filter", membership: .including))
-  filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding))
+
+  filters += try testFilters(forOptionArguments: args.filter, label: "--filter", membership: .including)
+  filters += try testFilters(forOptionArguments: args.skip, label: "--skip", membership: .excluding)
 
   configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) }
   if args.includeHiddenTests == true {

Sources/Testing/Running/Configuration.TestFilter.swift

@@ -52,6 +52,13 @@ extension Configuration {
       case tags(_ tags: Set<Tag>, anyOf: Bool, membership: Membership)
 
 #if canImport(_StringProcessing)
+      /// The test filter contains a pattern to predicate test tags against.
+      ///
+      /// - Parameters:
+      ///   - tagPatterns: The patterns to predicate test tags against
+      ///   - membership: How to interpret the result when predicating tests.
+      case tagPatterns(_ tagPatterns: [String], membership: Membership)
+
       /// The test filter contains a pattern to predicate test IDs against.
       ///
       /// - Parameters:
@@ -187,6 +194,32 @@ extension Configuration.TestFilter {
   public init(excludingAllOf tags: some Collection<Tag>) {
     self.init(_kind: .tags(Set(tags), anyOf: false, membership: .excluding))
   }
+
+  /// Initialize this instance to include tests with tags matching a pattern.
+  ///
+  /// - Parameters:
+  ///   - tagPatterns: The patterns, expressed as a `Regex`-compatible regular
+  ///     expressions, to match test tags against.
+  public init(includingTagsMatching tagPatterns: [String]) throws {
+    // See the comment above in init(membership:matchingAnyOf:) to understand why we construct regexes here.
+    for pattern in tagPatterns {
+      _ = try Regex(pattern)
+    }
+    self.init(_kind: .tagPatterns(tagPatterns, membership: .including))
+  }
+
+  /// Initialize this instance to exclude tests with tags matching a pattern.
+  ///
+  /// - Parameters:
+  ///   - tagPatterns: The patterns, expressed as a `Regex`-compatible regular
+  ///     expressions, to match test tags against.
+  public init(excludingTagsMatching tagPatterns: [String]) throws {
+    // See the comment above in init(membership:matchingAnyOf:) to understand why we construct regexes here.
+    for pattern in tagPatterns {
+      _ = try Regex(pattern)
+    }
+    self.init(_kind: .tagPatterns(tagPatterns, membership: .excluding))
+  }
 }
 
 // MARK: - Operations
@@ -252,6 +285,20 @@ extension Configuration.TestFilter.Kind {
       }
       return .function(predicate, membership: membership)
 #if canImport(_StringProcessing)
+    case let .tagPatterns(tagPatterns, membership):
+      nonisolated(unsafe) let regexes = try tagPatterns.map(Regex.init)
+      return .function({ item in
+        let tagNames = item.tags.map { tag in
+          switch tag.kind {
+          case let .staticMember(tagName): tagName
+          }
+        }
+        return tagNames.contains(where: { tagName in
+          regexes.contains(where: { regex in
+            tagName.contains(regex)
+          })
+        })
+      }, membership: membership)
     case let .patterns(patterns, membership):
       nonisolated(unsafe) let regexes = try patterns.map(Regex.init)
       return .function({ item in
@@ -505,7 +552,9 @@ extension Configuration.TestFilter.Kind {
       false
 #if canImport(_StringProcessing)
     case .patterns:
-      false
+        false
+    case .tagPatterns:
+        true
 #endif
     case .tags:
       true

Tests/TestingTests/SwiftPMTests.swift

@@ -17,6 +17,12 @@ private func configurationForEntryPoint(withArguments args: [String]) throws ->
 }
 
 #if !SWT_NO_CODABLE
+
+private extension Tag {
+  @Tag static var testTag: Self
+  @Tag static var testTagOther: Self
+  @Tag static var unrelatedTag: Self
+}
 /// Reads event stream output from the provided file matching event stream
 /// version `V`.
 private func decodedEventStreamRecords<V: ABI.Version>(fromPath filePath: String) throws -> [ABI.Record<V>] {
@@ -114,6 +120,18 @@ struct SwiftPMTests {
     #expect(!planTests.contains(test2))
   }
 
+
+  @Test("--filter argument with tag: prefix")
+  func filterByTag() async throws {
+    let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag"])
+    let test1 = Test(.tags(.testTag), name: "hello") {}
+    let test2 = Test(name: "goodbye") {}
+    let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration)
+    let planTests = plan.steps.map(\.test)
+    #expect(planTests.contains(test1))
+    #expect(!planTests.contains(test2))
+  }
+
   @Test("Multiple --filter arguments")
   func multipleFilter() async throws {
     let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "hello", "--filter", "sorry"])
@@ -157,6 +175,94 @@ struct SwiftPMTests {
     #expect(planTests.contains(test2))
   }
 
+  @Test("--skip argument with tag: prefix")
+  func skipByTag() async throws {
+    let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--skip", "tag:testTag"])
+    let test1 = Test(.tags(.testTag), name: "hello") {}
+    let test2 = Test(name: "goodbye") {}
+    let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration)
+    let planTests = plan.steps.map(\.test)
+    #expect(!planTests.contains(test1))
+    #expect(planTests.contains(test2))
+  }
+
+  @Test("--filter argument with tag: prefix supports regex patterns")
+  func filterByTagRegex() async throws {
+    let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag.*"])
+    let test1 = Test(.tags(.testTag), name: "hello") {}
+    let test2 = Test(.tags(.testTagOther), name: "hi") {}
+    let test3 = Test(.tags(.unrelatedTag), name: "goodbye") {}
+    let test4 = Test(name: "untagged") {}
+    let plan = await Runner.Plan(tests: [test1, test2, test3, test4], configuration: configuration)
+    let planTests = plan.steps.map(\.test)
+    #expect(planTests.contains(test1))
+    #expect(planTests.contains(test2))
+    #expect(!planTests.contains(test3))
+    #expect(!planTests.contains(test4))
+  }
+
+  @Test("--filter tag: argument strips backticks around tag names")
+  func filterByTagStripsBackticks() async throws {
+    let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:`testTag`"])
+    let test1 = Test(.tags(.testTag), name: "hello") {}
+    let test2 = Test(name: "goodbye") {}
+    let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration)
+    let planTests = plan.steps.map(\.test)
+    #expect(planTests.contains(test1))
+    #expect(!planTests.contains(test2))
+  }
+
+  @Test("--filter combining tag: and id: patterns AND's them together")
+  func mixedPrefixedAndUnprefixedFilters() async throws {
+    let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag", "--filter", "hello"])
+    let test1 = Test(.tags(.testTag), name: "hello") {}
+    let test2 = Test(.tags(.testTag), name: "goodbye") {}
+    let test3 = Test(name: "hello") {}
+    let test4 = Test(name: "goodbye") {}
+    let plan = await Runner.Plan(tests: [test1, test2, test3, test4], configuration: configuration)
+    let planTests = plan.steps.map(\.test)
+    #expect(planTests.contains(test1))
+    #expect(!planTests.contains(test2))
+    #expect(!planTests.contains(test3))
+    #expect(!planTests.contains(test4))
+  }
+
+  @Test("--filter argument with explicit id: prefix")
+  func filterByExplicitIdPrefix() async throws {
+    let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "id:hello"])
+    let test1 = Test(name: "hello") {}
+    let test2 = Test(name: "goodbye") {}
+    let plan = await Runner.Plan(tests: [test1, test2], configuration: configuration)
+    let planTests = plan.steps.map(\.test)
+    #expect(planTests.contains(test1))
+    #expect(!planTests.contains(test2))
+  }
+
+  @Test("--filter tag: combined with --skip id: in the same execution")
+  func filterByTagAndSkipById() async throws {
+    let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:testTag", "--skip", "id:goodbye"])
+    let test1 = Test(.tags(.testTag), name: "hello") {}
+    let test2 = Test(.tags(.testTag), name: "goodbye") {}
+    let test3 = Test(.tags(.unrelatedTag), name: "hello") {}
+    let test4 = Test(name: "untagged") {}
+    let plan = await Runner.Plan(tests: [test1, test2, test3, test4], configuration: configuration)
+    let planTests = plan.steps.map(\.test)
+    #expect(planTests.contains(test1))
+    #expect(!planTests.contains(test2))
+    #expect(!planTests.contains(test3))
+    #expect(!planTests.contains(test4))
+  }
+
+  @Test("--filter or --skip tag: argument with bad regex")
+  func filterByTagWithBadRegex() throws {
+    #expect(throws: (any Error).self) {
+      _ = try configurationForEntryPoint(withArguments: ["PATH", "--filter", "tag:("])
+    }
+    #expect(throws: (any Error).self) {
+      _ = try configurationForEntryPoint(withArguments: ["PATH", "--skip", "tag:)"])
+    }
+  }
+
   @Test("--filter or --skip argument as last argument")
   func filterOrSkipAsLast() async throws {
     _ = try configurationForEntryPoint(withArguments: ["PATH", "--filter"])