feat: automatic obtaining of tests packages via --project-name in command line

Jenkinsfile

@@ -143,11 +143,11 @@ Build Url: ${env.BUILD_URL}
 
 
 
-def defineIsoTestStage(stageName, projectName, testPackages){
+def defineIsoTestStage(stageName, projectName, testPackages=""){
     stage("Tests-ISO-" + stageName) {
         def testGroup = "Tests"
         timeout(2) {
-            dir(env.STAGE_NAME) {
+            dir(env.STAGE_NAME){
                 def PHARO_MAJOR = shellOutput('git describe --tags --first-parent | cut -d\'-\' -f 1 | cut -c 2- | cut -d\'.\' -f 1-1')
                 def PHARO_MINOR = shellOutput('git describe --tags --first-parent | cut -d\'-\' -f 1 | cut -c 2- | cut -d\'.\' -f 2-2')
                 def PHARO_SHORT = PHARO_MAJOR + PHARO_MINOR
@@ -157,7 +157,13 @@ def defineIsoTestStage(stageName, projectName, testPackages){
                 shell "bash -c './bootstrap/scripts/getPharoVM.sh ${PHARO_SHORT}'"
                 shell "bash -c './pharo metacello.image metacello install --save --strict --signalErrorOnWarning \"filetree://../src\" SUnit --groups Core'"
                 shell "bash -c './pharo metacello.image metacello install --save --strict --signalErrorOnWarning \"filetree://../src\" " + projectName + " --groups " + testGroup + "'"
-                shell "bash -c './pharo metacello.image test --junit-xml-output --stage-name ${env.STAGE_NAME}  " + testPackages + " '"
+                /* 
+                Some Baselines do specify tests in the Tests group that do not run on isolation.
+                For that scenario, users can define an explicit list of packages as `testPackages`.
+                In that case, take the packages specified by the user instead of the project packages.
+                */
+                def testPackageArguments = testPackages == "" ? "--project-name ${projectName}" : testPackages
+                shell "bash -c './pharo metacello.image test --junit-xml-output --stage-name ${env.STAGE_NAME} ${testPackageArguments}'"
                 junit allowEmptyResults: false, testResults: "${env.STAGE_NAME}*.xml"
             }
         }
@@ -183,17 +189,17 @@ def bootstrapImage(){
         }
 
         def isoTesters = [:]
-        isoTesters['SUnit'] = { defineIsoTestStage("SUnit", "SUnit", "\'SUnit-Tests\'  \'SUnit-Visitor-Tests\'  \'SUnit-MockObjects-Tests\'") }
+        isoTesters['SUnit'] = { defineIsoTestStage("SUnit", "SUnit") }
         isoTesters['Kernel'] = { defineIsoTestStage("Kernel", "Kernel", "\'Kernel-Tests\'  \'Kernel-CodeModel-Tests\'") }
         isoTesters['Compiler'] = { defineIsoTestStage("Compiler", "Compiler", "\'OpalCompiler-Tests\'  \'DebugInfo-Tests\' \'Kernel-Extended-Tests\' \'Kernel-Tests-WithCompiler\'") }      
-        isoTesters['Files'] = { defineIsoTestStage("Files", "Files", "\'Files-Tests\'") }
-        isoTesters['Zinc-Character-Encoding'] = { defineIsoTestStage("Zinc-Character-Encoding", "ZincCharacterEncoding", "\'Zinc-Character-Encoding-Tests\'") }
-        isoTesters['System-SessionManager'] = { defineIsoTestStage("System-SessionManager", "SystemSessionManager", "\'System-SessionManager-Tests\'") }
-        isoTesters['System-Platforms'] = { defineIsoTestStage("System-Platforms", "SystemPlatforms", "\'System-Platforms-Tests\'") }  
-        isoTesters['Announcements-Core'] = { defineIsoTestStage("Announcements-Core", "Announcements", "\'Announcements-Core-Tests\'") } 
-        isoTesters['Shift-ClassBuilder'] = { defineIsoTestStage("Shift-ClassBuilder", "Shift", "\'Shift-ClassBuilder-Tests\'") }        
-        isoTesters['System-CommandLineHandler'] = { defineIsoTestStage("System-CommandLineHandler", "SystemCommandLineHandler", "\'System-CommandLineHandler-Tests\'") }        
-        isoTesters['FileSystem'] = { defineIsoTestStage("FileSystem", "FileSystem", "\'FileSystem-Core-Tests\' \'FileSystem-Disk-Tests\' \'FileSystem-Tests-Attributes\'") }        
+        isoTesters['Files'] = { defineIsoTestStage("Files", "Files") }
+        isoTesters['Zinc-Character-Encoding'] = { defineIsoTestStage("Zinc-Character-Encoding", "ZincCharacterEncoding") }
+        isoTesters['System-SessionManager'] = { defineIsoTestStage("System-SessionManager", "SystemSessionManager") }
+        isoTesters['System-Platforms'] = { defineIsoTestStage("System-Platforms", "SystemPlatforms") }  
+        isoTesters['Announcements-Core'] = { defineIsoTestStage("Announcements-Core", "Announcements") } 
+        isoTesters['Shift-ClassBuilder'] = { defineIsoTestStage("Shift-ClassBuilder", "Shift") }        
+        isoTesters['System-CommandLineHandler'] = { defineIsoTestStage("System-CommandLineHandler", "SystemCommandLineHandler") }        
+        isoTesters['FileSystem'] = { defineIsoTestStage("FileSystem", "FileSystem") }        
         parallel isoTesters
 
         stage ("Full Image") {

src/SUnit-Basic-CLI/ClapTestRunner.class.st

@@ -4,6 +4,9 @@ I run unit tests found in specified packages, and display their results to stand
 Class {
 	#name : 'ClapTestRunner',
 	#superclass : 'ClapPharoApplication',
+	#instVars : [
+		'environment'
+	],
 	#category : 'SUnit-Basic-CLI',
 	#package : 'SUnit-Basic-CLI'
 }
@@ -18,8 +21,8 @@ ClapTestRunner class >> commandSpecification [
 		description: 'A Test runner to run test suites from a shell';
 		addFlag: #'junit-xml-output' description: 'Output results in JUnit-compatible XML format';
 		addFlagWithPositional: #'shuffle-seed' description: 'an integer specifying the seed used to shuffle the tests';
-		addFlagWithPositional: #'stage-name' description: 'adds a prefix to the generated xml, this is useful
-when running in the CI infrastructure';
+		addFlagWithPositional: #'stage-name' description: 'adds a prefix to the generated xml, this is useful when running in the CI infrastructure';
+		addFlagWithPositional: #'project-name' description: 'the name of the metacello project, what follows the BaselineOf prefix, used to obtain the tests from the project Tests group';
 		addFlag: #'no-xterm' description: 'use this option when no xterm available';
 		addFlag: #'fail-on-error' description: 'if there is a test error, it will exit with error code 1';
 		addFlag: #'fail-on-failure' description: 'if there is a test error or failure, it will exit with error code 1';
@@ -65,6 +68,13 @@ ClapTestRunner >> execute [
 	self runPackages
 ]
 
+{ #category : 'accessing' }
+ClapTestRunner >> hasProjectName [
+	"Return True if the project-name flag is set"
+
+	^ self hasFlag: #'project-name'
+]
+
 { #category : 'private - printing' }
 ClapTestRunner >> informResults: results [
 
@@ -78,9 +88,35 @@ ClapTestRunner >> informResults: results [
 				flush ]
 ]
 
+{ #category : 'initialization' }
+ClapTestRunner >> initialize [
+
+	super initialize.
+
+	"Set the environment to look for baselines"
+	environment := self class environment
+]
+
 { #category : 'accessing' }
 ClapTestRunner >> packages [
-	^ self positional: #PACKAGE
+	"Collects all test package names by merging explicit positional PACKAGE arguments with implicit test packages retrieved from the project's baseline via the --project-name flag"
+	| explicitNames projectNames |
+
+	explicitNames := (self positional: #PACKAGE ifAbsent: [ #() ]) 
+		collect: [ :each | each value ].
+
+	projectNames := self hasProjectName 
+		ifTrue: [ self testPackagesForProject: self projectName ]
+		ifFalse: [ #() ].
+
+	^ explicitNames, projectNames 
+]
+
+{ #category : 'accessing' }
+ClapTestRunner >> projectName [
+	"Return project name in the form of a String (ex: 'FileSystem'). Fails with an exception if the flag is missing."
+
+	^ (self positional: #'project-name') value
 ]
 
 { #category : 'execution' }
@@ -143,6 +179,28 @@ ClapTestRunner >> testPackages [
 	^ packages 
 ]
 
+{ #category : 'accessing' }
+ClapTestRunner >> testPackagesForLoadableName: loadableName inMetacelloVersion: version [
+	"Get the test packages in the project version, resolving nested groups recursively.
+	loadableName is a group name or a package name"
+
+	| spec |
+	spec := version packageNamed: loadableName ifAbsent: [ ^ #(  ) ].
+	^ spec isGroupSpec
+		  ifTrue: [ spec includes flatCollect: [ :each | self testPackagesForLoadableName: each inMetacelloVersion: version ] ]
+		  ifFalse: [ { spec name } ]
+]
+
+{ #category : 'accessing' }
+ClapTestRunner >> testPackagesForProject: aProjectName [
+	"Get the test packages associated with a projectName by its baseline, resolving nested groups recursively"
+
+	| version |
+	version := (environment at: ('BaselineOf' , aProjectName) asSymbol ifAbsent: [ ^ #(  ) ]) project version.
+
+	^ (self testPackagesForLoadableName: 'Tests' inMetacelloVersion: version) asSet asArray "delete duplicates"
+]
+
 { #category : 'accessing' }
 ClapTestRunner >> testRunner [
 

src/SUnit-UI-Tests/BaselineOfClapMockGeneralProject.class.st

@@ -0,0 +1,21 @@
+Class {
+	#name : 'BaselineOfClapMockGeneralProject',
+	#superclass : 'BaselineOf',
+	#category : 'SUnit-UI-Tests-Mocks',
+	#package : 'SUnit-UI-Tests',
+	#tag : 'Mocks'
+}
+
+{ #category : 'baselines' }
+BaselineOfClapMockGeneralProject >> baseline: spec [
+    <baseline>
+    spec for: #common do: [
+        spec package: 'FakePackage1'.
+        spec package: 'FakePackage2'.
+        spec package: 'FakePackage-Tests' with: [ spec requires: #('FakePackage1') ].
+
+        spec group: 'Core' with: #('FakePackage2').
+        spec group: 'aGroup' with: #('FakePackage1' 'Core').
+        spec group: 'Tests' with: #('FakePackage-Tests' 'aGroup')
+    ]
+]

src/SUnit-UI-Tests/BaselineOfClapMockNoTestGroup.class.st

@@ -0,0 +1,19 @@
+Class {
+	#name : 'BaselineOfClapMockNoTestGroup',
+	#superclass : 'BaselineOf',
+	#category : 'SUnit-UI-Tests-Mocks',
+	#package : 'SUnit-UI-Tests',
+	#tag : 'Mocks'
+}
+
+{ #category : 'baselines' }
+BaselineOfClapMockNoTestGroup >> baseline: spec [
+    <baseline>
+    spec for: #common do: [
+        spec package: 'FakePackage1'.
+        spec package: 'FakePackage2'.
+        spec package: 'FakePackage-Tests' with: [ spec requires: #('FakePackage1') ].
+
+        spec group: 'Core' with: #('FakePackage2').
+    ]
+]

src/SUnit-UI-Tests/ClapTestRunnerTest.class.st

@@ -86,6 +86,44 @@ ClapTestRunnerTest >> testDefaultTestRunnerIsVTermTestRunner [
 	self assert: command testRunner equals: VTermTestRunner
 ]
 
+{ #category : 'tests' }
+ClapTestRunnerTest >> testResolvePackagesForGeneralProject [
+	"Tests that a baseline mixing direct packages and nested groups in its Tests group is resolved correctly"
+	| runner packages |
+    runner := ClapTestRunner new.
+    packages := runner testPackagesForProject: 'ClapMockGeneralProject'.
+
+    self assert: (packages includes: 'FakePackage1').
+    self assert: (packages includes: 'FakePackage2').
+	 self assert: (packages includes: 'FakePackage-Tests').
+	 self assert: (packages includes: 'Core') not.
+
+    "Check there is no list in the list "
+    self assert: (packages allSatisfy: [ :each | each isString ]).
+]
+
+{ #category : 'tests' }
+ClapTestRunnerTest >> testResolvePackagesForProjectWithoutTestsGroup [
+	"Tests a project that has a baseline but no 'Tests' group"
+	| runner packages |
+	runner := ClapTestRunner new.
+
+	packages := runner testPackagesForProject: 'ClapMockNoTestGroup'. 
+
+	self assert: packages isEmpty.
+]
+
+{ #category : 'tests' }
+ClapTestRunnerTest >> testResolvePackagesForUnknownProject [
+	"Tests that an unknown project returns an empty collection without crashing"
+	| runner packages |
+	runner := ClapTestRunner new.
+
+	packages := runner testPackagesForProject: 'ThisProjectDoesNotExist404'.
+
+	self assert: packages isEmpty.
+]
+
 { #category : 'tests' }
 ClapTestRunnerTest >> testShouldFailOnErrorIsFalseWhenFailOnErrorFlagAbsent [
 

tests/pharo-cli-test-runner.bats

@@ -2,6 +2,9 @@
 
 load test_helper.bash
 
+BASE_TEST_COUNT=19
+TOTAL_TEST_COUNT=20
+
 setup() {
   setup-workdir
 }
@@ -12,25 +15,25 @@ teardown() {
 
 @test "test --help prints help" {
   run_pharo test --help
-  assert_output --partial "Usage: test [--help] [--junit-xml-output] [--shuffle-seed <shuffle-seed-value>] [--stage-name <stage-name-value>] [--no-xterm] [--fail-on-error] [--fail-on-failure] [--save] [--rename] [<PACKAGE>]"
+  assert_output --partial "Usage: test [--help] [--junit-xml-output] [--shuffle-seed <shuffle-seed-value>] [--stage-name <stage-name-value>] [--project-name <project-name-value>] [--no-xterm] [--fail-on-error] [--fail-on-failure] [--save] [--rename] [<PACKAGE>]"
   assert_success
 }
 
 @test "test can run tests on a package" {
   run_pharo test SUnit-UI-Tests
   assert_success
-  assert_line "Finished running 16 Tests"
+  assert_line "Finished running ${BASE_TEST_COUNT} Tests"
   assert_line --partial "Finished to run tests of SUnit-UI-Tests in"
-  assert_line "16 run, 16 passes, 0 failures, 0 errors."
+  assert_line "${BASE_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 0 failures, 0 errors."
 }
 
 @test "test can run tests on a regex" {
   run_pharo test SUnit-UI.*
   assert_success
   assert_line "Running tests in 2 Packages"
-  assert_line "Finished running 16 Tests"
+  assert_line "Finished running ${BASE_TEST_COUNT} Tests"
   assert_line --partial "Finished to run tests of SUnit-UI-Tests in"
-  assert_line "16 run, 16 passes, 0 failures, 0 errors."
+  assert_line "${BASE_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 0 failures, 0 errors."
 }
 
 @test "test can run tests with junit XML output" {
@@ -47,7 +50,7 @@ teardown() {
 
   run_pharo test --fail-on-failure SUnit-UI-Tests
   assert_failure
-  assert_line --partial "17 run, 16 passes, 1 failures, 0 errors."
+  assert_line --partial "${TOTAL_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 1 failures, 0 errors."
 }
 
 @test "test runner does not fail if a test fails with flag --fail-on-error" {
@@ -57,7 +60,7 @@ teardown() {
 
   run_pharo test --fail-on-error SUnit-UI-Tests
   assert_success
-  assert_line --partial "17 run, 16 passes, 1 failures, 0 errors."
+  assert_line --partial "${TOTAL_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 1 failures, 0 errors."
 }
 
 @test "test runner fails if a test fails with error with flag --fail-on-error" {
@@ -67,5 +70,5 @@ teardown() {
 
   run_pharo test --fail-on-error SUnit-UI-Tests
   assert_failure
-  assert_line --partial "17 run, 16 passes, 0 failures, 1 errors."
-}
+  assert_line --partial "${TOTAL_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 0 failures, 1 errors."
+}