diff --git a/Jenkinsfile b/Jenkinsfile index 912bb624556..2ad1f47a08b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -143,11 +143,11 @@ Build Url: ${env.BUILD_URL} -def defineIsoTestStage(stageName, projectName, testPackages){ +def defineIsoTestStage(stageName, projectName, testPackages=""){ stage("Tests-ISO-" + stageName) { def testGroup = "Tests" timeout(2) { - dir(env.STAGE_NAME) { + dir(env.STAGE_NAME){ def PHARO_MAJOR = shellOutput('git describe --tags --first-parent | cut -d\'-\' -f 1 | cut -c 2- | cut -d\'.\' -f 1-1') def PHARO_MINOR = shellOutput('git describe --tags --first-parent | cut -d\'-\' -f 1 | cut -c 2- | cut -d\'.\' -f 2-2') def PHARO_SHORT = PHARO_MAJOR + PHARO_MINOR @@ -157,7 +157,13 @@ def defineIsoTestStage(stageName, projectName, testPackages){ shell "bash -c './bootstrap/scripts/getPharoVM.sh ${PHARO_SHORT}'" shell "bash -c './pharo metacello.image metacello install --save --strict --signalErrorOnWarning \"filetree://../src\" SUnit --groups Core'" shell "bash -c './pharo metacello.image metacello install --save --strict --signalErrorOnWarning \"filetree://../src\" " + projectName + " --groups " + testGroup + "'" - shell "bash -c './pharo metacello.image test --junit-xml-output --stage-name ${env.STAGE_NAME} " + testPackages + " '" + /* + Some Baselines do specify tests in the Tests group that do not run on isolation. + For that scenario, users can define an explicit list of packages as `testPackages`. + In that case, take the packages specified by the user instead of the project packages. + */ + def testPackageArguments = testPackages == "" ? "--project-name ${projectName}" : testPackages + shell "bash -c './pharo metacello.image test --junit-xml-output --stage-name ${env.STAGE_NAME} ${testPackageArguments}'" junit allowEmptyResults: false, testResults: "${env.STAGE_NAME}*.xml" } } @@ -183,17 +189,17 @@ def bootstrapImage(){ } def isoTesters = [:] - isoTesters['SUnit'] = { defineIsoTestStage("SUnit", "SUnit", "\'SUnit-Tests\' \'SUnit-Visitor-Tests\' \'SUnit-MockObjects-Tests\'") } + isoTesters['SUnit'] = { defineIsoTestStage("SUnit", "SUnit") } isoTesters['Kernel'] = { defineIsoTestStage("Kernel", "Kernel", "\'Kernel-Tests\' \'Kernel-CodeModel-Tests\'") } isoTesters['Compiler'] = { defineIsoTestStage("Compiler", "Compiler", "\'OpalCompiler-Tests\' \'DebugInfo-Tests\' \'Kernel-Extended-Tests\' \'Kernel-Tests-WithCompiler\'") } - isoTesters['Files'] = { defineIsoTestStage("Files", "Files", "\'Files-Tests\'") } - isoTesters['Zinc-Character-Encoding'] = { defineIsoTestStage("Zinc-Character-Encoding", "ZincCharacterEncoding", "\'Zinc-Character-Encoding-Tests\'") } - isoTesters['System-SessionManager'] = { defineIsoTestStage("System-SessionManager", "SystemSessionManager", "\'System-SessionManager-Tests\'") } - isoTesters['System-Platforms'] = { defineIsoTestStage("System-Platforms", "SystemPlatforms", "\'System-Platforms-Tests\'") } - isoTesters['Announcements-Core'] = { defineIsoTestStage("Announcements-Core", "Announcements", "\'Announcements-Core-Tests\'") } - isoTesters['Shift-ClassBuilder'] = { defineIsoTestStage("Shift-ClassBuilder", "Shift", "\'Shift-ClassBuilder-Tests\'") } - isoTesters['System-CommandLineHandler'] = { defineIsoTestStage("System-CommandLineHandler", "SystemCommandLineHandler", "\'System-CommandLineHandler-Tests\'") } - isoTesters['FileSystem'] = { defineIsoTestStage("FileSystem", "FileSystem", "\'FileSystem-Core-Tests\' \'FileSystem-Disk-Tests\' \'FileSystem-Tests-Attributes\'") } + isoTesters['Files'] = { defineIsoTestStage("Files", "Files") } + isoTesters['Zinc-Character-Encoding'] = { defineIsoTestStage("Zinc-Character-Encoding", "ZincCharacterEncoding") } + isoTesters['System-SessionManager'] = { defineIsoTestStage("System-SessionManager", "SystemSessionManager") } + isoTesters['System-Platforms'] = { defineIsoTestStage("System-Platforms", "SystemPlatforms") } + isoTesters['Announcements-Core'] = { defineIsoTestStage("Announcements-Core", "Announcements") } + isoTesters['Shift-ClassBuilder'] = { defineIsoTestStage("Shift-ClassBuilder", "Shift") } + isoTesters['System-CommandLineHandler'] = { defineIsoTestStage("System-CommandLineHandler", "SystemCommandLineHandler") } + isoTesters['FileSystem'] = { defineIsoTestStage("FileSystem", "FileSystem") } parallel isoTesters stage ("Full Image") { diff --git a/src/SUnit-Basic-CLI/ClapTestRunner.class.st b/src/SUnit-Basic-CLI/ClapTestRunner.class.st index 2ee408632c7..775e7a920df 100644 --- a/src/SUnit-Basic-CLI/ClapTestRunner.class.st +++ b/src/SUnit-Basic-CLI/ClapTestRunner.class.st @@ -4,6 +4,9 @@ I run unit tests found in specified packages, and display their results to stand Class { #name : 'ClapTestRunner', #superclass : 'ClapPharoApplication', + #instVars : [ + 'environment' + ], #category : 'SUnit-Basic-CLI', #package : 'SUnit-Basic-CLI' } @@ -18,8 +21,8 @@ ClapTestRunner class >> commandSpecification [ description: 'A Test runner to run test suites from a shell'; addFlag: #'junit-xml-output' description: 'Output results in JUnit-compatible XML format'; addFlagWithPositional: #'shuffle-seed' description: 'an integer specifying the seed used to shuffle the tests'; - addFlagWithPositional: #'stage-name' description: 'adds a prefix to the generated xml, this is useful -when running in the CI infrastructure'; + addFlagWithPositional: #'stage-name' description: 'adds a prefix to the generated xml, this is useful when running in the CI infrastructure'; + addFlagWithPositional: #'project-name' description: 'the name of the metacello project, what follows the BaselineOf prefix, used to obtain the tests from the project Tests group'; addFlag: #'no-xterm' description: 'use this option when no xterm available'; addFlag: #'fail-on-error' description: 'if there is a test error, it will exit with error code 1'; addFlag: #'fail-on-failure' description: 'if there is a test error or failure, it will exit with error code 1'; @@ -65,6 +68,13 @@ ClapTestRunner >> execute [ self runPackages ] +{ #category : 'accessing' } +ClapTestRunner >> hasProjectName [ + "Return True if the project-name flag is set" + + ^ self hasFlag: #'project-name' +] + { #category : 'private - printing' } ClapTestRunner >> informResults: results [ @@ -78,9 +88,35 @@ ClapTestRunner >> informResults: results [ flush ] ] +{ #category : 'initialization' } +ClapTestRunner >> initialize [ + + super initialize. + + "Set the environment to look for baselines" + environment := self class environment +] + { #category : 'accessing' } ClapTestRunner >> packages [ - ^ self positional: #PACKAGE + "Collects all test package names by merging explicit positional PACKAGE arguments with implicit test packages retrieved from the project's baseline via the --project-name flag" + | explicitNames projectNames | + + explicitNames := (self positional: #PACKAGE ifAbsent: [ #() ]) + collect: [ :each | each value ]. + + projectNames := self hasProjectName + ifTrue: [ self testPackagesForProject: self projectName ] + ifFalse: [ #() ]. + + ^ explicitNames, projectNames +] + +{ #category : 'accessing' } +ClapTestRunner >> projectName [ + "Return project name in the form of a String (ex: 'FileSystem'). Fails with an exception if the flag is missing." + + ^ (self positional: #'project-name') value ] { #category : 'execution' } @@ -143,6 +179,28 @@ ClapTestRunner >> testPackages [ ^ packages ] +{ #category : 'accessing' } +ClapTestRunner >> testPackagesForLoadableName: loadableName inMetacelloVersion: version [ + "Get the test packages in the project version, resolving nested groups recursively. + loadableName is a group name or a package name" + + | spec | + spec := version packageNamed: loadableName ifAbsent: [ ^ #( ) ]. + ^ spec isGroupSpec + ifTrue: [ spec includes flatCollect: [ :each | self testPackagesForLoadableName: each inMetacelloVersion: version ] ] + ifFalse: [ { spec name } ] +] + +{ #category : 'accessing' } +ClapTestRunner >> testPackagesForProject: aProjectName [ + "Get the test packages associated with a projectName by its baseline, resolving nested groups recursively" + + | version | + version := (environment at: ('BaselineOf' , aProjectName) asSymbol ifAbsent: [ ^ #( ) ]) project version. + + ^ (self testPackagesForLoadableName: 'Tests' inMetacelloVersion: version) asSet asArray "delete duplicates" +] + { #category : 'accessing' } ClapTestRunner >> testRunner [ diff --git a/src/SUnit-UI-Tests/BaselineOfClapMockGeneralProject.class.st b/src/SUnit-UI-Tests/BaselineOfClapMockGeneralProject.class.st new file mode 100644 index 00000000000..478f5736e0a --- /dev/null +++ b/src/SUnit-UI-Tests/BaselineOfClapMockGeneralProject.class.st @@ -0,0 +1,21 @@ +Class { + #name : 'BaselineOfClapMockGeneralProject', + #superclass : 'BaselineOf', + #category : 'SUnit-UI-Tests-Mocks', + #package : 'SUnit-UI-Tests', + #tag : 'Mocks' +} + +{ #category : 'baselines' } +BaselineOfClapMockGeneralProject >> baseline: spec [ + + spec for: #common do: [ + spec package: 'FakePackage1'. + spec package: 'FakePackage2'. + spec package: 'FakePackage-Tests' with: [ spec requires: #('FakePackage1') ]. + + spec group: 'Core' with: #('FakePackage2'). + spec group: 'aGroup' with: #('FakePackage1' 'Core'). + spec group: 'Tests' with: #('FakePackage-Tests' 'aGroup') + ] +] diff --git a/src/SUnit-UI-Tests/BaselineOfClapMockNoTestGroup.class.st b/src/SUnit-UI-Tests/BaselineOfClapMockNoTestGroup.class.st new file mode 100644 index 00000000000..27389132795 --- /dev/null +++ b/src/SUnit-UI-Tests/BaselineOfClapMockNoTestGroup.class.st @@ -0,0 +1,19 @@ +Class { + #name : 'BaselineOfClapMockNoTestGroup', + #superclass : 'BaselineOf', + #category : 'SUnit-UI-Tests-Mocks', + #package : 'SUnit-UI-Tests', + #tag : 'Mocks' +} + +{ #category : 'baselines' } +BaselineOfClapMockNoTestGroup >> baseline: spec [ + + spec for: #common do: [ + spec package: 'FakePackage1'. + spec package: 'FakePackage2'. + spec package: 'FakePackage-Tests' with: [ spec requires: #('FakePackage1') ]. + + spec group: 'Core' with: #('FakePackage2'). + ] +] diff --git a/src/SUnit-UI-Tests/ClapTestRunnerTest.class.st b/src/SUnit-UI-Tests/ClapTestRunnerTest.class.st index 39f3ab5126f..2ee2510581d 100644 --- a/src/SUnit-UI-Tests/ClapTestRunnerTest.class.st +++ b/src/SUnit-UI-Tests/ClapTestRunnerTest.class.st @@ -86,6 +86,44 @@ ClapTestRunnerTest >> testDefaultTestRunnerIsVTermTestRunner [ self assert: command testRunner equals: VTermTestRunner ] +{ #category : 'tests' } +ClapTestRunnerTest >> testResolvePackagesForGeneralProject [ + "Tests that a baseline mixing direct packages and nested groups in its Tests group is resolved correctly" + | runner packages | + runner := ClapTestRunner new. + packages := runner testPackagesForProject: 'ClapMockGeneralProject'. + + self assert: (packages includes: 'FakePackage1'). + self assert: (packages includes: 'FakePackage2'). + self assert: (packages includes: 'FakePackage-Tests'). + self assert: (packages includes: 'Core') not. + + "Check there is no list in the list " + self assert: (packages allSatisfy: [ :each | each isString ]). +] + +{ #category : 'tests' } +ClapTestRunnerTest >> testResolvePackagesForProjectWithoutTestsGroup [ + "Tests a project that has a baseline but no 'Tests' group" + | runner packages | + runner := ClapTestRunner new. + + packages := runner testPackagesForProject: 'ClapMockNoTestGroup'. + + self assert: packages isEmpty. +] + +{ #category : 'tests' } +ClapTestRunnerTest >> testResolvePackagesForUnknownProject [ + "Tests that an unknown project returns an empty collection without crashing" + | runner packages | + runner := ClapTestRunner new. + + packages := runner testPackagesForProject: 'ThisProjectDoesNotExist404'. + + self assert: packages isEmpty. +] + { #category : 'tests' } ClapTestRunnerTest >> testShouldFailOnErrorIsFalseWhenFailOnErrorFlagAbsent [ diff --git a/tests/pharo-cli-test-runner.bats b/tests/pharo-cli-test-runner.bats index b70966463a1..fa9c48fe12d 100644 --- a/tests/pharo-cli-test-runner.bats +++ b/tests/pharo-cli-test-runner.bats @@ -2,6 +2,9 @@ load test_helper.bash +BASE_TEST_COUNT=19 +TOTAL_TEST_COUNT=20 + setup() { setup-workdir } @@ -12,25 +15,25 @@ teardown() { @test "test --help prints help" { run_pharo test --help - assert_output --partial "Usage: test [--help] [--junit-xml-output] [--shuffle-seed ] [--stage-name ] [--no-xterm] [--fail-on-error] [--fail-on-failure] [--save] [--rename] []" + assert_output --partial "Usage: test [--help] [--junit-xml-output] [--shuffle-seed ] [--stage-name ] [--project-name ] [--no-xterm] [--fail-on-error] [--fail-on-failure] [--save] [--rename] []" assert_success } @test "test can run tests on a package" { run_pharo test SUnit-UI-Tests assert_success - assert_line "Finished running 16 Tests" + assert_line "Finished running ${BASE_TEST_COUNT} Tests" assert_line --partial "Finished to run tests of SUnit-UI-Tests in" - assert_line "16 run, 16 passes, 0 failures, 0 errors." + assert_line "${BASE_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 0 failures, 0 errors." } @test "test can run tests on a regex" { run_pharo test SUnit-UI.* assert_success assert_line "Running tests in 2 Packages" - assert_line "Finished running 16 Tests" + assert_line "Finished running ${BASE_TEST_COUNT} Tests" assert_line --partial "Finished to run tests of SUnit-UI-Tests in" - assert_line "16 run, 16 passes, 0 failures, 0 errors." + assert_line "${BASE_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 0 failures, 0 errors." } @test "test can run tests with junit XML output" { @@ -47,7 +50,7 @@ teardown() { run_pharo test --fail-on-failure SUnit-UI-Tests assert_failure - assert_line --partial "17 run, 16 passes, 1 failures, 0 errors." + assert_line --partial "${TOTAL_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 1 failures, 0 errors." } @test "test runner does not fail if a test fails with flag --fail-on-error" { @@ -57,7 +60,7 @@ teardown() { run_pharo test --fail-on-error SUnit-UI-Tests assert_success - assert_line --partial "17 run, 16 passes, 1 failures, 0 errors." + assert_line --partial "${TOTAL_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 1 failures, 0 errors." } @test "test runner fails if a test fails with error with flag --fail-on-error" { @@ -67,5 +70,5 @@ teardown() { run_pharo test --fail-on-error SUnit-UI-Tests assert_failure - assert_line --partial "17 run, 16 passes, 0 failures, 1 errors." -} \ No newline at end of file + assert_line --partial "${TOTAL_TEST_COUNT} run, ${BASE_TEST_COUNT} passes, 0 failures, 1 errors." +}