From 190ee75e7c15af4b0289c1f422a97b9b3990793e Mon Sep 17 00:00:00 2001 From: Joshua Dunn Date: Sat, 28 Mar 2026 23:56:44 +0000 Subject: [PATCH 1/4] feat: defaults tests:vm to not be excluded vm tests are now include, but explicity excluded instandard cicd test suite. test-est still picks these vm tests up as before. --- .github/workflows/test.yml | 6 +++--- Makefile | 2 +- conftest.py | 9 +++++++++ skpro/tests/_config.py | 5 +++++ skpro/tests/test_switch.py | 15 +++++++++++---- 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c96ad8d5..b5b89caaf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -173,7 +173,7 @@ jobs: run: git branch -a - name: Run tests - run: make test + run: make test PYTESTOPTIONS="--skip_vm_tests - name: Publish code coverage uses: codecov/codecov-action@v5 @@ -219,7 +219,7 @@ jobs: run: git branch -a - name: Run tests - run: make test + run: make test PYTESTOPTIONS="--skip_vm_tests - name: Publish code coverage uses: codecov/codecov-action@v5 @@ -311,4 +311,4 @@ jobs: run: git branch -a - name: Run tests - run: make test + run: make test PYTESTOPTIONS="--skip_vm_tests diff --git a/Makefile b/Makefile index dc944673e..f01faaf37 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ test: ## Run unit tests mkdir -p ${TEST_DIR} cp .coveragerc ${TEST_DIR} cp setup.cfg ${TEST_DIR} - python -m pytest + python -m pytest $(PYTESTOPTIONS) test_check_suite: ## run only estimator contract tests in TestAll classes -rm -rf ${TEST_DIR} diff --git a/conftest.py b/conftest.py index 0e57e0053..8eba0dbfd 100644 --- a/conftest.py +++ b/conftest.py @@ -32,6 +32,12 @@ def pytest_addoption(parser): help="test only estimators from modules that have changed compared to main", ) + parser.addoption( + "--skip_vm_tests", + default=False, + help="skip tests for estimators with 'tests:vm' tag (useful in CI/CD)", + ) + def pytest_configure(config): """Pytest configuration preamble.""" @@ -39,3 +45,6 @@ def pytest_configure(config): if config.getoption("--only_changed_modules") in [True, "True"]: _config.ONLY_CHANGED_MODULES = True + + if config.getoption("--skip_vm_tests") in [True, "True"]: + _config.SKIP_VM_TESTS = True diff --git a/skpro/tests/_config.py b/skpro/tests/_config.py index 3118287bd..b0abce721 100644 --- a/skpro/tests/_config.py +++ b/skpro/tests/_config.py @@ -8,6 +8,11 @@ # default is False, can be set to True by pytest --only_changed_modules True flag ONLY_CHANGED_MODULES = False +# whether to skip tests for estimators with "tests:vm" tag +# default is False, can be set to True by pytest --skip_vm_tests flag +# useful in CI/CD where VM tests are run separately in dedicated VMs +SKIP_VM_TESTS = False + # list of str, names of estimators to exclude from testing # WARNING: tests for these estimators will be skipped diff --git a/skpro/tests/test_switch.py b/skpro/tests/test_switch.py index 786d754ea..1d6d39d7e 100644 --- a/skpro/tests/test_switch.py +++ b/skpro/tests/test_switch.py @@ -92,7 +92,7 @@ class for which to determine whether it should be tested * otherwise, any reasons to run cause the entire list to be run * otherwise, the list is not run due to "no change" """ - from skpro.tests._config import ONLY_CHANGED_MODULES + from skpro.tests._config import ONLY_CHANGED_MODULES, SKIP_VM_TESTS def _return(run, reason): if return_reason: @@ -144,7 +144,9 @@ def _return(run, reason): # now we know that cls is a class or function, # and not on the exclude list - run, reason = _run_test_for_class(cls, only_changed_modules=ONLY_CHANGED_MODULES) + run, reason = _run_test_for_class( + cls, only_changed_modules=ONLY_CHANGED_MODULES, skip_vm_tests=SKIP_VM_TESTS + ) return _return(run, reason) @@ -154,6 +156,7 @@ def _run_test_for_class( ignore_deps=False, only_changed_modules=True, only_vm_required=False, + skip_vm_tests=False, ): """Check if test should run - cached with hashable cls. @@ -171,6 +174,10 @@ class for which to determine whether it should be tested whether th return only classes that require their own VM. If True, will only return classes with tag "tests:vm"=True. If False, will only return classes with tag "tests:vm"=False. + skip_vm_tests : boolean, default=False + whether to skip tests for classes that require their own VM. + If True, will skip classes with tag "tests:vm"=True. + This is useful in CI/CD where VM tests are run separately. Returns ------- @@ -261,8 +268,8 @@ def _is_impacted_by_lib_dep_change(cls, only_changed_modules): return False, "False_required_deps_missing" # otherwise, continue - # if only_vm_required=False, and the class requires a test vm, skip - if not only_vm_required and _requires_vm(cls): + # if skip_vm_tests=True, and the class requires a test vm, skip + if skip_vm_tests and _requires_vm(cls): return False, "False_requires_vm" # if only_vm_required=True, and the class does not require a test vm, skip if only_vm_required and not _requires_vm(cls): From 924c2eaa8c15c54bb6d277187470facb98f2ff91 Mon Sep 17 00:00:00 2001 From: Joshua Dunn Date: Sun, 29 Mar 2026 10:40:29 +0100 Subject: [PATCH 2/4] chore: fix yaml file closing quote --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b5b89caaf..469481b78 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -173,7 +173,7 @@ jobs: run: git branch -a - name: Run tests - run: make test PYTESTOPTIONS="--skip_vm_tests + run: make test PYTESTOPTIONS="--skip_vm_tests" - name: Publish code coverage uses: codecov/codecov-action@v5 @@ -219,7 +219,7 @@ jobs: run: git branch -a - name: Run tests - run: make test PYTESTOPTIONS="--skip_vm_tests + run: make test PYTESTOPTIONS="--skip_vm_tests" - name: Publish code coverage uses: codecov/codecov-action@v5 @@ -311,4 +311,4 @@ jobs: run: git branch -a - name: Run tests - run: make test PYTESTOPTIONS="--skip_vm_tests + run: make test PYTESTOPTIONS="--skip_vm_tests" From 95f3ea66e0ae99a00027773d5cfe554a644e3729 Mon Sep 17 00:00:00 2001 From: Joshua Dunn Date: Sun, 29 Mar 2026 11:47:33 +0100 Subject: [PATCH 3/4] feat: remove expected param --- conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 8eba0dbfd..647d0e536 100644 --- a/conftest.py +++ b/conftest.py @@ -28,12 +28,14 @@ def pytest_addoption(parser): """Pytest command line parser options adder.""" parser.addoption( "--only_changed_modules", + action="store_true", default=False, help="test only estimators from modules that have changed compared to main", ) parser.addoption( "--skip_vm_tests", + action="store_true", default=False, help="skip tests for estimators with 'tests:vm' tag (useful in CI/CD)", ) @@ -43,8 +45,8 @@ def pytest_configure(config): """Pytest configuration preamble.""" from skpro.tests import _config - if config.getoption("--only_changed_modules") in [True, "True"]: + if config.getoption("--only_changed_modules"): _config.ONLY_CHANGED_MODULES = True - if config.getoption("--skip_vm_tests") in [True, "True"]: + if config.getoption("--skip_vm_tests"): _config.SKIP_VM_TESTS = True From 6a8410d5d824e9c222be4f143e77f548edf3f09f Mon Sep 17 00:00:00 2001 From: Joshua Dunn Date: Sun, 29 Mar 2026 12:28:39 +0100 Subject: [PATCH 4/4] feat: add run all modules so everything can be run if needed --- conftest.py | 12 +++++++++++- setup.cfg | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 647d0e536..5989e480d 100644 --- a/conftest.py +++ b/conftest.py @@ -33,6 +33,13 @@ def pytest_addoption(parser): help="test only estimators from modules that have changed compared to main", ) + parser.addoption( + "--run_all_modules", + action="store_true", + default=False, + help="run tests for all modules (override --only_changed_modules)", + ) + parser.addoption( "--skip_vm_tests", action="store_true", @@ -45,7 +52,10 @@ def pytest_configure(config): """Pytest configuration preamble.""" from skpro.tests import _config - if config.getoption("--only_changed_modules"): + # --run_all_modules overrides --only_changed_modules + if config.getoption("--run_all_modules"): + _config.ONLY_CHANGED_MODULES = False + elif config.getoption("--only_changed_modules"): _config.ONLY_CHANGED_MODULES = True if config.getoption("--skip_vm_tests"): diff --git a/setup.cfg b/setup.cfg index 8b6c4d685..bfaad56d4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,7 @@ addopts = --cov-report xml --cov-report html --showlocals - --only_changed_modules True + --only_changed_modules -n auto filterwarnings = ignore::UserWarning