Model REQUIRE/REQUIRE_FALSE and REQUIRE_THROWS/NOTHROW/THROWS_AS early termination for Static Analysis mode

.github/workflows/linux-other-builds.yml

@@ -127,3 +127,26 @@ jobs:
 
     - name: Run clang-tidy
       run: cmake --build build
+
+  static-analysis-tests:
+    name: static analysis support tests
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Prepare environment
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y ninja-build clang-18 clang-tidy-18
+
+    - name: Configure
+      run: |
+        cmake --preset basic-tests -GNinja \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DCMAKE_CXX_COMPILER=clang++-18 \
+              -DCMAKE_CXX_STANDARD=17 \
+              -DCATCH_ENABLE_STATIC_ANALYSIS_TESTS=ON \
+              -DCATCH_CLANG_TIDY_COMMAND=clang-tidy-18
+
+    - name: Test
+      run: ctest --test-dir build -R "StaticAnalysis" --output-on-failure

CMakeLists.txt

@@ -26,6 +26,8 @@ cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CAT
 cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_ENABLE_CMAKE_HELPER_TESTS "Enable CMake helper tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
+cmake_dependent_option(CATCH_ENABLE_STATIC_ANALYSIS_TESTS "Enable static analysis tests. WARNING: VERY EXPENSIVE, needs clang-tidy 17 or newer" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
+set(CATCH_CLANG_TIDY_COMMAND "clang-tidy" CACHE STRING "clang-tidy binary used by the static analysis tests, has to be version 17 or newer")
 
 # Catch2's build breaks if done in-tree. You probably should not build
 # things in tree anyway, but we can allow projects that include Catch2

CMakePresets.json

@@ -33,7 +33,8 @@
             "description": "Enables development build with examples and ALL tests",
             "cacheVariables": {
                 "CATCH_ENABLE_CONFIGURE_TESTS": "ON",
-                "CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON"
+                "CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON",
+                "CATCH_ENABLE_STATIC_ANALYSIS_TESTS": "ON"
             }
         }
     ]   

docs/contributing.md

@@ -84,12 +84,19 @@ time. This can be either because they take a long time to run, or because
 they take a long time to compile, e.g. because they test compile time
 configuration and require separate compilation.
 
-Finally, CMake config tests test that you set Catch2's compile-time
+CMake config tests test that you set Catch2's compile-time
 configuration options through CMake, using CMake options of the same name.
 
+Finally, static analysis tests check Catch2's static analysis support, by
+running clang-tidy over dedicated test files and comparing its warnings
+against the expected ones. They need clang-tidy 17 or newer, either on
+`PATH` or pointed at with `-DCATCH_CLANG_TIDY_COMMAND=<binary>`, and are
+skipped otherwise.
+
 These test categories can be enabled one by one, by passing
-`-DCATCH_BUILD_EXAMPLES=ON`, `-DCATCH_BUILD_EXTRA_TESTS=ON`, and
-`-DCATCH_ENABLE_CONFIGURE_TESTS=ON` when configuring the build.
+`-DCATCH_BUILD_EXAMPLES=ON`, `-DCATCH_BUILD_EXTRA_TESTS=ON`,
+`-DCATCH_ENABLE_CONFIGURE_TESTS=ON`, and
+`-DCATCH_ENABLE_STATIC_ANALYSIS_TESTS=ON` when configuring the build.
 
 Catch2 also provides a preset that promises to enable _all_ test types,
 `all-tests`.

src/catch2/internal/catch_result_type.hpp

@@ -57,6 +57,9 @@ namespace Catch {
     constexpr bool isFalseTest( int flags ) {
         return ( flags & ResultDisposition::FalseTest ) != 0;
     }
+    constexpr bool shouldTerminateOnFailure( int flags ) {
+        return ( flags & ResultDisposition::Normal ) != 0;
+    }
     constexpr bool shouldSuppressFailure( int flags ) {
         return ( flags & ResultDisposition::SuppressFail ) != 0;
     }

src/catch2/internal/catch_test_macro_impl.hpp

@@ -10,9 +10,12 @@
 
 #include <catch2/catch_user_config.hpp>
 #include <catch2/internal/catch_assertion_handler.hpp>
+#include <catch2/internal/catch_config_static_analysis_support.hpp>
 #include <catch2/internal/catch_preprocessor_internal_stringify.hpp>
+#include <catch2/internal/catch_result_type.hpp>
 #include <catch2/internal/catch_stringref.hpp>
 #include <catch2/internal/catch_source_line_info.hpp>
+#include <catch2/internal/catch_unreachable.hpp>
 
 namespace Catch {
     namespace Detail {
@@ -57,6 +60,8 @@ namespace Catch {
         (void)0, (false) && static_cast<const bool&>( !!(__VA_ARGS__) )
 #endif
 
+#if !defined( CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT )
+
 ///////////////////////////////////////////////////////////////////////////////
 #define INTERNAL_CATCH_TEST( macroName, resultDisposition, ... ) \
     do { /* NOLINT(bugprone-infinite-loop) */ \
@@ -72,6 +77,25 @@ namespace Catch {
         catchAssertionHandler.complete(); \
     } while( INTERNAL_CATCH_TEST_LOOP_CONDITION( __VA_ARGS__ ) ) // the expression here is never evaluated at runtime, but is shown to some compilers
 
+#else  // ^^ !CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT | vv CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+///////////////////////////////////////////////////////////////////////////////
+#    define INTERNAL_CATCH_TEST( macroName, resultDisposition, ... ) \
+        do { \
+            const bool catchInternalAssertionResult = static_cast<bool>( __VA_ARGS__ ); \
+            if ( Catch::shouldTerminateOnFailure( resultDisposition ) ) { \
+                if ( Catch::isFalseTest( resultDisposition ) ) { \
+                    if ( catchInternalAssertionResult ) { Catch::Detail::Unreachable(); } \
+                } else { \
+                    if ( !catchInternalAssertionResult ) { Catch::Detail::Unreachable(); } \
+                } \
+            } \
+        } while ( false )
+
+#endif // CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+#if !defined( CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT )
+
 ///////////////////////////////////////////////////////////////////////////////
 #define INTERNAL_CATCH_IF( macroName, resultDisposition, ... ) \
     INTERNAL_CATCH_TEST( macroName, resultDisposition, __VA_ARGS__ ); \
@@ -82,6 +106,20 @@ namespace Catch {
     INTERNAL_CATCH_TEST( macroName, resultDisposition, __VA_ARGS__ ); \
     if( !Catch::Detail::lastAssertionPassed() )
 
+#else  // ^^ !CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT | vv CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+///////////////////////////////////////////////////////////////////////////////
+#    define INTERNAL_CATCH_IF( macroName, resultDisposition, ... ) \
+        if ( __VA_ARGS__ )
+
+///////////////////////////////////////////////////////////////////////////////
+#    define INTERNAL_CATCH_ELSE( macroName, resultDisposition, ... ) \
+        if ( !( __VA_ARGS__ ) )
+
+#endif // CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+#if !defined( CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT )
+
 ///////////////////////////////////////////////////////////////////////////////
 #define INTERNAL_CATCH_NO_THROW( macroName, resultDisposition, ... ) \
     do { \
@@ -99,6 +137,27 @@ namespace Catch {
         catchAssertionHandler.complete(); \
     } while( false )
 
+#else  // ^^ !CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT | vv CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+///////////////////////////////////////////////////////////////////////////////
+#    define INTERNAL_CATCH_NO_THROW( macroName, resultDisposition, ... ) \
+        do { \
+            try { \
+                CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \
+                CATCH_INTERNAL_SUPPRESS_USELESS_CAST_WARNINGS \
+                static_cast<void>(__VA_ARGS__); \
+                CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION \
+            } catch ( ... ) { \
+                if ( Catch::shouldTerminateOnFailure( resultDisposition ) ) { \
+                    Catch::Detail::Unreachable(); \
+                } \
+            } \
+        } while ( false )
+
+#endif // CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+#if !defined( CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT )
+
 ///////////////////////////////////////////////////////////////////////////////
 #define INTERNAL_CATCH_THROWS( macroName, resultDisposition, ... ) \
     do { \
@@ -120,6 +179,31 @@ namespace Catch {
         catchAssertionHandler.complete(); \
     } while( false )
 
+#else  // ^^ !CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT | vv CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+///////////////////////////////////////////////////////////////////////////////
+#    define INTERNAL_CATCH_THROWS( macroName, resultDisposition, ... ) \
+        do { \
+            bool catchInternalThrew = false; \
+            try { \
+                CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \
+                CATCH_INTERNAL_SUPPRESS_UNUSED_RESULT \
+                CATCH_INTERNAL_SUPPRESS_USELESS_CAST_WARNINGS \
+                static_cast<void>(__VA_ARGS__); \
+                CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION \
+            } catch ( ... ) { \
+                catchInternalThrew = true; \
+            } \
+            if ( !catchInternalThrew && \
+                 Catch::shouldTerminateOnFailure( resultDisposition ) ) { \
+                Catch::Detail::Unreachable(); \
+            } \
+        } while ( false )
+
+#endif // CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+#if !defined( CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT )
+
 ///////////////////////////////////////////////////////////////////////////////
 #define INTERNAL_CATCH_THROWS_AS( macroName, exceptionType, resultDisposition, expr ) \
     do { \
@@ -144,6 +228,30 @@ namespace Catch {
         catchAssertionHandler.complete(); \
     } while( false )
 
+#else  // ^^ !CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT | vv CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
+///////////////////////////////////////////////////////////////////////////////
+#    define INTERNAL_CATCH_THROWS_AS( macroName, exceptionType, resultDisposition, expr ) \
+        do { \
+            bool catchInternalCaughtExpected = false; \
+            try { \
+                CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \
+                CATCH_INTERNAL_SUPPRESS_UNUSED_RESULT \
+                CATCH_INTERNAL_SUPPRESS_USELESS_CAST_WARNINGS \
+                static_cast<void>(expr); \
+                CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION \
+            } catch ( exceptionType const& ) { \
+                catchInternalCaughtExpected = true; \
+            } catch ( ... ) { \
+            } \
+            if ( !catchInternalCaughtExpected && \
+                 Catch::shouldTerminateOnFailure( resultDisposition ) ) { \
+                Catch::Detail::Unreachable(); \
+            } \
+        } while ( false )
+
+#endif // CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT
+
 
 
 ///////////////////////////////////////////////////////////////////////////////

tests/CMakeLists.txt

@@ -679,6 +679,21 @@ if(CATCH_ENABLE_CMAKE_HELPER_TESTS)
   )
 endif()
 
+if(CATCH_ENABLE_STATIC_ANALYSIS_TESTS)
+  add_test(NAME "StaticAnalysis::AssertionModelling"
+    COMMAND
+      Python3::Interpreter "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testStaticAnalysisSupport.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" "${CATCH_CLANG_TIDY_COMMAND}" "${CMAKE_CXX_COMPILER}"
+  )
+  set_tests_properties("StaticAnalysis::AssertionModelling"
+    PROPERTIES
+      COST 240
+      LABELS "uses-python"
+      # The script exits with 77 when clang-tidy is too old to model
+      # [[noreturn]], in which case the test cannot tell anything apart.
+      SKIP_RETURN_CODE 77
+  )
+endif()
+
 foreach(reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
                        "compact"
                        "console"

tests/TestProjects/StaticAnalysis/.clang-tidy

@@ -0,0 +1,10 @@
+---
+# This configuration deliberately does not inherit the repository-wide one.
+#
+InheritParentConfig: false
+Checks: '-*,bugprone-unchecked-optional-access'
+WarningsAsErrors: ''
+HeaderFilterRegex: ''
+FormatStyle: none
+CheckOptions: []
+...

tests/TestProjects/StaticAnalysis/CMakeLists.txt

@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 3.16)
+project(static-analysis-tests LANGUAGES CXX)
+
+# std::optional, which is what bugprone-unchecked-optional-access reasons about
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+add_subdirectory(${CATCH2_PATH} catch2-build)
+
+set(SA_TEST_SOURCES
+  RequireTerminates.cpp
+  CheckedIf.cpp
+  NoThrowTerminates.cpp
+  ThrowsTerminates.cpp
+  ThrowsAsTerminates.cpp
+)
+
+# These TUs compile but never link. 
+# Per-target clang-tidy keeps Catch2 itself out of the analysis.
+foreach(sourceFile ${SA_TEST_SOURCES})
+  get_filename_component(targetName "${sourceFile}" NAME_WE)
+  add_library(${targetName} OBJECT "${sourceFile}")
+  target_link_libraries(${targetName} PRIVATE Catch2::Catch2)
+  set_target_properties(${targetName}
+    PROPERTIES
+      CXX_CLANG_TIDY "${STATIC_ANALYSIS_CLANG_TIDY}"
+  )
+endforeach()

tests/TestProjects/StaticAnalysis/CheckedIf.cpp

@@ -0,0 +1,24 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/catch_test_macros.hpp>
+
+#include <optional>
+
+// Deliberately not defined, so that the analysis cannot see through it.
+std::optional<int> maybe_value();
+
+TEST_CASE( "CHECKED_IF branches on the condition" ) {
+    std::optional<int> opt = maybe_value();
+    CHECKED_IF( opt.has_value() ) { CHECK( *opt == 42 ); }
+}
+
+TEST_CASE( "CHECKED_ELSE branches on the negated condition" ) {
+    std::optional<int> opt = maybe_value();
+    CHECKED_ELSE( opt.has_value() ) { CHECK( *opt == 42 ); } // expect-warning: bugprone-unchecked-optional-access
+}

tests/TestProjects/StaticAnalysis/NoThrowTerminates.cpp

@@ -0,0 +1,24 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/catch_test_macros.hpp>
+
+#include <optional>
+#include <stdexcept>
+
+TEST_CASE( "REQUIRE_NOTHROW terminates the test case when the expression throws" ) {
+    std::optional<int> opt;
+    REQUIRE_NOTHROW( opt ? 0 : throw std::runtime_error("empty") );
+    CHECK( *opt == 42 );
+}
+
+TEST_CASE( "CHECK_NOTHROW keeps going when the expression throws" ) {
+    std::optional<int> opt;
+    CHECK_NOTHROW( opt ? 0 : throw std::runtime_error("empty") );
+    CHECK( *opt == 42 ); // expect-warning: bugprone-unchecked-optional-access
+}

tests/TestProjects/StaticAnalysis/RequireTerminates.cpp

@@ -0,0 +1,32 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/catch_test_macros.hpp>
+
+#include <optional>
+
+// Deliberately not defined, so that the analysis cannot see through it.
+std::optional<int> maybe_value();
+
+TEST_CASE( "REQUIRE terminates the test case on failure" ) {
+    std::optional<int> opt = maybe_value();
+    REQUIRE( opt.has_value() );
+    CHECK( *opt == 42 );
+}
+
+TEST_CASE( "REQUIRE_FALSE terminates the test case on failure" ) {
+    std::optional<int> opt = maybe_value();
+    REQUIRE_FALSE( !opt.has_value() );
+    CHECK( *opt == 42 );
+}
+
+TEST_CASE( "CHECK does not terminate the test case on failure" ) {
+    std::optional<int> opt = maybe_value();
+    CHECK( opt.has_value() );
+    CHECK( *opt == 42 ); // expect-warning: bugprone-unchecked-optional-access
+}