ci: switch to ui-test instead of trybuild

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/workflows/build.yml

@@ -98,8 +98,8 @@ jobs:
         run: nox -s set-msrv-package-versions
 
       - if: inputs.rust != 'stable'
-        name: Ignore changed error messages when using trybuild
-        run: echo "TRYBUILD=overwrite" >> "$GITHUB_ENV"
+        name: Ignore changed error messages for ui tests (still run for coverage)
+        run: echo "UI_TEST=ignore" >> "$GITHUB_ENV"
 
       - uses: dorny/paths-filter@v4
         if: ${{ inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') }}

.github/workflows/ci.yml

@@ -378,7 +378,7 @@ jobs:
     env:
       CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: valgrind --leak-check=no --error-exitcode=1
       RUST_BACKTRACE: 1
-      TRYBUILD: overwrite
+      UI_TEST: skip
 
   careful:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
@@ -400,7 +400,7 @@ jobs:
       - run: nox -s test-rust -- careful skip-full
     env:
       RUST_BACKTRACE: 1
-      TRYBUILD: overwrite
+      UI_TEST: skip
 
   docsrs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}

Cargo.toml

@@ -69,7 +69,6 @@ portable-atomic = "1.0"
 assert_approx_eq = "1.1.0"
 chrono = "0.4.25"
 chrono-tz = ">= 0.10, < 0.11"
-trybuild = ">=1.0.115"
 proptest = { version = "1.0", default-features = false, features = ["std"] }
 send_wrapper = "0.6"
 serde = { version = "1.0", features = ["derive"] }
@@ -80,6 +79,10 @@ tempfile = "3.12.0"
 static_assertions = "1.1.0"
 uuid = { version = "1.10.0", features = ["v4"] }
 parking_lot = { version = "0.12.3", features = ["arc_lock"] }
+ui_test = "0.30.4"
+regex = "1.12.3"
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+ctrlc = "3.5.2"
 
 [build-dependencies]
 pyo3-build-config = { path = "pyo3-build-config", version = "=0.28.3", features = ["resolve-config"] }
@@ -180,6 +183,7 @@ members = [
     "pyo3-macros",
     "pyo3-macros-backend",
     "pyo3-introspection",
+    "tests/ui/base",
     "pytests",
     "examples",
 ]
@@ -222,3 +226,8 @@ bare_urls = "warn"
 
 [lints]
 workspace = true
+
+[[test]]
+name = "test_compile_error"
+path = "tests/test_compile_error.rs"
+harness = false

Contributing.md

@@ -45,7 +45,7 @@ Use  `nox -l` to list the full set of subcommands you can run.
 
 #### UI Tests
 
-PyO3 uses [`trybuild`][trybuild] to develop UI tests to capture error messages from the Rust compiler for some of the macro functionality.
+PyO3 uses [`ui_test`][ui_test] to develop UI tests to capture error messages from the Rust compiler for some of the macro functionality.
 
 The Rust compiler's error output differs depending on whether the `rust-src` component is installed. PyO3's CI has `rust-src` installed, so you need it locally for your UI test output to match:
 
@@ -265,4 +265,4 @@ In the meanwhile, some of our maintainers have personal GitHub sponsorship pages
 [lychee]: https://github.com/lycheeverse/lychee
 [nox]: https://github.com/theacodes/nox
 [pipx]: https://pipx.pypa.io/stable/
-[trybuild]: https://github.com/dtolnay/trybuild
+[ui_test]: https://github.com/oli-obk/ui_test

noxfile.py

@@ -1418,7 +1418,7 @@ def check_feature_powerset(session: nox.Session):
 @nox.session(name="update-ui-tests", venv_backend="none")
 def update_ui_tests(session: nox.Session):
     env = os.environ.copy()
-    env["TRYBUILD"] = "overwrite"
+    env["UI_TEST"] = "bless"
     command = ["test", "--test", "test_compile_error"]
     _run_cargo(session, *command, env=env)
     _run_cargo(session, *command, "--features=full", env=env)

pyo3-ffi/src/structmember.rs

@@ -2,6 +2,8 @@ use std::ffi::c_int;
 
 pub use crate::PyMemberDef;
 
+#[allow(deprecated)]
+pub use crate::_Py_T_OBJECT as T_OBJECT;
 pub use crate::Py_T_BOOL as T_BOOL;
 pub use crate::Py_T_BYTE as T_BYTE;
 pub use crate::Py_T_CHAR as T_CHAR;
@@ -19,12 +21,10 @@ pub use crate::Py_T_UINT as T_UINT;
 pub use crate::Py_T_ULONG as T_ULONG;
 pub use crate::Py_T_ULONGLONG as T_ULONGLONG;
 pub use crate::Py_T_USHORT as T_USHORT;
-#[allow(deprecated)]
-pub use crate::_Py_T_OBJECT as T_OBJECT;
 
-pub use crate::Py_T_PYSSIZET as T_PYSSIZET;
 #[allow(deprecated)]
 pub use crate::_Py_T_NONE as T_NONE;
+pub use crate::Py_T_PYSSIZET as T_PYSSIZET;
 
 /* Flags */
 pub use crate::Py_READONLY as READONLY;

src/pyclass/create_type_object.rs

@@ -11,7 +11,7 @@ use crate::{
             assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc,
             tp_dealloc_with_gc, PyClassImpl, PyClassItemsIter, PyObjectOffset,
         },
-        pymethods::{Getter, PyGetterDef, PyMethodDefType, PySetterDef, Setter, _call_clear},
+        pymethods::{_call_clear, Getter, PyGetterDef, PyMethodDefType, PySetterDef, Setter},
         trampoline::trampoline,
     },
     pycell::impl_::PyClassObjectLayout,

tests/test_compile_error.rs

@@ -1,99 +1,171 @@
-#![cfg(feature = "macros")]
-
-#[cfg(not(target_arch = "wasm32"))] // Not possible to invoke compiler from wasm
-#[test]
-fn test_compile_errors() {
-    let t = trybuild::TestCases::new();
-
-    t.compile_fail("tests/ui/deprecated_pyfn.rs");
-    #[cfg(not(feature = "experimental-inspect"))]
-    t.compile_fail("tests/ui/invalid_property_args.rs");
-    t.compile_fail("tests/ui/invalid_proto_pymethods.rs");
-    #[cfg(not(feature = "experimental-inspect"))]
-    #[cfg(not(all(Py_LIMITED_API, not(Py_3_10))))] // to avoid PyFunctionArgument for &str
-    t.compile_fail("tests/ui/invalid_pyclass_args.rs");
-    t.compile_fail("tests/ui/invalid_pyclass_doc.rs");
-    t.compile_fail("tests/ui/invalid_pyclass_enum.rs");
-    t.compile_fail("tests/ui/invalid_pyclass_init.rs");
-    t.compile_fail("tests/ui/invalid_pyclass_item.rs");
-    #[cfg(Py_3_9)]
-    t.compile_fail("tests/ui/invalid_pyclass_generic.rs");
-    #[cfg(Py_3_9)]
-    t.compile_fail("tests/ui/pyclass_generic_enum.rs");
-    #[cfg(not(feature = "experimental-inspect"))]
-    #[cfg(not(all(Py_LIMITED_API, not(Py_3_10))))] // to avoid PyFunctionArgument for &str
-    t.compile_fail("tests/ui/invalid_pyfunction_argument.rs");
-    t.compile_fail("tests/ui/invalid_pyfunction_definition.rs");
-    t.compile_fail("tests/ui/invalid_pyfunction_signatures.rs");
-    #[cfg(any(not(Py_LIMITED_API), Py_3_11))]
-    t.compile_fail("tests/ui/invalid_pymethods_buffer.rs");
-    // The output is not stable across abi3 / not abi3 and features
-    #[cfg(all(not(Py_LIMITED_API), feature = "full"))]
-    t.compile_fail("tests/ui/invalid_pymethods_duplicates.rs");
-    t.compile_fail("tests/ui/invalid_pymethod_enum.rs");
-    t.compile_fail("tests/ui/invalid_pymethod_names.rs");
-    t.compile_fail("tests/ui/invalid_pymodule_args.rs");
-    t.compile_fail("tests/ui/invalid_pycallargs.rs");
-    t.compile_fail("tests/ui/reject_generics.rs");
-    t.compile_fail("tests/ui/invalid_closure.rs");
-    t.compile_fail("tests/ui/pyclass_send.rs");
-    #[cfg(not(feature = "experimental-inspect"))]
-    t.compile_fail("tests/ui/invalid_annotation.rs");
-    #[cfg(not(feature = "experimental-inspect"))]
-    t.compile_fail("tests/ui/invalid_annotation_return.rs");
-    t.compile_fail("tests/ui/invalid_argument_attributes.rs");
-    t.compile_fail("tests/ui/invalid_intopy_derive.rs");
-    #[cfg(not(windows))]
-    t.compile_fail("tests/ui/invalid_intopy_with.rs");
-    t.compile_fail("tests/ui/invalid_frompy_derive.rs");
-    t.compile_fail("tests/ui/static_ref.rs");
-    t.compile_fail("tests/ui/wrong_aspyref_lifetimes.rs");
-    #[cfg(not(feature = "uuid"))]
-    t.compile_fail("tests/ui/invalid_pyfunctions.rs");
-    t.compile_fail("tests/ui/invalid_pymethods.rs");
-    // output changes with async feature
-    #[cfg(all(not(Py_3_12), Py_LIMITED_API, feature = "experimental-async"))]
-    t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs");
-    #[cfg(not(feature = "experimental-async"))]
-    t.compile_fail("tests/ui/invalid_async.rs");
-    t.compile_fail("tests/ui/invalid_intern_arg.rs");
-    t.compile_fail("tests/ui/invalid_frozen_pyclass_borrow.rs");
-    #[cfg(not(any(feature = "hashbrown", feature = "indexmap")))]
-    t.compile_fail("tests/ui/invalid_pymethod_receiver.rs");
-    #[cfg(not(feature = "experimental-inspect"))]
-    t.compile_fail("tests/ui/missing_intopy.rs");
-    // adding extra error conversion impls changes the output
-    #[cfg(not(any(windows, feature = "eyre", feature = "anyhow", Py_LIMITED_API)))]
-    t.compile_fail("tests/ui/invalid_result_conversion.rs");
-    t.compile_fail("tests/ui/not_send.rs");
-    t.compile_fail("tests/ui/not_send2.rs");
-    t.compile_fail("tests/ui/get_set_all.rs");
-    t.compile_fail("tests/ui/traverse.rs");
-    t.compile_fail("tests/ui/invalid_pymodule_in_root.rs");
-    t.compile_fail("tests/ui/invalid_pymodule_glob.rs");
-    t.compile_fail("tests/ui/invalid_pymodule_trait.rs");
-    t.compile_fail("tests/ui/invalid_pymodule_two_pymodule_init.rs");
-    #[cfg(all(feature = "experimental-async", not(feature = "experimental-inspect")))]
-    #[cfg(any(not(Py_LIMITED_API), Py_3_10))] // to avoid PyFunctionArgument for &str
-    t.compile_fail("tests/ui/invalid_cancel_handle.rs");
-    t.pass("tests/ui/pymodule_missing_docs.rs");
-    #[cfg(not(any(Py_LIMITED_API, feature = "experimental-inspect")))]
-    t.pass("tests/ui/forbid_unsafe.rs");
-    #[cfg(all(Py_LIMITED_API, not(Py_3_12), feature = "experimental-async"))]
-    // output changes with async feature
-    t.compile_fail("tests/ui/abi3_inheritance.rs");
-    #[cfg(all(Py_LIMITED_API, not(Py_3_9)))]
-    t.compile_fail("tests/ui/abi3_weakref.rs");
-    #[cfg(all(Py_LIMITED_API, not(Py_3_9)))]
-    t.compile_fail("tests/ui/abi3_dict.rs");
-    #[cfg(not(feature = "experimental-inspect"))]
-    t.compile_fail("tests/ui/duplicate_pymodule_submodule.rs");
-    #[cfg(all(not(Py_LIMITED_API), Py_3_11))]
-    t.compile_fail("tests/ui/invalid_base_class.rs");
-    #[cfg(any(not(Py_3_10), all(not(Py_3_14), Py_LIMITED_API)))]
-    t.compile_fail("tests/ui/immutable_type.rs");
-    t.pass("tests/ui/ambiguous_associated_items.rs");
-    t.pass("tests/ui/pyclass_probe.rs");
-    t.compile_fail("tests/ui/invalid_pyfunction_warn.rs");
-    t.compile_fail("tests/ui/invalid_pymethods_warn.rs");
+#[cfg(all(
+    // Requires "macros" feature to actually do any meaningful testing
+    feature = "macros",
+    // Not possible to invoke compiler from wasm
+    not(target_arch = "wasm32")
+))]
+fn main() {
+    use std::{env::VarError, path::PathBuf};
+
+    use regex::bytes::Regex;
+    use ui_test::{run_tests, spanned::Span, Config, OptWithLine};
+
+    let mut config = Config::rustc("tests/ui");
+
+    // Various configurations of of
+    match std::env::var("UI_TEST").as_deref() {
+        // Default is to run the test as normal, erroring if output is not as expected.
+        Err(VarError::NotPresent) => {
+            config.output_conflict_handling = ui_test::error_on_output_conflict
+        }
+        // Used to update the output files to match expected output
+        Ok("bless") => config.output_conflict_handling = ui_test::bless_output_files,
+        // This mode is useful for exercising coverage of the proc macros, e.g. on the
+        // nightly compiler and MSRV, where the output may differ from expected.
+        Ok("ignore") => {
+            // Ignore mismatches on stderr / stdout files
+            config.output_conflict_handling = ui_test::ignore_output_conflict;
+
+            // This combination of settings helps ui test ignore the annotations on
+            // the test files themselves:
+
+            // The annotations by default start with //~, changing this to a pattern
+            // which never appears in the files effectively means "ignore all annotations"
+            config.comment_start = "/*DISABLED*/";
+            // Don't error if there are no annotations
+            config.comment_defaults.base().require_annotations =
+                OptWithLine::new(false, Span::default());
+            // Don't error if the test "passes" because there were no annotations
+            config.comment_defaults.base().exit_status = OptWithLine::default();
+        }
+        // Completely running the tests, e.g. under `cargo careful` there is some issue which
+        // doesn't seem worth understanding (we don't gain anything from extra assertions in
+        // the proc-macro code, which is all quite pedestrian).
+        Ok("skip") => return,
+        Err(e) => panic!("error reading UI_TEST environment variable: {e}"),
+        Ok(unknown) => panic!("invalid UI_TEST value: {unknown}"),
+    }
+
+    config.bless_command = Some("UI_TEST=bless cargo test --test test_compile_error".into());
+
+    // There doesn't seem to be a good way to forward all these features automatically,
+    // so have to just list the relevant ones here.
+    let deps_features = [
+        #[cfg(feature = "macros")]
+        "pyo3/macros".to_string(),
+        #[cfg(feature = "abi3")]
+        "pyo3/abi3".to_string(),
+        #[cfg(feature = "abi3-py38")]
+        "pyo3/abi3-py38".to_string(),
+        #[cfg(feature = "abi3-py39")]
+        "pyo3/abi3-py39".to_string(),
+        #[cfg(feature = "abi3-py310")]
+        "pyo3/abi3-py310".to_string(),
+        #[cfg(feature = "abi3-py311")]
+        "pyo3/abi3-py311".to_string(),
+        #[cfg(feature = "abi3-py312")]
+        "pyo3/abi3-py312".to_string(),
+        #[cfg(feature = "abi3-py313")]
+        "pyo3/abi3-py313".to_string(),
+        #[cfg(feature = "abi3-py314")]
+        "pyo3/abi3-py314".to_string(),
+        #[cfg(feature = "full")]
+        "pyo3/full".to_string(),
+    ];
+
+    let mut deps_cargo = ui_test::CommandBuilder::cargo();
+    deps_cargo.args.push("--features".into());
+    deps_cargo.args.push(deps_features.join(",").into());
+
+    config.comment_defaults.base().set_custom(
+        "dependencies",
+        ui_test::dependencies::DependencyBuilder {
+            crate_manifest_path: PathBuf::from(
+                env!("CARGO_MANIFEST_DIR").to_owned() + "/tests/ui/base/Cargo.toml",
+            ),
+            program: deps_cargo,
+            ..Default::default()
+        },
+    );
+
+    if let Ok(target) = std::env::var("CARGO_BUILD_TARGET") {
+        config.target = Some(target);
+    }
+
+    config
+        .comment_defaults
+        .base()
+        .compile_flags
+        .push("--diagnostic-width=140".into());
+
+    config.skip_files.extend([
+        // not a test file, used to configure dependencies for the tests
+        "base/src/lib.rs".into(),
+        // similarly, just a component of `invalid_pymodule_in_root.rs`
+        "empty.rs".into(),
+        // abi3-only tests only need to check when the feature is unsupported
+        #[cfg(any(not(Py_LIMITED_API), Py_3_9))]
+        "abi3_dict".into(),
+        #[cfg(any(not(Py_LIMITED_API), Py_3_9))]
+        "abi3_weakref".into(),
+        #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
+        "abi3_nativetype_inheritance".into(),
+        #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
+        "abi3_inheritance".into(),
+        // this test doesn't work properly without the full API available
+        #[cfg(Py_LIMITED_API)]
+        "forbid_unsafe.rs".into(),
+        // buffer protocol only supported on 3.11+ with abi3
+        #[cfg(all(Py_LIMITED_API, not(Py_3_11)))]
+        "buffer".into(),
+        // only needs to run on versions where `#[pyclass(immutable_type)]` is unsupported
+        #[cfg(any(Py_3_14, all(Py_3_10, not(Py_LIMITED_API))))]
+        "immutable_type.rs".into(),
+        // generic pyclasses only supported on 3.9+, doesn't fail gracefully on older versions
+        #[cfg(not(Py_3_9))]
+        "invalid_pyclass_generic.rs".into(),
+        // an extra "note" is emitted on abi3
+        #[cfg(any(not(Py_LIMITED_API), not(Py_3_12)))]
+        "invalid_base_class.rs".into(),
+        #[cfg(all(Py_LIMITED_API, not(Py_3_10)))]
+        "invalid_pyfunction_argument.rs".into(),
+        #[cfg(all(Py_LIMITED_API, not(Py_3_10)))]
+        "invalid_pyclass_args.rs".into(),
+        // tests that async functions are rejected without the feature
+        #[cfg(feature = "experimental-async")]
+        "invalid_async.rs".into(),
+        // requires the async feature
+        #[cfg(not(feature = "experimental-async"))]
+        "invalid_cancel_handle.rs".into(),
+    ]);
+
+    // differs on `experimental-inspect` feature
+    #[cfg(feature = "experimental-inspect")]
+    config.skip_files.extend([
+        // some functionality requires the feature
+        "invalid_annotation.rs".into(),
+        "invalid_annotation_return.rs".into(),
+        // extra error messages appear due to additional macro processing
+        // would be nice to somehow make this not a problem
+        "duplicate_pymodule_submodule.rs".into(),
+        "missing_intopy.rs".into(),
+        "invalid_pyclass_args.rs".into(),
+        "invalid_property_args.rs".into(),
+        "invalid_pyfunction_argument.rs".into(),
+    ]);
+
+    // Normalize multiple trailing newlines to a single newline
+    config
+        .comment_defaults
+        .base()
+        .normalize_stderr
+        .push((Regex::new("\n\n$").unwrap().into(), vec![b'\n']));
+
+    let abort_check = config.abort_check.clone();
+    ctrlc::set_handler(move || abort_check.abort()).unwrap();
+
+    run_tests(config).unwrap();
 }
+
+#[cfg(any(not(feature = "macros"), target_arch = "wasm32"))]
+fn main() {}