diff --git a/newsfragments/5904.added.md b/newsfragments/5904.added.md new file mode 100644 index 00000000000..cd5eeb1f123 --- /dev/null +++ b/newsfragments/5904.added.md @@ -0,0 +1 @@ +`pyo3-introspection`: add a small CLI to generate stubs \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index 42cf158e1b5..4dfad9efa21 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1413,35 +1413,62 @@ def update_ui_tests(session: nox.Session): @nox.session(name="test-introspection") def test_introspection(session: nox.Session): - session.install("maturin") - session.install("ruff") - options = [] - target = os.environ.get("CARGO_BUILD_TARGET") - if target is not None: - options += ("--target", target) - profile = os.environ.get("CARGO_BUILD_PROFILE") - if profile == "release": - options.append("--release") - session.run_always( - "maturin", - "develop", - "-m", - "./pytests/Cargo.toml", - "--features", - "experimental-async,experimental-inspect", - *options, - ) - lib_file = session.run( - "python", - "-c", - "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)", - silent=True, - ).strip() - _run_cargo_test( - session, - package="pyo3-introspection", - env={"PYO3_PYTEST_LIB_PATH": lib_file}, - ) + with tempfile.TemporaryDirectory() as stub_dir: + session.install("maturin") + session.install("ruff") + options = [] + target = os.environ.get("CARGO_BUILD_TARGET") + if target is not None: + options += ("--target", target) + profile = os.environ.get("CARGO_BUILD_PROFILE") + if profile == "release": + options.append("--release") + _run( + session, + "maturin", + "develop", + "-m", + "./pytests/Cargo.toml", + "--features", + "experimental-async,experimental-inspect", + *options, + ) + lib_file = session.run( + "python", + "-c", + "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)", + silent=True, + ).strip() + _run_cargo( + session, + "run", + "-p", + "pyo3-introspection", + "--", + lib_file, + "pyo3_pytests", + stub_dir, + ) + _run(session, "ruff", "format", stub_dir) + _ensure_directory_equals(Path(stub_dir), Path("pytests/stubs")) + + +def _ensure_directory_equals(expected_dir: Path, actual_dir: Path): + # Assert all expected files are in actual and are equals + for expected_file_path in expected_dir.rglob("*"): + file_path = expected_file_path.relative_to(expected_dir) + actual_file_path = actual_dir / file_path + assert actual_file_path.exists(), f"File {file_path} does not exist" + assert expected_file_path.read_text() == actual_file_path.read_text(), ( + f"Content is different in {file_path}" + ) + # Assert all actual files are expected + for actual_file_path in actual_dir.rglob("*"): + file_path = actual_file_path.relative_to(actual_dir) + expected_file_path = expected_dir / file_path + assert expected_file_path.exists(), ( + f"File {file_path} exist even if not expected" + ) def _build_docs_for_ffi_check(session: nox.Session) -> None: diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml index 1d06082d0b8..34d2b04e992 100644 --- a/pyo3-introspection/Cargo.toml +++ b/pyo3-introspection/Cargo.toml @@ -14,8 +14,5 @@ goblin = ">=0.9, <0.11" serde = { version = "1", features = ["derive"] } serde_json = "1" -[dev-dependencies] -tempfile = "3.12.0" - [lints] workspace = true diff --git a/pyo3-introspection/src/main.rs b/pyo3-introspection/src/main.rs new file mode 100644 index 00000000000..f0c64031da2 --- /dev/null +++ b/pyo3-introspection/src/main.rs @@ -0,0 +1,24 @@ +//! Small CLI entry point to introspect a Python cdylib built using PyO3 and generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html). + +use anyhow::{anyhow, Context, Result}; +use pyo3_introspection::{introspect_cdylib, module_stub_files}; +use std::path::Path; +use std::{env, fs}; + +fn main() -> Result<()> { + let [_, binary_path, module_name, output_path] = env::args().collect::>().try_into().map_err(|_| anyhow!("pyo3-introspection takes three arguments, the path of the binary to introspect, the name of the python module to introspect and and the path of the directory to write the stub to"))?; + let module = introspect_cdylib(&binary_path, &module_name) + .with_context(|| format!("Failed to introspect module {binary_path}"))?; + let actual_stubs = module_stub_files(&module); + for (path, module) in actual_stubs { + let file_path = Path::new(&output_path).join(path); + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent).with_context(|| { + format!("Failed to create output directory {}", file_path.display()) + })?; + } + fs::write(&file_path, module) + .with_context(|| format!("Failed to write module {}", file_path.display()))?; + } + Ok(()) +} diff --git a/pyo3-introspection/tests/test.rs b/pyo3-introspection/tests/test.rs deleted file mode 100644 index cf01329e9b1..00000000000 --- a/pyo3-introspection/tests/test.rs +++ /dev/null @@ -1,106 +0,0 @@ -use anyhow::{ensure, Result}; -use pyo3_introspection::{introspect_cdylib, module_stub_files}; -use std::collections::HashMap; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::{env, fs}; -use tempfile::NamedTempFile; - -#[test] -fn pytests_stubs() -> Result<()> { - // We run the introspection - let binary = env::var_os("PYO3_PYTEST_LIB_PATH") - .expect("The PYO3_PYTEST_LIB_PATH constant must be set and target the pyo3-pytests cdylib"); - let module = introspect_cdylib(binary, "pyo3_pytests")?; - let actual_stubs = module_stub_files(&module); - - // We read the expected stubs - let expected_subs_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("pytests") - .join("stubs"); - let mut expected_subs = HashMap::new(); - add_dir_files( - &expected_subs_dir, - &expected_subs_dir.canonicalize()?, - &mut expected_subs, - )?; - - // We ensure we do not have extra generated files - for file_name in actual_stubs.keys() { - assert!( - expected_subs.contains_key(file_name), - "The generated file {} is not in the expected stubs directory pytests/stubs", - file_name.display() - ); - } - - // We ensure the expected files are generated properly - for (file_name, expected_file_content) in &expected_subs { - let actual_file_content = actual_stubs.get(file_name).unwrap_or_else(|| { - panic!( - "The expected stub file {} has not been generated", - file_name.display() - ) - }); - - let actual_file_content = format_with_ruff(actual_file_content)?; - - // We normalize line jumps for compatibility with Windows - assert_eq!( - expected_file_content.replace('\r', ""), - actual_file_content.replace('\r', ""), - "The content of file {} is different", - file_name.display() - ) - } - - Ok(()) -} - -fn add_dir_files( - dir_path: &Path, - base_dir_path: &Path, - output: &mut HashMap, -) -> Result<()> { - for entry in fs::read_dir(dir_path)? { - let entry = entry?; - if entry.file_type()?.is_dir() { - add_dir_files(&entry.path(), base_dir_path, output)?; - } else { - output.insert( - entry - .path() - .canonicalize()? - .strip_prefix(base_dir_path)? - .into(), - fs::read_to_string(entry.path())?, - ); - } - } - Ok(()) -} - -fn format_with_ruff(code: &str) -> Result { - let temp_file = NamedTempFile::with_suffix(".pyi")?; - // Write to file - { - let mut file = temp_file.as_file(); - file.write_all(code.as_bytes())?; - file.flush()?; - file.seek(SeekFrom::Start(0))?; - } - ensure!( - Command::new("ruff") - .arg("format") - .arg(temp_file.path()) - .status()? - .success(), - "Failed to run ruff" - ); - let mut content = String::new(); - temp_file.as_file().read_to_string(&mut content)?; - Ok(content) -}