diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index baa858709a0c6..9826649e51fb6 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -238,6 +238,36 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path } + + fn unpack_archive(archive: &PathBuf, output: &Path, mut skip: F) -> io::Result<()> + where + F: FnMut(&str) -> bool + 'static, + { + let archive = ArchiveRO::open(archive).unwrap(); + + for child in archive.iter() { + let lib = child.map_err(string_to_io_error)?; + let Some(name) = &lib.name() else { + continue; + }; + + if skip(name) { + continue; + } + + let data = lib.data(); + let out = output.join(name); + + match std::fs::write(out, data) { + Ok(_) => {} + Err(err) => { + panic!("{}", err.kind().to_string()); + } + } + } + + Ok(()) + } } impl<'a> LlvmArchiveBuilder<'a> { diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs index 53550b049db09..861e07f924832 100644 --- a/compiler/rustc_codegen_ssa/src/back/archive.rs +++ b/compiler/rustc_codegen_ssa/src/back/archive.rs @@ -53,6 +53,10 @@ pub trait ArchiveBuilder<'a> { fn sess(&self) -> &Session; + fn unpack_archive(archive: &PathBuf, output: &Path, skip: F) -> io::Result<()> + where + F: FnMut(&str) -> bool + 'static; + /// Creates a DLL Import Library . /// and returns the path on disk to that import library. /// This functions doesn't take `self` so that it can be called from diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index f0d320c7c21c9..e61d3b9142d31 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -300,54 +300,56 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( } } - // Note that in this loop we are ignoring the value of `lib.cfg`. That is, - // we may not be configured to actually include a static library if we're - // adding it here. That's because later when we consume this rlib we'll - // decide whether we actually needed the static library or not. - // - // To do this "correctly" we'd need to keep track of which libraries added - // which object files to the archive. We don't do that here, however. The - // #[link(cfg(..))] feature is unstable, though, and only intended to get - // liblibc working. In that sense the check below just indicates that if - // there are any libraries we want to omit object files for at link time we - // just exclude all custom object files. - // - // Eventually if we want to stabilize or flesh out the #[link(cfg(..))] - // feature then we'll need to figure out how to record what objects were - // loaded from the libraries found here and then encode that into the - // metadata of the rlib we're generating somehow. - for lib in codegen_results.crate_info.used_libraries.iter() { - match lib.kind { - NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } - if flavor == RlibFlavor::Normal => - { - // Don't allow mixing +bundle with +whole_archive since an rlib may contain - // multiple native libs, some of which are +whole-archive and some of which are - // -whole-archive and it isn't clear how we can currently handle such a - // situation correctly. - // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897 - sess.err( - "the linking modifiers `+bundle` and `+whole-archive` are not compatible \ - with each other when generating rlibs", + if flavor == RlibFlavor::StaticlibBase + || !sess.opts.unstable_opts.separate_native_rlib_dependencies + { + // Note that in this loop we are ignoring the value of `lib.cfg`. That is, + // we may not be configured to actually include a static library if we're + // adding it here. That's because later when we consume this rlib we'll + // decide whether we actually needed the static library or not. + // + // To do this "correctly" we'd need to keep track of which libraries added + // which object files to the archive. We don't do that here, however. The + // #[link(cfg(..))] feature is unstable, though, and only intended to get + // liblibc working. In that sense the check below just indicates that if + // there are any libraries we want to omit object files for at link time we + // just exclude all custom object files. + // + // Eventually if we want to stabilize or flesh out the #[link(cfg(..))] + // feature then we'll need to figure out how to record what objects were + // loaded from the libraries found here and then encode that into the + // metadata of the rlib we're generating somehow. + for lib in codegen_results.crate_info.used_libraries.iter() { + match lib.kind { + NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } => { + if flavor == RlibFlavor::Normal { + sess.err( + "the linking modifiers `+bundle` and `+whole-archive` are not compatible \ + with each other when generating rlibs"); + } + } + NativeLibKind::Static { bundle: None | Some(true), .. } => {} + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Framework { .. } + | NativeLibKind::RawDylib + | NativeLibKind::Unspecified => continue, + } + if let Some(name) = lib.name { + let location = find_library( + name.as_str(), + lib.verbatim.unwrap_or(false), + &lib_search_paths, + sess, ); + ab.add_archive(&location, |_| false).unwrap_or_else(|e| { + sess.fatal(&format!( + "failed to add native library {}: {}", + location.to_string_lossy(), + e + )); + }); } - NativeLibKind::Static { bundle: None | Some(true), .. } => {} - NativeLibKind::Static { bundle: Some(false), .. } - | NativeLibKind::Dylib { .. } - | NativeLibKind::Framework { .. } - | NativeLibKind::RawDylib - | NativeLibKind::Unspecified => continue, - } - if let Some(name) = lib.name { - let location = - find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess); - ab.add_archive(&location, |_| false).unwrap_or_else(|e| { - sess.fatal(&format!( - "failed to add native library {}: {}", - location.to_string_lossy(), - e - )); - }); } } @@ -386,6 +388,30 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( ab.add_file(&trailing_metadata); } + // add all native rlib dependencies + // archieves added to the end of .rlib archieve + if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.separate_native_rlib_dependencies { + for lib in codegen_results.crate_info.used_libraries.iter() { + match lib.kind { + NativeLibKind::Static { bundle: None | Some(true), .. } => {} + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Framework { .. } + | NativeLibKind::RawDylib + | NativeLibKind::Unspecified => continue, + } + if let Some(name) = lib.name { + let location = find_library( + name.as_str(), + lib.verbatim.unwrap_or(false), + &lib_search_paths, + sess, + ); + ab.add_file(&location) + } + } + } + return Ok(ab); } @@ -2364,6 +2390,12 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( // will provide them to the linker itself. if sess.opts.unstable_opts.link_native_libraries { let mut last = (None, NativeLibKind::Unspecified, None); + if sess.opts.unstable_opts.separate_native_rlib_dependencies { + B::unpack_archive(&src.rlib.as_ref().unwrap().0, tmpdir, |fname: &str| { + !fname.ends_with(".a") + }) + .unwrap(); + } for lib in &codegen_results.crate_info.native_libraries[&cnum] { let Some(name) = lib.name else { continue; @@ -2379,7 +2411,29 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( } else { (lib.name, lib.kind, lib.verbatim) }; - + /* + match lib.kind { + NativeLibKind::Static { bundle: Some(false), whole_archive } + | NativeLibKind::Static { bundle: Some(true) | None, whole_archive } + if sess.opts.unstable_opts.separate_native_rlib_dependencies => + { + let verbatim = lib.verbatim.unwrap_or(false); + let tmp_path = &[PathBuf::from(tmpdir)]; + let path: &[PathBuf] = + if sess.opts.unstable_opts.separate_native_rlib_dependencies { + tmp_path + } else { + search_path.get_or_init(|| archive_search_paths(sess)) + }; + if whole_archive == Some(true) { + cmd.link_whole_staticlib(name, verbatim, path); + } else { + cmd.link_staticlib(name, verbatim); + } + } + _ => {} + }; + */ if let NativeLibKind::Static { bundle: Some(false), whole_archive } = lib.kind { @@ -2394,6 +2448,25 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( cmd.link_staticlib(name, verbatim); } } + if sess.opts.unstable_opts.separate_native_rlib_dependencies { + if let NativeLibKind::Static { + bundle: Some(true) | None, + whole_archive, + } = lib.kind + { + let verbatim = lib.verbatim.unwrap_or(false); + if whole_archive == Some(true) { + cmd.link_whole_staticlib( + name, + verbatim, + &[PathBuf::from(tmpdir)], + ); + } else { + cmd.link_staticlib(name, verbatim); + } + } + } + // } } } @@ -2502,6 +2575,12 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( let skip_because_lto = upstream_rust_objects_already_included && is_rust_object && is_builtins; + if + // sess.opts.unstable_opts.separate_native_rlib_dependencies && + f.ends_with(".a") { + return true; + } + if skip_because_cfg_say_so || skip_because_lto { return true; } diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 21d9eaccf67e7..8d22d9c3d0e97 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -784,6 +784,7 @@ fn test_unstable_options_tracking_hash() { tracked!(sanitizer_memory_track_origins, 2); tracked!(sanitizer_recover, SanitizerSet::ADDRESS); tracked!(saturating_float_casts, Some(true)); + tracked!(separate_native_rlib_dependencies, true); tracked!(share_generics, Some(true)); tracked!(show_span, Some(String::from("abc"))); tracked!(simulate_remapped_rust_src_base, Some(PathBuf::from("/rustc/abc"))); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 501997679f4bf..fbfdd7616d19f 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1510,6 +1510,8 @@ options! { `instructions:u` (retired instructions, userspace-only) `instructions-minus-irqs:u` (subtracting hardware interrupt counts for extra accuracy)" ), + separate_native_rlib_dependencies: bool = (false, parse_bool, [TRACKED], + "change rlib format to store native libraries as archieves"), share_generics: Option = (None, parse_opt_bool, [TRACKED], "make the current crate share its generic instantiations"), show_span: Option = (None, parse_opt_string, [TRACKED], diff --git a/src/test/run-make/issue_99429/Makefile b/src/test/run-make/issue_99429/Makefile new file mode 100644 index 0000000000000..8dffec365c0ee --- /dev/null +++ b/src/test/run-make/issue_99429/Makefile @@ -0,0 +1,35 @@ +-include ../../run-make-fulldeps/tools.mk + +# We're using the llvm-nm instead of the system nm to ensure it is compatible +# with the LLVM bitcode generated by rustc. +NM = "$(LLVM_BIN_DIR)"/llvm-nm + +all: $(call NATIVE_STATICLIB,native_dep_1) $(call NATIVE_STATICLIB,native_dep_2) $(call NATIVE_STATICLIB,native_dep_3) + # Build native libs + # $(RUSTC) native_dep_1.rs --crate-type=staticlib + # $(RUSTC) native_dep_2.rs --crate-type=staticlib + # $(RUSTC) native_dep_3.rs --crate-type=staticlib + $(NM) $(TMPDIR)/libnative_dep_1.a | $(CGREP) "T native_f1" + $(NM) $(TMPDIR)/libnative_dep_2.a | $(CGREP) "T native_f2" + $(NM) $(TMPDIR)/libnative_dep_3.a | $(CGREP) "T native_f3" + + # Build new rlibs + $(RUSTC) rust_dep_up.rs --crate-type=rlib -Zseparate_native_rlib_dependencies + $(NM) $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "U native_f2" + $(NM) $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "U native_f3" + $(NM) $(TMPDIR)/librust_dep_up.rlib | $(CGREP) -e "T.*rust_dep_up" + ar t $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "libnative_dep_2.a" # FIXME: crossplatform + ar t $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "libnative_dep_3.a" # FIXME: crossplatform + $(RUSTC) rust_dep_local.rs --extern rlib=$(TMPDIR)/librust_dep_up.rlib -Zseparate_native_rlib_dependencies --crate-type=rlib -Z unstable-options + $(NM) $(TMPDIR)/librust_dep_local.rlib | $(CGREP) "U native_f1" + $(NM) $(TMPDIR)/librust_dep_local.rlib | $(CGREP) -e "T.*rust_dep_local" + ar t $(TMPDIR)/librust_dep_local.rlib | $(CGREP) "libnative_dep_1.a" # FIXME: crossplatform + + # Build bin + $(RUSTC) main.rs --extern lib=$(TMPDIR)/librust_dep_local.rlib --crate-type=bin -Zseparate_native_rlib_dependencies --print link-args | $(CGREP) -e 'native_dep_1.*native_dep_2.*native_dep_3' + $(NM) $(TMPDIR)/main | $(CGREP) "T native_f1" + $(NM) $(TMPDIR)/main | $(CGREP) "T native_f2" + $(NM) $(TMPDIR)/main | $(CGREP) "T native_f3" + $(NM) $(TMPDIR)/main | $(CGREP) -e "T.*rust_dep_local" + $(NM) $(TMPDIR)/main | $(CGREP) -e "T.*rust_dep_up" + diff --git a/src/test/run-make/issue_99429/main.rs b/src/test/run-make/issue_99429/main.rs new file mode 100644 index 0000000000000..21b7066dfb676 --- /dev/null +++ b/src/test/run-make/issue_99429/main.rs @@ -0,0 +1,5 @@ +extern crate rust_dep_local; + +pub fn main() { + rust_dep_local::rust_dep_local(); +} diff --git a/src/test/run-make/issue_99429/native_dep_1.c b/src/test/run-make/issue_99429/native_dep_1.c new file mode 100644 index 0000000000000..8df82a26bda80 --- /dev/null +++ b/src/test/run-make/issue_99429/native_dep_1.c @@ -0,0 +1,3 @@ +int native_f1() { + return 1; +} diff --git a/src/test/run-make/issue_99429/native_dep_2.c b/src/test/run-make/issue_99429/native_dep_2.c new file mode 100644 index 0000000000000..929cb1054adc3 --- /dev/null +++ b/src/test/run-make/issue_99429/native_dep_2.c @@ -0,0 +1,3 @@ +int native_f2() { + return 2; +} diff --git a/src/test/run-make/issue_99429/native_dep_3.c b/src/test/run-make/issue_99429/native_dep_3.c new file mode 100644 index 0000000000000..dc7e61bbd5a70 --- /dev/null +++ b/src/test/run-make/issue_99429/native_dep_3.c @@ -0,0 +1,3 @@ +int native_f3() { + return 3; +} diff --git a/src/test/run-make/issue_99429/rust_dep_local.rs b/src/test/run-make/issue_99429/rust_dep_local.rs new file mode 100644 index 0000000000000..8280c7d6c5156 --- /dev/null +++ b/src/test/run-make/issue_99429/rust_dep_local.rs @@ -0,0 +1,13 @@ +#[link(name = "native_dep_1", kind = "static")] +extern "C" { + fn native_f1() -> i32; +} + +extern crate rust_dep_up; + +pub fn rust_dep_local() { + unsafe { + assert!(native_f1() == 1); + } + rust_dep_up::rust_dep_up(); +} diff --git a/src/test/run-make/issue_99429/rust_dep_up.rs b/src/test/run-make/issue_99429/rust_dep_up.rs new file mode 100644 index 0000000000000..edcd7c5212984 --- /dev/null +++ b/src/test/run-make/issue_99429/rust_dep_up.rs @@ -0,0 +1,13 @@ +#[link(name = "native_dep_2", kind = "static")] +#[link(name = "native_dep_3", kind = "static")] +extern "C" { + fn native_f2() -> i32; + fn native_f3() -> i32; +} + +pub fn rust_dep_up() { + unsafe { + assert!(native_f2() == 2); + assert!(native_f3() == 3); + } +} diff --git a/src/test/rustdoc-ui/z-help.stdout b/src/test/rustdoc-ui/z-help.stdout index 86e42440bd0bc..5708f77cfd063 100644 --- a/src/test/rustdoc-ui/z-help.stdout +++ b/src/test/rustdoc-ui/z-help.stdout @@ -140,6 +140,7 @@ `wall-time` (monotonic clock, i.e. `std::time::Instant`) `instructions:u` (retired instructions, userspace-only) `instructions-minus-irqs:u` (subtracting hardware interrupt counts for extra accuracy) + -Z separate-native-rlib-dependencies=val -- change rlib format to store native libraries as archieves -Z share-generics=val -- make the current crate share its generic instantiations -Z show-span=val -- show spans for compiler debugging (expr|pat|ty) -Z span-debug=val -- forward proc_macro::Span's `Debug` impl to `Span`