Add -Zsanitize=kernel-hwaddress

compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs

@@ -587,6 +587,7 @@ impl<S: Stage> SingleAttributeParser<S> for SanitizeParser {
         r#"kernel_address = "on|off""#,
         r#"cfi = "on|off""#,
         r#"hwaddress = "on|off""#,
+        r#"kernel_hwaddress = "on|off""#,
         r#"kcfi = "on|off""#,
         r#"memory = "on|off""#,
         r#"memtag = "on|off""#,
@@ -654,7 +655,9 @@ impl<S: Stage> SingleAttributeParser<S> for SanitizeParser {
                 Some(sym::memtag) => apply(SanitizerSet::MEMTAG),
                 Some(sym::shadow_call_stack) => apply(SanitizerSet::SHADOWCALLSTACK),
                 Some(sym::thread) => apply(SanitizerSet::THREAD),
-                Some(sym::hwaddress) => apply(SanitizerSet::HWADDRESS),
+                Some(sym::hwaddress) | Some(sym::kernel_hwaddress) => {
+                    apply(SanitizerSet::HWADDRESS | SanitizerSet::KERNELHWADDRESS)
+                }
                 Some(sym::realtime) => match value.value_as_str() {
                     Some(sym::nonblocking) => rtsan = Some(RtsanSetting::Nonblocking),
                     Some(sym::blocking) => rtsan = Some(RtsanSetting::Blocking),
@@ -679,6 +682,7 @@ impl<S: Stage> SingleAttributeParser<S> for SanitizeParser {
                             sym::shadow_call_stack,
                             sym::thread,
                             sym::hwaddress,
+                            sym::kernel_hwaddress,
                             sym::realtime,
                         ],
                     );

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -120,7 +120,8 @@ pub(crate) fn sanitize_attrs<'ll, 'tcx>(
     if enabled.contains(SanitizerSet::THREAD) {
         attrs.push(llvm::AttributeKind::SanitizeThread.create_attr(cx.llcx));
     }
-    if enabled.contains(SanitizerSet::HWADDRESS) {
+    if enabled.contains(SanitizerSet::HWADDRESS) || enabled.contains(SanitizerSet::KERNELHWADDRESS)
+    {
         attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx));
     }
     if enabled.contains(SanitizerSet::SHADOWCALLSTACK) {

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -652,6 +652,10 @@ pub(crate) unsafe fn llvm_optimize(
             sanitize_kernel_address_recover: config
                 .sanitizer_recover
                 .contains(SanitizerSet::KERNELADDRESS),
+            sanitize_kernel_hwaddress: config.sanitizer.contains(SanitizerSet::KERNELHWADDRESS),
+            sanitize_kernel_hwaddress_recover: config
+                .sanitizer_recover
+                .contains(SanitizerSet::KERNELHWADDRESS),
         })
     } else {
         None

compiler/rustc_codegen_llvm/src/base.rs

@@ -210,10 +210,14 @@ pub(crate) fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility {
 }
 
 pub(crate) fn set_variable_sanitizer_attrs(llval: &Value, attrs: &CodegenFnAttrs) {
-    if attrs.sanitizers.disabled.contains(SanitizerSet::ADDRESS) {
+    if attrs.sanitizers.disabled.contains(SanitizerSet::ADDRESS)
+        || attrs.sanitizers.disabled.contains(SanitizerSet::KERNELADDRESS)
+    {
         unsafe { llvm::LLVMRustSetNoSanitizeAddress(llval) };
     }
-    if attrs.sanitizers.disabled.contains(SanitizerSet::HWADDRESS) {
+    if attrs.sanitizers.disabled.contains(SanitizerSet::HWADDRESS)
+        || attrs.sanitizers.disabled.contains(SanitizerSet::KERNELHWADDRESS)
+    {
         unsafe { llvm::LLVMRustSetNoSanitizeHWAddress(llval) };
     }
 }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -464,6 +464,8 @@ pub(crate) struct SanitizerOptions {
     pub sanitize_hwaddress_recover: bool,
     pub sanitize_kernel_address: bool,
     pub sanitize_kernel_address_recover: bool,
+    pub sanitize_kernel_hwaddress: bool,
+    pub sanitize_kernel_hwaddress_recover: bool,
 }
 
 /// LLVMRustRelocModel

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -1341,6 +1341,7 @@ fn add_sanitizer_libraries(
     if sanitizer.contains(SanitizerSet::LEAK)
         && !sanitizer.contains(SanitizerSet::ADDRESS)
         && !sanitizer.contains(SanitizerSet::HWADDRESS)
+        && !sanitizer.contains(SanitizerSet::KERNELHWADDRESS)
     {
         link_sanitizer_runtime(sess, flavor, linker, "lsan");
     }

compiler/rustc_feature/src/builtin_attrs.rs

@@ -773,7 +773,7 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         DuplicatesOk, EncodeCrossCrate::No, effective_target_features, experimental!(force_target_feature)
     ),
     gated!(
-        sanitize, Normal, template!(List: &[r#"address = "on|off""#, r#"kernel_address = "on|off""#, r#"cfi = "on|off""#, r#"hwaddress = "on|off""#, r#"kcfi = "on|off""#, r#"memory = "on|off""#, r#"memtag = "on|off""#, r#"shadow_call_stack = "on|off""#, r#"thread = "on|off""#]), ErrorPreceding,
+        sanitize, Normal, template!(List: &[r#"address = "on|off""#, r#"kernel_address = "on|off""#, r#"cfi = "on|off""#, r#"hwaddress = "on|off""#, r#"kernel_hwaddress = "on|off""#, r#"kcfi = "on|off""#, r#"memory = "on|off""#, r#"memtag = "on|off""#, r#"shadow_call_stack = "on|off""#, r#"thread = "on|off""#]), ErrorPreceding,
         EncodeCrossCrate::No, sanitize, experimental!(sanitize),
     ),
     gated!(

compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

@@ -540,6 +540,8 @@ struct LLVMRustSanitizerOptions {
   bool SanitizeHWAddressRecover;
   bool SanitizeKernelAddress;
   bool SanitizeKernelAddressRecover;
+  bool SanitizeKernelHWAddress;
+  bool SanitizeKernelHWAddressRecover;
 };
 
 extern "C" typedef void (*registerEnzymeAndPassPipelineFn)(
@@ -767,13 +769,15 @@ extern "C" LLVMRustResult LLVMRustOptimize(
                                  !TM->getTargetTriple().isOSWindows()));
       });
     }
-    if (SanitizerOptions->SanitizeHWAddress) {
+    if (SanitizerOptions->SanitizeHWAddress ||
+        SanitizerOptions->SanitizeKernelHWAddress) {
       OptimizerLastEPCallbacks.push_back(
           [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level,
                              ThinOrFullLTOPhase phase) {
             HWAddressSanitizerOptions opts(
-                /*CompileKernel=*/false,
-                SanitizerOptions->SanitizeHWAddressRecover,
+                SanitizerOptions->SanitizeKernelHWAddress,
+                SanitizerOptions->SanitizeHWAddressRecover ||
+                    SanitizerOptions->SanitizeKernelHWAddressRecover,
                 /*DisableOptimization=*/false);
             MPM.addPass(HWAddressSanitizerPass(opts));
           });

compiler/rustc_session/src/config/cfg.rs

@@ -229,6 +229,10 @@ pub(crate) fn default_configuration(sess: &Session) -> Cfg {
         if s == SanitizerSet::KERNELADDRESS {
             s = SanitizerSet::ADDRESS;
         }
+        // KHWASAN is still HWASAN under the hood, so it uses the same attribute.
+        if s == SanitizerSet::KERNELHWADDRESS {
+            s = SanitizerSet::HWADDRESS;
+        }
         ins_str!(sym::sanitize, &s.to_string());
     }
 

compiler/rustc_session/src/options.rs

@@ -106,6 +106,7 @@ mod target_modifier_consistency_check {
             | SanitizerSet::SHADOWCALLSTACK
             | SanitizerSet::KCFI
             | SanitizerSet::KERNELADDRESS
+            | SanitizerSet::KERNELHWADDRESS
             | SanitizerSet::SAFESTACK
             | SanitizerSet::DATAFLOW;
 
@@ -810,7 +811,7 @@ mod desc {
     pub(crate) const parse_patchable_function_entry: &str = "either two comma separated integers (total_nops,prefix_nops), with prefix_nops <= total_nops, or one integer (total_nops)";
     pub(crate) const parse_opt_panic_strategy: &str = parse_panic_strategy;
     pub(crate) const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
-    pub(crate) const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `dataflow`, `hwaddress`, `kcfi`, `kernel-address`, `leak`, `memory`, `memtag`, `safestack`, `shadow-call-stack`, `thread`, or 'realtime'";
+    pub(crate) const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `dataflow`, `hwaddress`, `kcfi`, `kernel-address`, `kernel-hwaddress`, `leak`, `memory`, `memtag`, `safestack`, `shadow-call-stack`, `thread`, or 'realtime'";
     pub(crate) const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2";
     pub(crate) const parse_cfguard: &str =
         "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
@@ -1252,6 +1253,7 @@ pub mod parse {
                     "dataflow" => SanitizerSet::DATAFLOW,
                     "kcfi" => SanitizerSet::KCFI,
                     "kernel-address" => SanitizerSet::KERNELADDRESS,
+                    "kernel-hwaddress" => SanitizerSet::KERNELHWADDRESS,
                     "leak" => SanitizerSet::LEAK,
                     "memory" => SanitizerSet::MEMORY,
                     "memtag" => SanitizerSet::MEMTAG,

compiler/rustc_session/src/session.rs

@@ -527,9 +527,12 @@ impl Session {
     pub fn emit_lifetime_markers(&self) -> bool {
         self.opts.optimize != config::OptLevel::No
         // AddressSanitizer and KernelAddressSanitizer uses lifetimes to detect use after scope bugs.
+        //
         // MemorySanitizer uses lifetimes to detect use of uninitialized stack variables.
-        // HWAddressSanitizer will use lifetimes to detect use after scope bugs in the future.
-        || self.sanitizers().intersects(SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS | SanitizerSet::MEMORY | SanitizerSet::HWADDRESS)
+        //
+        // HWAddressSanitizer and KernelHWAddressSanitizer will use lifetimes to detect use after
+        // scope bugs in the future.
+        || self.sanitizers().intersects(SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS | SanitizerSet::MEMORY | SanitizerSet::HWADDRESS | SanitizerSet::KERNELHWADDRESS)
     }
 
     pub fn diagnostic_width(&self) -> usize {

compiler/rustc_span/src/symbol.rs

@@ -1115,6 +1115,7 @@ symbols! {
         iterator_collect_fn,
         kcfi,
         kernel_address,
+        kernel_hwaddress,
         keylocker_x86,
         keyword,
         kind,

compiler/rustc_target/src/spec/mod.rs

@@ -1175,9 +1175,10 @@ bitflags::bitflags! {
         const SHADOWCALLSTACK = 1 << 7;
         const KCFI    = 1 << 8;
         const KERNELADDRESS = 1 << 9;
-        const SAFESTACK = 1 << 10;
-        const DATAFLOW = 1 << 11;
-        const REALTIME = 1 << 12;
+        const KERNELHWADDRESS = 1 << 10;
+        const SAFESTACK = 1 << 11;
+        const DATAFLOW = 1 << 12;
+        const REALTIME = 1 << 13;
     }
 }
 rustc_data_structures::external_bitflags_debug! { SanitizerSet }
@@ -1191,24 +1192,32 @@ impl SanitizerSet {
         (SanitizerSet::ADDRESS, SanitizerSet::HWADDRESS),
         (SanitizerSet::ADDRESS, SanitizerSet::MEMTAG),
         (SanitizerSet::ADDRESS, SanitizerSet::KERNELADDRESS),
+        (SanitizerSet::ADDRESS, SanitizerSet::KERNELHWADDRESS),
         (SanitizerSet::ADDRESS, SanitizerSet::SAFESTACK),
         (SanitizerSet::LEAK, SanitizerSet::MEMORY),
         (SanitizerSet::LEAK, SanitizerSet::THREAD),
         (SanitizerSet::LEAK, SanitizerSet::KERNELADDRESS),
+        (SanitizerSet::LEAK, SanitizerSet::KERNELHWADDRESS),
         (SanitizerSet::LEAK, SanitizerSet::SAFESTACK),
         (SanitizerSet::MEMORY, SanitizerSet::THREAD),
         (SanitizerSet::MEMORY, SanitizerSet::HWADDRESS),
         (SanitizerSet::MEMORY, SanitizerSet::KERNELADDRESS),
+        (SanitizerSet::MEMORY, SanitizerSet::KERNELHWADDRESS),
         (SanitizerSet::MEMORY, SanitizerSet::SAFESTACK),
         (SanitizerSet::THREAD, SanitizerSet::HWADDRESS),
         (SanitizerSet::THREAD, SanitizerSet::KERNELADDRESS),
+        (SanitizerSet::THREAD, SanitizerSet::KERNELHWADDRESS),
         (SanitizerSet::THREAD, SanitizerSet::SAFESTACK),
         (SanitizerSet::HWADDRESS, SanitizerSet::MEMTAG),
         (SanitizerSet::HWADDRESS, SanitizerSet::KERNELADDRESS),
+        (SanitizerSet::HWADDRESS, SanitizerSet::KERNELHWADDRESS),
         (SanitizerSet::HWADDRESS, SanitizerSet::SAFESTACK),
         (SanitizerSet::CFI, SanitizerSet::KCFI),
         (SanitizerSet::MEMTAG, SanitizerSet::KERNELADDRESS),
+        (SanitizerSet::MEMTAG, SanitizerSet::KERNELHWADDRESS),
+        (SanitizerSet::KERNELADDRESS, SanitizerSet::KERNELHWADDRESS),
         (SanitizerSet::KERNELADDRESS, SanitizerSet::SAFESTACK),
+        (SanitizerSet::KERNELHWADDRESS, SanitizerSet::SAFESTACK),
     ];
 
     /// Return sanitizer's name
@@ -1221,6 +1230,7 @@ impl SanitizerSet {
             SanitizerSet::DATAFLOW => "dataflow",
             SanitizerSet::KCFI => "kcfi",
             SanitizerSet::KERNELADDRESS => "kernel-address",
+            SanitizerSet::KERNELHWADDRESS => "kernel-hwaddress",
             SanitizerSet::LEAK => "leak",
             SanitizerSet::MEMORY => "memory",
             SanitizerSet::MEMTAG => "memtag",
@@ -1266,6 +1276,7 @@ impl FromStr for SanitizerSet {
             "dataflow" => SanitizerSet::DATAFLOW,
             "kcfi" => SanitizerSet::KCFI,
             "kernel-address" => SanitizerSet::KERNELADDRESS,
+            "kernel-hwaddress" => SanitizerSet::KERNELHWADDRESS,
             "leak" => SanitizerSet::LEAK,
             "memory" => SanitizerSet::MEMORY,
             "memtag" => SanitizerSet::MEMTAG,

compiler/rustc_target/src/spec/targets/aarch64_be_unknown_none_softfloat.rs

@@ -22,7 +22,9 @@ pub(crate) fn target() -> Target {
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),
-        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
+        supported_sanitizers: SanitizerSet::KCFI
+            | SanitizerSet::KERNELADDRESS
+            | SanitizerSet::KERNELHWADDRESS,
         stack_probes: StackProbeType::Inline,
         panic_strategy: PanicStrategy::Abort,
         endian: Endian::Big,

compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs

@@ -21,7 +21,9 @@ pub(crate) fn target() -> Target {
             &["--fix-cortex-a53-843419"],
         ),
         features: "+v8a,+strict-align,+neon".into(),
-        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
+        supported_sanitizers: SanitizerSet::KCFI
+            | SanitizerSet::KERNELADDRESS
+            | SanitizerSet::KERNELHWADDRESS,
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),

compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs

@@ -21,7 +21,9 @@ pub(crate) fn target() -> Target {
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),
-        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
+        supported_sanitizers: SanitizerSet::KCFI
+            | SanitizerSet::KERNELADDRESS
+            | SanitizerSet::KERNELHWADDRESS,
         stack_probes: StackProbeType::Inline,
         panic_strategy: PanicStrategy::Abort,
         default_uwtable: true,

compiler/rustc_target/src/spec/targets/aarch64_unknown_nuttx.rs

@@ -21,7 +21,9 @@ pub(crate) fn target() -> Target {
             &["--fix-cortex-a53-843419"],
         ),
         features: "+v8a,+strict-align,+neon".into(),
-        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
+        supported_sanitizers: SanitizerSet::KCFI
+            | SanitizerSet::KERNELADDRESS
+            | SanitizerSet::KERNELHWADDRESS,
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),

compiler/rustc_target/src/spec/targets/aarch64v8r_unknown_none.rs

@@ -8,7 +8,9 @@ pub(crate) fn target() -> Target {
         // based off the aarch64-unknown-none target at time of addition
         linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
         linker: Some("rust-lld".into()),
-        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
+        supported_sanitizers: SanitizerSet::KCFI
+            | SanitizerSet::KERNELADDRESS
+            | SanitizerSet::KERNELHWADDRESS,
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),

compiler/rustc_target/src/spec/targets/aarch64v8r_unknown_none_softfloat.rs

@@ -12,7 +12,9 @@ pub(crate) fn target() -> Target {
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),
-        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
+        supported_sanitizers: SanitizerSet::KCFI
+            | SanitizerSet::KERNELADDRESS
+            | SanitizerSet::KERNELHWADDRESS,
         stack_probes: StackProbeType::Inline,
         panic_strategy: PanicStrategy::Abort,
         default_uwtable: true,

src/tools/compiletest/src/common.rs

@@ -168,6 +168,7 @@ pub enum Sanitizer {
     Dataflow,
     Kcfi,
     KernelAddress,
+    KernelHwaddress,
     Leak,
     Memory,
     Memtag,

src/tools/compiletest/src/directives/directive_names.rs

@@ -179,6 +179,7 @@ pub(crate) const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "needs-sanitizer-hwaddress",
     "needs-sanitizer-kasan",
     "needs-sanitizer-kcfi",
+    "needs-sanitizer-khwasan",
     "needs-sanitizer-leak",
     "needs-sanitizer-memory",
     "needs-sanitizer-memtag",

src/tools/compiletest/src/directives/needs.rs

@@ -46,6 +46,11 @@ pub(super) fn handle_needs(
             condition: cache.sanitizer_kasan,
             ignore_reason: "ignored on targets without kernel address sanitizer",
         },
+        Need {
+            name: "needs-sanitizer-khwasan",
+            condition: cache.sanitizer_khwasan,
+            ignore_reason: "ignored on targets without kernel hardware-assisted address sanitizer",
+        },
         Need {
             name: "needs-sanitizer-leak",
             condition: cache.sanitizer_leak,
@@ -332,6 +337,7 @@ pub(super) struct CachedNeedsConditions {
     sanitizer_dataflow: bool,
     sanitizer_kcfi: bool,
     sanitizer_kasan: bool,
+    sanitizer_khwasan: bool,
     sanitizer_leak: bool,
     sanitizer_memory: bool,
     sanitizer_thread: bool,
@@ -359,6 +365,7 @@ impl CachedNeedsConditions {
             sanitizer_dataflow: sanitizers.contains(&Sanitizer::Dataflow),
             sanitizer_kcfi: sanitizers.contains(&Sanitizer::Kcfi),
             sanitizer_kasan: sanitizers.contains(&Sanitizer::KernelAddress),
+            sanitizer_khwasan: sanitizers.contains(&Sanitizer::KernelHwaddress),
             sanitizer_leak: sanitizers.contains(&Sanitizer::Leak),
             sanitizer_memory: sanitizers.contains(&Sanitizer::Memory),
             sanitizer_thread: sanitizers.contains(&Sanitizer::Thread),

tests/assembly-llvm/sanitizer/hwasan-vs-khwasan.rs

@@ -0,0 +1,31 @@
+// Verifies that HWASAN and KHWASAN emit different assembly instrumentation on AArch64.
+//
+//@ add-minicore
+//@ assembly-output: emit-asm
+//@ revisions: hwasan khwasan
+//@[hwasan] compile-flags: --target aarch64-unknown-linux-gnu -Zsanitizer=hwaddress
+//@[hwasan] needs-llvm-components: aarch64
+//@[khwasan] compile-flags: --target aarch64-unknown-none -Zsanitizer=kernel-hwaddress
+//@[khwasan] needs-llvm-components: aarch64
+//@ compile-flags: -Copt-level=1
+
+#![crate_type = "lib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+extern crate minicore;
+
+// hwasan-LABEL: test:
+// hwasan:       adrp    x{{[0-9]+}}, :gottprel:__hwasan_tls
+// hwasan:       mrs     x{{[0-9]+}}, TPIDR_EL0
+// hwasan:       bl      __hwasan_check_x0_0_short_v2
+
+// khwasan-LABEL: test:
+// khwasan-NOT:   __hwasan_tls
+// khwasan:       orr     x{{[0-9]+}}, x0, #0xff00000000000000
+// khwasan:       bl      __hwasan_check_x0_67043328_fixed_0_short_v2
+
+#[no_mangle]
+pub fn test(b: &mut u8) -> u8 {
+    *b
+}