diff --git a/.gitignore b/.gitignore index afb1c120d73d..27d56d49c279 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ link.tar.gz .direnv .envrc /tests/tmp +.codex diff --git a/lib/cli/src/backend.rs b/lib/cli/src/backend.rs index bd548d64207e..5902bd707072 100644 --- a/lib/cli/src/backend.rs +++ b/lib/cli/src/backend.rs @@ -218,6 +218,12 @@ pub struct RuntimeOptions { #[clap(long = "enable-nan-canonicalization")] enable_nan_canonicalization: bool, + /// Allow unaligned memory accesses. + /// + /// Available for Cranelift and Singlepass (RISC-V). + #[clap(long = "enable-unaligned-memory-accesses")] + enable_unaligned_memory_accesses: bool, + #[clap(flatten)] features: WasmFeatures, } @@ -461,6 +467,9 @@ impl RuntimeOptions { #[cfg(feature = "singlepass")] BackendType::Singlepass => { let mut config = wasmer_compiler_singlepass::Singlepass::new(); + if self.enable_unaligned_memory_accesses { + config.allow_unaligned_memory_accesses(true); + } if self.enable_verifier { config.enable_verifier(); } @@ -486,6 +495,9 @@ impl RuntimeOptions { #[cfg(feature = "cranelift")] BackendType::Cranelift => { let mut config = wasmer_compiler_cranelift::Cranelift::new(); + if self.enable_unaligned_memory_accesses { + config.allow_unaligned_memory_accesses(true); + } if self.enable_verifier { config.enable_verifier(); } @@ -605,6 +617,9 @@ impl BackendType { #[cfg(feature = "singlepass")] Self::Singlepass => { let mut config = wasmer_compiler_singlepass::Singlepass::new(); + if runtime_opts.enable_unaligned_memory_accesses { + config.allow_unaligned_memory_accesses(true); + } let supported_features = config.supported_features_for_target(target); if runtime_opts.enable_verifier { config.enable_verifier(); @@ -636,6 +651,9 @@ impl BackendType { #[cfg(feature = "cranelift")] Self::Cranelift => { let mut config = wasmer_compiler_cranelift::Cranelift::new(); + if runtime_opts.enable_unaligned_memory_accesses { + config.allow_unaligned_memory_accesses(true); + } let supported_features = config.supported_features_for_target(target); if runtime_opts.enable_verifier { config.enable_verifier(); diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index 359d3f50fe83..613600a47e27 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -321,7 +321,7 @@ impl CraneliftCompiler { let mut custom_sections = PrimaryMap::new(); #[cfg(not(feature = "rayon"))] - let mut func_translator = FuncTranslator::new(); + let mut func_translator = FuncTranslator::new(self.config.allow_unaligned_memory_accesses); #[cfg(not(feature = "rayon"))] let results = function_body_inputs .iter() @@ -351,7 +351,7 @@ impl CraneliftCompiler { translate_function_buckets( &pool, - FuncTranslator::new, + || FuncTranslator::new(self.config.allow_unaligned_memory_accesses), |func_translator, i, input| compile_function(func_translator, i, input), progress.clone(), &buckets, diff --git a/lib/compiler-cranelift/src/config.rs b/lib/compiler-cranelift/src/config.rs index 2977c3a435e5..e290382818b0 100644 --- a/lib/compiler-cranelift/src/config.rs +++ b/lib/compiler-cranelift/src/config.rs @@ -106,6 +106,7 @@ pub enum CraneliftOptLevel { #[derive(Debug, Clone)] pub struct Cranelift { enable_nan_canonicalization: bool, + pub(crate) allow_unaligned_memory_accesses: bool, enable_verifier: bool, pub(crate) enable_perfmap: bool, enable_pic: bool, @@ -123,6 +124,7 @@ impl Cranelift { pub fn new() -> Self { Self { enable_nan_canonicalization: false, + allow_unaligned_memory_accesses: false, enable_verifier: false, opt_level: CraneliftOptLevel::Speed, enable_pic: false, @@ -142,6 +144,14 @@ impl Cranelift { self } + /// Enable run-time handling of potentially unaligned memory accesses. + /// Unaligned memory accesses occur when you try to read N bytes of data starting + /// from an address that is not evenly divisible by N. + pub fn allow_unaligned_memory_accesses(&mut self, enable: bool) -> &mut Self { + self.allow_unaligned_memory_accesses = enable; + self + } + /// Set the number of threads to use for compilation. pub fn num_threads(&mut self, num_threads: NonZero) -> &mut Self { self.num_threads = num_threads; @@ -286,6 +296,10 @@ impl CompilerConfig for Cranelift { self.enable_perfmap = true; } + fn enable_unaligned_memory_accesses(&mut self) { + self.allow_unaligned_memory_accesses = true; + } + fn canonicalize_nans(&mut self, enable: bool) { self.enable_nan_canonicalization = enable; } diff --git a/lib/compiler-cranelift/src/translator/code_translator.rs b/lib/compiler-cranelift/src/translator/code_translator.rs index 18a91732cf7c..dc8ac881cb99 100644 --- a/lib/compiler-cranelift/src/translator/code_translator.rs +++ b/lib/compiler-cranelift/src/translator/code_translator.rs @@ -101,7 +101,7 @@ use wasmer_compiler::wasmparser::{self, Catch, MemArg, Operator}; use wasmer_compiler::{ModuleTranslationState, from_binaryreadererror_wasmerror, wasm_unsupported}; use wasmer_types::{ CATCH_ALL_TAG_VALUE, FunctionIndex, GlobalIndex, MemoryIndex, SignatureIndex, TableIndex, - TagIndex, WasmResult, + TagIndex, WasmError, WasmResult, }; /// Given a `Reachability`, unwrap the inner `T` or, when unreachable, set @@ -132,6 +132,7 @@ pub fn translate_operator( builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FuncEnvironment<'_>, + allow_unaligned_memory_accesses: bool, ) -> WasmResult<()> { if !state.reachable { translate_unreachable_operator(module_translation_state, op, builder, state, environ)?; @@ -765,91 +766,211 @@ pub fn translate_operator( Operator::I32Load8U { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Uload8, I32, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Uload8, + I32, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I32Load16U { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Uload16, I32, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Uload16, + I32, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I32Load8S { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Sload8, I32, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Sload8, + I32, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I32Load16S { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Sload16, I32, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Sload16, + I32, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I64Load8U { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Uload8, I64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Uload8, + I64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I64Load16U { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Uload16, I64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Uload16, + I64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I64Load8S { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Sload8, I64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Sload8, + I64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I64Load16S { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Sload16, I64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Sload16, + I64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I64Load32S { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Sload32, I64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Sload32, + I64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I64Load32U { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Uload32, I64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Uload32, + I64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I32Load { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Load, I32, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Load, + I32, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::F32Load { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Load, F32, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Load, + F32, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::I64Load { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Load, I64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Load, + I64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::F64Load { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Load, F64, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Load, + F64, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::V128Load { memarg } => { unwrap_or_return_unreachable_state!( state, - translate_load(memarg, ir::Opcode::Load, I8X16, builder, state, environ)? + translate_load( + memarg, + ir::Opcode::Load, + I8X16, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )? ); } Operator::V128Load8x8S { memarg } => { @@ -909,19 +1030,54 @@ pub fn translate_operator( | Operator::I64Store { memarg } | Operator::F32Store { memarg } | Operator::F64Store { memarg } => { - translate_store(memarg, ir::Opcode::Store, builder, state, environ)?; + translate_store( + memarg, + ir::Opcode::Store, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )?; } Operator::I32Store8 { memarg } | Operator::I64Store8 { memarg } => { - translate_store(memarg, ir::Opcode::Istore8, builder, state, environ)?; + translate_store( + memarg, + ir::Opcode::Istore8, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )?; } Operator::I32Store16 { memarg } | Operator::I64Store16 { memarg } => { - translate_store(memarg, ir::Opcode::Istore16, builder, state, environ)?; + translate_store( + memarg, + ir::Opcode::Istore16, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )?; } Operator::I64Store32 { memarg } => { - translate_store(memarg, ir::Opcode::Istore32, builder, state, environ)?; + translate_store( + memarg, + ir::Opcode::Istore32, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )?; } Operator::V128Store { memarg } => { - translate_store(memarg, ir::Opcode::Store, builder, state, environ)?; + translate_store( + memarg, + ir::Opcode::Store, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )?; } /****************************** Nullary Operators ************************************/ Operator::I32Const { value } => { @@ -1640,6 +1796,7 @@ pub fn translate_operator( builder, state, environ, + allow_unaligned_memory_accesses, )? ); let splatted = builder.ins().splat(type_of(op), state.pop1()); @@ -1655,6 +1812,7 @@ pub fn translate_operator( builder, state, environ, + allow_unaligned_memory_accesses, )? ); let as_vector = builder.ins().scalar_to_vector(type_of(op), state.pop1()); @@ -1674,6 +1832,7 @@ pub fn translate_operator( builder, state, environ, + allow_unaligned_memory_accesses, )? ); let replacement = state.pop1(); @@ -1685,7 +1844,14 @@ pub fn translate_operator( | Operator::V128Store64Lane { memarg, lane } => { let vector = pop1_with_bitcast(state, type_of(op), builder); state.push1(builder.ins().extractlane(vector, *lane)); - translate_store(memarg, ir::Opcode::Store, builder, state, environ)?; + translate_store( + memarg, + ir::Opcode::Store, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )?; } Operator::I8x16ExtractLaneS { lane } | Operator::I16x8ExtractLaneS { lane } => { let vector = pop1_with_bitcast(state, type_of(op), builder); @@ -2854,6 +3020,7 @@ fn translate_load( builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FuncEnvironment<'_>, + allow_unaligned_memory_accesses: bool, ) -> WasmResult> { let mem_op_size = mem_op_size(opcode, result_ty); let (flags, _wasm_index, base) = @@ -2862,10 +3029,72 @@ fn translate_load( Reachability::Reachable((f, i, b)) => (f, i, b), }; - let (load, dfg) = builder - .ins() - .Load(opcode, result_ty, flags, Offset32::new(0), base); - state.push1(dfg.first_result(load)); + // TODO: maybe support also v128 + if allow_unaligned_memory_accesses && mem_op_size > 1 && mem_op_size < 16 { + // Test and handle aligned / unaligned loads separately + let block_aligned = builder.create_block(); + let block_unaligned = builder.create_block(); + let block_merge = builder.create_block(); + builder.append_block_param(block_merge, result_ty); + + let alignment_check = builder.ins().band_imm(base, (mem_op_size - 1) as i64); + builder + .ins() + .brif(alignment_check, block_unaligned, &[], block_aligned, &[]); + + builder.seal_block(block_aligned); + builder.seal_block(block_unaligned); + + builder.switch_to_block(block_aligned); + let (fast_load, fast_dfg) = + builder + .ins() + .Load(opcode, result_ty, flags, Offset32::new(0), base); + let fast_val = fast_dfg.first_result(fast_load); + builder.ins().jump(block_merge, &[fast_val.into()]); + + builder.switch_to_block(block_unaligned); + + // We're going to build the final value as an unsigned integer type that will be later bitcasted. + let result_uint_type = Type::int_with_byte_size(u16::try_from(result_ty.bytes()).unwrap()) + .ok_or(WasmError::Generic( + "cannot get uint type for memory load".to_string(), + ))?; + let raw_uint_type = Type::int_with_byte_size(u16::from(mem_op_size)).ok_or( + WasmError::Generic("cannot get uint type for memory load".to_string()), + )?; + let mut slow_val = builder.ins().uload8(result_uint_type, flags, base, 0); + for i in 1..mem_op_size { + let byte = builder + .ins() + .uload8(result_uint_type, flags, base, i as i32); + let shifted = builder.ins().ishl_imm(byte, (i * 8) as i64); + slow_val = builder.ins().bor(slow_val, shifted); + } + if matches!( + opcode, + ir::Opcode::Sload8 | ir::Opcode::Sload16 | ir::Opcode::Sload32 + ) { + let narrow = builder.ins().ireduce(raw_uint_type, slow_val); + slow_val = builder.ins().sextend(result_uint_type, narrow); + } + let slow_val = builder.ins().bitcast( + result_ty, + MemFlags::new().with_endianness(ir::Endianness::Little), + slow_val, + ); + builder.ins().jump(block_merge, &[slow_val.into()]); + + builder.seal_block(block_merge); + builder.switch_to_block(block_merge); + state.push1(builder.block_params(block_merge)[0]); + } else { + let (load, dfg) = builder + .ins() + .Load(opcode, result_ty, flags, Offset32::new(0), base); + state.push1(dfg.first_result(load)); + } + Ok(Reachability::Reachable(())) } @@ -2876,6 +3105,7 @@ fn translate_store( builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FuncEnvironment<'_>, + allow_unaligned_memory_accesses: bool, ) -> WasmResult<()> { let val = state.pop1(); let val_ty = builder.func.dfg.value_type(val); @@ -2886,9 +3116,54 @@ fn translate_store( prepare_addr(memarg, mem_op_size, builder, state, environ)? ); - builder - .ins() - .Store(opcode, val_ty, flags, Offset32::new(0), val, base); + if allow_unaligned_memory_accesses && mem_op_size > 1 && mem_op_size < 16 { + let block_aligned = builder.create_block(); + let block_unaligned = builder.create_block(); + let block_merge = builder.create_block(); + + let alignment_check = builder.ins().band_imm(base, (mem_op_size - 1) as i64); + builder + .ins() + .brif(alignment_check, block_unaligned, &[], block_aligned, &[]); + + builder.seal_block(block_aligned); + builder.seal_block(block_unaligned); + + builder.switch_to_block(block_aligned); + builder + .ins() + .Store(opcode, val_ty, flags, Offset32::new(0), val, base); + builder.ins().jump(block_merge, &[]); + + builder.switch_to_block(block_unaligned); + let val = if val_ty.is_int() { + val + } else { + let result_uint_type = Type::int_with_byte_size(u16::from(mem_op_size)).ok_or( + WasmError::Generic(format!( + "cannot get uint type of size {mem_op_size} bytes for memory store from {val_ty:?}", + )), + )?; + builder.ins().bitcast( + result_uint_type, + MemFlags::new().with_endianness(ir::Endianness::Little), + val, + ) + }; + for i in 0..mem_op_size { + let shifted = builder.ins().ushr_imm(val, (i * 8) as i64); + builder.ins().istore8(flags, shifted, base, i as i32); + } + builder.ins().jump(block_merge, &[]); + + builder.seal_block(block_merge); + builder.switch_to_block(block_merge); + } else { + builder + .ins() + .Store(opcode, val_ty, flags, Offset32::new(0), val, base); + } + Ok(()) } diff --git a/lib/compiler-cranelift/src/translator/func_translator.rs b/lib/compiler-cranelift/src/translator/func_translator.rs index c68d3f5342f3..e299d577c888 100644 --- a/lib/compiler-cranelift/src/translator/func_translator.rs +++ b/lib/compiler-cranelift/src/translator/func_translator.rs @@ -31,16 +31,18 @@ use wasmer_types::{LocalFunctionIndex, WasmResult}; pub struct FuncTranslator { func_ctx: FunctionBuilderContext, state: FuncTranslationState, + allow_unaligned_memory_accesses: bool, } impl wasmer_compiler::FuncTranslator for FuncTranslator {} impl FuncTranslator { /// Create a new translator. - pub fn new() -> Self { + pub fn new(allow_unaligned_memory_accesses: bool) -> Self { Self { func_ctx: FunctionBuilderContext::new(), state: FuncTranslationState::new(), + allow_unaligned_memory_accesses, } } @@ -119,6 +121,7 @@ impl FuncTranslator { &mut builder, &mut self.state, environ, + self.allow_unaligned_memory_accesses, )?; builder.finalize(); @@ -239,6 +242,7 @@ fn parse_function_body( builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FuncEnvironment<'_>, + allow_unaligned_memory_accesses: bool, ) -> WasmResult<()> { // The control stack is initialized with a single block representing the whole function. debug_assert_eq!(state.control_stack.len(), 1, "State not initialized"); @@ -247,7 +251,14 @@ fn parse_function_body( while !state.control_stack.is_empty() { builder.set_srcloc(cur_srcloc(reader)); let op = reader.read_operator()?; - translate_operator(module_translation_state, &op, builder, state, environ)?; + translate_operator( + module_translation_state, + &op, + builder, + state, + environ, + allow_unaligned_memory_accesses, + )?; } // The final `End` operator left us in the exit block where we need to manually add a return diff --git a/lib/compiler-singlepass/src/compiler.rs b/lib/compiler-singlepass/src/compiler.rs index 9e8e012d4c81..2b76f098bbd3 100644 --- a/lib/compiler-singlepass/src/compiler.rs +++ b/lib/compiler-singlepass/src/compiler.rs @@ -217,7 +217,10 @@ impl SinglepassCompiler { generator.finalize(input, arch) } Architecture::Riscv64(_) => { - let machine = MachineRiscv::new(Some(target.clone()))?; + let machine = MachineRiscv::new( + Some(target.clone()), + self.config.allow_unaligned_memory_accesses, + )?; let mut generator = FuncGen::new( module, &self.config, diff --git a/lib/compiler-singlepass/src/config.rs b/lib/compiler-singlepass/src/config.rs index 1f4b2fd65fe1..284a36b35ad1 100644 --- a/lib/compiler-singlepass/src/config.rs +++ b/lib/compiler-singlepass/src/config.rs @@ -76,6 +76,7 @@ impl SinglepassCallbacks { #[derive(Debug, Clone)] pub struct Singlepass { pub(crate) enable_nan_canonicalization: bool, + pub(crate) allow_unaligned_memory_accesses: bool, /// The middleware chain. pub(crate) middlewares: Vec>, @@ -92,6 +93,7 @@ impl Singlepass { pub fn new() -> Self { Self { enable_nan_canonicalization: true, + allow_unaligned_memory_accesses: false, middlewares: vec![], callbacks: None, num_threads: std::thread::available_parallelism().unwrap_or(NonZero::new(1).unwrap()), @@ -103,6 +105,14 @@ impl Singlepass { self } + /// Enable run-time handling of potentially unaligned memory accesses. + /// Unaligned memory accesses occur when you try to read N bytes of data starting + /// from an address that is not evenly divisible by N. + pub fn allow_unaligned_memory_accesses(&mut self, enable: bool) -> &mut Self { + self.allow_unaligned_memory_accesses = enable; + self + } + /// Callbacks that will triggered in the different compilation /// phases in Singlepass. pub fn callbacks(&mut self, callbacks: Option) -> &mut Self { diff --git a/lib/compiler-singlepass/src/machine.rs b/lib/compiler-singlepass/src/machine.rs index 215516d479c3..37b00b9e6bf8 100644 --- a/lib/compiler-singlepass/src/machine.rs +++ b/lib/compiler-singlepass/src/machine.rs @@ -2368,7 +2368,7 @@ pub fn gen_std_trampoline( machine.gen_std_trampoline(sig, calling_convention) } Architecture::Riscv64(_) => { - let machine = MachineRiscv::new(Some(target.clone()))?; + let machine = MachineRiscv::new(Some(target.clone()), false)?; machine.gen_std_trampoline(sig, calling_convention) } _ => Err(CompileError::UnsupportedTarget( @@ -2394,7 +2394,7 @@ pub fn gen_std_dynamic_import_trampoline( machine.gen_std_dynamic_import_trampoline(vmoffsets, sig, calling_convention) } Architecture::Riscv64(_) => { - let machine = MachineRiscv::new(Some(target.clone()))?; + let machine = MachineRiscv::new(Some(target.clone()), false)?; machine.gen_std_dynamic_import_trampoline(vmoffsets, sig, calling_convention) } _ => Err(CompileError::UnsupportedTarget( @@ -2420,7 +2420,7 @@ pub fn gen_import_call_trampoline( machine.gen_import_call_trampoline(vmoffsets, index, sig, calling_convention) } Architecture::Riscv64(_) => { - let machine = MachineRiscv::new(Some(target.clone()))?; + let machine = MachineRiscv::new(Some(target.clone()), false)?; machine.gen_import_call_trampoline(vmoffsets, index, sig, calling_convention) } _ => Err(CompileError::UnsupportedTarget( diff --git a/lib/compiler-singlepass/src/machine_riscv.rs b/lib/compiler-singlepass/src/machine_riscv.rs index 6d81840b2529..068ce6b1bd5f 100644 --- a/lib/compiler-singlepass/src/machine_riscv.rs +++ b/lib/compiler-singlepass/src/machine_riscv.rs @@ -74,6 +74,7 @@ impl DerefMut for AssemblerRiscv { /// The RISC-V machine state and code emitter. pub struct MachineRiscv { assembler: AssemblerRiscv, + allow_unaligned_memory_accesses: bool, used_gprs: FixedBitSet, used_fprs: FixedBitSet, trap_table: TrapTable, @@ -90,10 +91,14 @@ const SCRATCH_REG: GPR = GPR::X28; impl MachineRiscv { /// Creates a new RISC-V machine for code generation. - pub fn new(target: Option) -> Result { + pub fn new( + target: Option, + allow_unaligned_memory_accesses: bool, + ) -> Result { // TODO: for now always require FPU Ok(MachineRiscv { assembler: AssemblerRiscv::new(0, target)?, + allow_unaligned_memory_accesses, used_gprs: FixedBitSet::with_capacity(32), used_fprs: FixedBitSet::with_capacity(32), trap_table: TrapTable::default(), @@ -1105,6 +1110,10 @@ impl MachineRiscv { dst: Location, src: GPR, ) -> Result<(), CompileError> { + if !self.allow_unaligned_memory_accesses { + return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0)); + } + if let Size::S8 = sz { return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0)); } @@ -1213,6 +1222,10 @@ impl MachineRiscv { src: Location, dst: GPR, ) -> Result<(), CompileError> { + if !self.allow_unaligned_memory_accesses { + return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0)); + } + if let Size::S8 = sz { // `emit_relaxed_store` uses wrong order of src and dst. // The `src` parameter of `emit_relaxed_store` actually stores diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 0f5f8b8a8f5a..2e4abf07b1d3 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -54,6 +54,10 @@ pub trait CompilerConfig { /// (but are not 100% SPEC compliant). fn enable_non_volatile_memops(&mut self) {} + /// Enable run-time handling of potentially unaligned memory accesses. + /// Available for Cranelift and Singlepass (RISC-V). + fn enable_unaligned_memory_accesses(&mut self) {} + /// Enables treating eligible funcref tables as read-only so the backend can /// place them in read-only data. fn enable_readonly_funcref_table(&mut self) {} diff --git a/tests/compilers/config.rs b/tests/compilers/config.rs index 4d7f9b0bf629..64915c95898d 100644 --- a/tests/compilers/config.rs +++ b/tests/compilers/config.rs @@ -19,6 +19,7 @@ pub struct Config { pub features: Option, pub middlewares: Vec>, pub canonicalize_nans: bool, + pub allow_unaligned_memory_accesses: bool, } impl Config { @@ -27,6 +28,7 @@ impl Config { compiler, features: None, canonicalize_nans: false, + allow_unaligned_memory_accesses: false, middlewares: vec![], } } @@ -43,8 +45,13 @@ impl Config { self.canonicalize_nans = canonicalize_nans; } + pub fn set_allow_unaligned_memory_accesses(&mut self, enable: bool) { + self.allow_unaligned_memory_accesses = enable; + } + pub fn store(&self) -> Store { - let compiler_config = self.compiler_config(self.canonicalize_nans); + let compiler_config = + self.compiler_config(self.canonicalize_nans, self.allow_unaligned_memory_accesses); let engine = self.engine(compiler_config); Store::new(engine) } @@ -69,6 +76,7 @@ impl Config { pub fn compiler_config( &self, #[allow(unused_variables)] canonicalize_nans: bool, + #[allow(unused_variables)] allow_unaligned_memory_accesses: bool, ) -> Box { let debug_dir = std::env::var("WASMER_COMPILER_DEBUG_DIR") .ok() @@ -81,6 +89,7 @@ impl Config { let mut compiler = wasmer_compiler_cranelift::Cranelift::new(); compiler.canonicalize_nans(canonicalize_nans); + compiler.allow_unaligned_memory_accesses(allow_unaligned_memory_accesses); compiler.enable_verifier(); if let Some(mut debug_dir) = debug_dir { debug_dir.push("cranelift"); @@ -112,6 +121,7 @@ impl Config { Compiler::Singlepass => { let mut compiler = wasmer_compiler_singlepass::Singlepass::new(); compiler.canonicalize_nans(canonicalize_nans); + compiler.allow_unaligned_memory_accesses(allow_unaligned_memory_accesses); compiler.enable_verifier(); if let Some(mut debug_dir) = debug_dir { use wasmer_compiler_singlepass::SinglepassCallbacks; diff --git a/tests/compilers/wast.rs b/tests/compilers/wast.rs index 43ce38a68e35..d804cbc58741 100644 --- a/tests/compilers/wast.rs +++ b/tests/compilers/wast.rs @@ -34,6 +34,7 @@ pub fn run_wast(mut config: crate::Config, wast_path: &str) -> anyhow::Result<() || wast_path.ends_with("throw_ref.wast") || wast_path.ends_with("imports.wast"); let is_wide_arithmetic = wast_path.contains("wide-arithmetic"); + let is_unaligned_memory = wast_path.contains("unaligned-memory"); let is_tail_call = wast_path.contains("return_call") || wast_path.ends_with("try_table.wast"); @@ -60,6 +61,9 @@ pub fn run_wast(mut config: crate::Config, wast_path: &str) -> anyhow::Result<() } config.set_features(features); config.set_nan_canonicalization(try_nan_canonicalization); + if is_unaligned_memory { + config.set_allow_unaligned_memory_accesses(true); + } let store = config.store(); let mut wast = Wast::new_with_spectest(store); diff --git a/tests/wast/wasmer/unaligned-memory-access.wast b/tests/wast/wasmer/unaligned-memory-access.wast new file mode 100644 index 000000000000..5d5d9b21830e --- /dev/null +++ b/tests/wast/wasmer/unaligned-memory-access.wast @@ -0,0 +1,64 @@ +(module + (memory 1) + + (func (export "unaligned-i32-load") (result i32) + (i32.store (i32.const 0x1) (i32.const 0x78563412)) + (i32.load (i32.const 0x1)) + ) + + (func (export "unaligned-i64-load") (result i64) + (i64.store (i32.const 0x1) (i64.const 0x0123456789abcdef)) + (i64.load (i32.const 0x1)) + ) + + (func (export "unaligned-f32-load-bits") (result i32) + (f32.store (i32.const 0x1) (f32.reinterpret_i32 (i32.const 0x41200000))) + (i32.reinterpret_f32 (f32.load (i32.const 0x1))) + ) + + (func (export "unaligned-f64-load-bits") (result i64) + (f64.store (i32.const 0x1) (f64.reinterpret_i64 (i64.const 0x4028cccccccccccd))) + (i64.reinterpret_f64 (f64.load (i32.const 0x1))) + ) + + (func (export "unaligned-i32-load16-u") (result i32) + (i32.store16 (i32.const 0x1) (i32.const 0xff80)) + (i32.load16_u (i32.const 0x1)) + ) + + (func (export "unaligned-i32-load16-s") (result i32) + (i32.store16 (i32.const 0x1) (i32.const 0xff80)) + (i32.load16_s (i32.const 0x1)) + ) + + (func (export "unaligned-i64-load16-u") (result i64) + (i64.store16 (i32.const 0x1) (i64.const 0xff80)) + (i64.load16_u (i32.const 0x1)) + ) + + (func (export "unaligned-i64-load16-s") (result i64) + (i64.store16 (i32.const 0x1) (i64.const 0xff80)) + (i64.load16_s (i32.const 0x1)) + ) + + (func (export "unaligned-i64-load32-u") (result i64) + (i64.store32 (i32.const 0x1) (i64.const 0x89abcdef)) + (i64.load32_u (i32.const 0x1)) + ) + + (func (export "unaligned-i64-load32-s") (result i64) + (i64.store32 (i32.const 0x1) (i64.const 0x89abcdef)) + (i64.load32_s (i32.const 0x1)) + ) +) + +(assert_return (invoke "unaligned-i32-load") (i32.const 0x78563412)) +(assert_return (invoke "unaligned-i64-load") (i64.const 0x0123456789abcdef)) +(assert_return (invoke "unaligned-f32-load-bits") (i32.const 0x41200000)) +(assert_return (invoke "unaligned-f64-load-bits") (i64.const 0x4028cccccccccccd)) +(assert_return (invoke "unaligned-i32-load16-u") (i32.const 65408)) +(assert_return (invoke "unaligned-i32-load16-s") (i32.const -128)) +(assert_return (invoke "unaligned-i64-load16-u") (i64.const 65408)) +(assert_return (invoke "unaligned-i64-load16-s") (i64.const -128)) +(assert_return (invoke "unaligned-i64-load32-u") (i64.const 2309737967)) +(assert_return (invoke "unaligned-i64-load32-s") (i64.const -1985229329))