diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 2c5ec9dad59f1..1d2e39119817a 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -378,6 +378,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } + // patchable-function is only implemented on x86 on LLVM + if cx.sess().opts.unstable_opts.hotpatch && cx.sess().target.is_x86() { + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "patchable-function", + "prologue-short-redirect", + )); + } + // FIXME: none of these functions interact with source level attributes. to_add.extend(frame_pointer_type_attr(cx)); to_add.extend(function_return_attr(cx)); diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 44c30d22a9e9c..c866c77495cd0 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -35,6 +35,7 @@ impl OwnedTargetMachine { emit_stack_size_section: bool, relax_elf_relocations: bool, use_init_array: bool, + is_hotpatchable: bool, split_dwarf_file: &CStr, output_obj_file: &CStr, debug_info_compression: &CStr, @@ -67,6 +68,7 @@ impl OwnedTargetMachine { emit_stack_size_section, relax_elf_relocations, use_init_array, + is_hotpatchable, split_dwarf_file.as_ptr(), output_obj_file.as_ptr(), debug_info_compression.as_ptr(), diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index afdd2b581b86e..b2212950b7377 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -225,6 +225,11 @@ pub(crate) fn target_machine_factory( let use_init_array = !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); + // this makes LLVM add a hotpatch flag in the codeview S_COMPILE3 record, + // which is required by linkers for the functionpadmin option + // aarch64 is always hotpatchable + let is_hotpatchable = sess.opts.unstable_opts.hotpatch || sess.target.arch.contains("aarch64"); + let path_mapping = sess.source_map().path_mapping().clone(); let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); @@ -297,6 +302,7 @@ pub(crate) fn target_machine_factory( emit_stack_size_section, relax_elf_relocations, use_init_array, + is_hotpatchable, &split_dwarf_file, &output_obj_file, &debuginfo_compression, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 661debbb9f126..2eb661b0fc2a1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2170,6 +2170,7 @@ unsafe extern "C" { EmitStackSizeSection: bool, RelaxELFRelocations: bool, UseInitArray: bool, + IsHotpatchable: bool, SplitDwarfFile: *const c_char, OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 536ce154cd071..b030271a64245 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -788,6 +788,7 @@ fn test_unstable_options_tracking_hash() { tracked!(fuel, Some(("abc".to_string(), 99))); tracked!(function_return, FunctionReturn::ThunkExtern); tracked!(function_sections, Some(false)); + tracked!(hotpatch, true); tracked!(human_readable_cgu_names, true); tracked!(incremental_ignore_spans, true); tracked!(inline_in_all_cgus, Some(true)); diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 8f0b1b8127657..69fab1a1f58bd 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -396,7 +396,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( bool FunctionSections, bool DataSections, bool UniqueSectionNames, bool TrapUnreachable, bool Singlethread, bool VerboseAsm, bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray, - const char *SplitDwarfFile, const char *OutputObjFile, + bool IsHotpatchable, const char *SplitDwarfFile, const char *OutputObjFile, const char *DebugInfoCompression, bool UseEmulatedTls, const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { @@ -426,6 +426,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( // Always preserve comments that were written by the user Options.MCOptions.PreserveAsmComments = true; Options.MCOptions.ABIName = ABIStr; + Options.Hotpatch = IsHotpatchable; if (SplitDwarfFile) { Options.MCOptions.SplitDwarfFile = SplitDwarfFile; } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index d63276db4938b..e520154fb78cf 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1810,6 +1810,10 @@ options! { "explicitly enable the `cfg(target_thread_local)` directive"), hir_stats: bool = (false, parse_bool, [UNTRACKED], "print some statistics about AST and HIR (default: no)"), + hotpatch: bool = (false, parse_bool, [TRACKED], + "ensures hotpatching is always possible by ensuring that the first instruction of \ + each function is at least two bytes, and no jump within the function goes to the first instruction. \ + Should be combined with link-arg passing -functionpadmin to the linker. Currently only supported for x86 (default: false)"), human_readable_cgu_names: bool = (false, parse_bool, [TRACKED], "generate human-readable, predictable names for codegen units (default: no)"), identify_regions: bool = (false, parse_bool, [UNTRACKED], diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 82e11a3afce32..77b1be581c3fb 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -2070,6 +2070,9 @@ impl Target { Ok(dl) } + pub fn is_x86(&self) -> bool { + ["x86", "x86_64"].contains(&&self.arch[..]) + } } pub trait HasTargetSpec { diff --git a/src/tools/run-make-support/src/external_deps/llvm.rs b/src/tools/run-make-support/src/external_deps/llvm.rs index 38a9ac923b4dc..d644488548251 100644 --- a/src/tools/run-make-support/src/external_deps/llvm.rs +++ b/src/tools/run-make-support/src/external_deps/llvm.rs @@ -254,6 +254,13 @@ impl LlvmFilecheck { self } + /// Specify the prefix (without :) for patterns to match. By default, these patterns are prefixed with "CHECK:". + pub fn check_prefix(&mut self, prefix: &str) -> &mut Self { + self.cmd.arg("--check-prefix"); + self.cmd.arg(prefix); + self + } + /// `--input-file` option. pub fn input_file>(&mut self, input_file: P) -> &mut Self { self.cmd.arg("--input-file"); diff --git a/tests/codegen/hotpatch.rs b/tests/codegen/hotpatch.rs new file mode 100644 index 0000000000000..70ca8b27bff91 --- /dev/null +++ b/tests/codegen/hotpatch.rs @@ -0,0 +1,15 @@ +// check if functions get the attribute, so that LLVM ensures they are hotpatchable +// the attribute is only implemented for x86, aarch64 does not require it + +//@ revisions: x32 x64 +//@[x32] only-x86 +//@[x64] only-x86_64 +//@ compile-flags: -Z hotpatch + +#![crate_type = "lib"] + +#[no_mangle] +pub fn foo() {} + +// CHECK-LABEL: @foo() unnamed_addr #0 +// CHECK: attributes #0 = { {{.*}} "patchable-function"="prologue-short-redirect" {{.*}}} diff --git a/tests/run-make/hotpatch-unaffected/lib.rs b/tests/run-make/hotpatch-unaffected/lib.rs new file mode 100644 index 0000000000000..d441d3e412544 --- /dev/null +++ b/tests/run-make/hotpatch-unaffected/lib.rs @@ -0,0 +1,25 @@ +// hotpatch has two requirements: +// 1. the first instruction of a functin must be at least two bytes long +// 2. there must not be a jump to the first instruction + +// the functions in this file already fulfill the conditions so hotpatch should not affect them + +// -------------------------------------------------------------------------------------------- + +#[no_mangle] +#[inline(never)] +pub fn return_42() -> i32 { + 42 +} + +// -------------------------------------------------------------------------------------------- +// This tailcall does not jump to the first instruction so hotpatch should leave it unaffected + +#[no_mangle] +pub fn tailcall(a: i32) -> i32 { + if a > 10000 { + return a; + } + + if a % 2 == 0 { tailcall(a / 2) } else { tailcall(a * 3 + 1) } +} diff --git a/tests/run-make/hotpatch-unaffected/rmake.rs b/tests/run-make/hotpatch-unaffected/rmake.rs new file mode 100644 index 0000000000000..b50047e529c41 --- /dev/null +++ b/tests/run-make/hotpatch-unaffected/rmake.rs @@ -0,0 +1,48 @@ +// Check if hotpatch leaves the functions that are already hotpatchable untouched + +use run_make_support::{assertion_helpers, llvm, rustc}; + +fn main() { + // hotpatch is only implemented for X86 and aarch64 + #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] + { + fn base_rustc() -> rustc::Rustc { + let mut rustc = rustc(); + rustc.input("lib.rs").crate_type("lib").opt_level("3"); + rustc + } + + fn dump_lib(libname: &str) -> String { + llvm::llvm_objdump() + .arg("--disassemble-symbols=return_42,tailcall") + .input(libname) + .run() + .stdout_utf8() + } + + base_rustc().crate_name("regular").run(); + let regular_dump = dump_lib("libregular.rlib"); + + base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run(); + let hotpatch_dump = dump_lib("libhotpatch.rlib"); + + { + let mut lines_regular = regular_dump.lines(); + let mut lines_hotpatch = hotpatch_dump.lines(); + + loop { + match (lines_regular.next(), lines_hotpatch.next()) { + (None, None) => break, + (Some(r), Some(h)) => { + if r.contains("libregular.rlib") { + assertion_helpers::assert_contains(h, "libhotpatch.rlib") + } else { + assertion_helpers::assert_equals(&r, &h) + } + } + _ => panic!("expected files to have equal number of lines"), + } + } + } + } +} diff --git a/tests/run-make/hotpatch/lib.rs b/tests/run-make/hotpatch/lib.rs new file mode 100644 index 0000000000000..627f035fbb66d --- /dev/null +++ b/tests/run-make/hotpatch/lib.rs @@ -0,0 +1,27 @@ +// hotpatch has two requirements: +// 1) the first instruction of a functin must be at least two bytes long +// 2) there must not be a jump to the first instruction + +// The LLVM attribute we use '"patchable-function", "prologue-short-redirect"' only ensures 1) +// However in practice 2) rarely matters. Its rare that it occurs and the problems it caused can be +// avoided by the hotpatch tool. +// In this test we check if 1) is ensured by inserted nops as needed + +// ---------------------------------------------------------------------------------------------- + +// empty_fn just returns. Note that 'ret' is a single byte instruction, but hotpatch requires +// a two or more byte instructions to be at the start of the functions. +// Preferably we would also tests a different single byte instruction, +// but I was not able to find an example with another one byte intstruction. + +// check that if the first instruction is just a single byte, so our test is valid +// CHECK-LABEL: : +// CHECK-NOT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}} + +// check that the first instruction is at least 2 bytes long +// HOTPATCH-LABEL: : +// HOTPATCH-NEXT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}} + +#[no_mangle] +#[inline(never)] +pub fn empty_fn() {} diff --git a/tests/run-make/hotpatch/rmake.rs b/tests/run-make/hotpatch/rmake.rs new file mode 100644 index 0000000000000..4f43aaa985143 --- /dev/null +++ b/tests/run-make/hotpatch/rmake.rs @@ -0,0 +1,42 @@ +// Check if hotpatch makes the functions hotpachable that were not +// More details in lib.rs + +use run_make_support::{llvm, rustc}; + +fn main() { + // hotpatch is only implemented for x86 and aarch64, but for aarch64 functions + // are always hotpatchable so we don't need to check it + #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] + { + fn base_rustc() -> rustc::Rustc { + let mut rustc = rustc(); + rustc.input("lib.rs").crate_type("lib").opt_level("3"); + rustc + } + + fn dump_lib(libname: &str) -> String { + llvm::llvm_objdump() + .arg("--disassemble-symbols=empty_fn") + .input(libname) + .run() + .stdout_utf8() + } + + { + base_rustc().crate_name("regular").run(); + let regular_dump = dump_lib("libregular.rlib"); + llvm::llvm_filecheck().patterns("lib.rs").stdin_buf(regular_dump).run(); + } + + { + base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run(); + let hotpatch_dump = dump_lib("libhotpatch.rlib"); + + llvm::llvm_filecheck() + .patterns("lib.rs") + .check_prefix("HOTPATCH") + .stdin_buf(hotpatch_dump) + .run(); + } + } +} diff --git a/tests/run-make/hotpatch_pdb/main.rs b/tests/run-make/hotpatch_pdb/main.rs new file mode 100644 index 0000000000000..248ccf37d6115 --- /dev/null +++ b/tests/run-make/hotpatch_pdb/main.rs @@ -0,0 +1,6 @@ +// CHECK: S_OBJNAME{{.*}}hotpatch_pdb{{.*}}.o +// CHECK: S_COMPILE3 +// CHECK-NOT: S_ +// CHECK: flags = {{.*}}hot patchable + +pub fn main() {} diff --git a/tests/run-make/hotpatch_pdb/rmake.rs b/tests/run-make/hotpatch_pdb/rmake.rs new file mode 100644 index 0000000000000..800245f87cdc8 --- /dev/null +++ b/tests/run-make/hotpatch_pdb/rmake.rs @@ -0,0 +1,30 @@ +// Check if hotpatch flag is present in the Codeview. +// This is need so linkers actually pad functions when given the functionpadmin arg. + +use run_make_support::{llvm, rustc}; + +fn main() { + // PDBs are windows only and hotpatch is only implemented for x86 and aarch64 + #[cfg(all( + target_os = "windows", + any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64") + ))] + { + let output = rustc() + .input("main.rs") + .arg("-g") + .arg("-Zhotpatch") + .crate_name("hotpatch_pdb") + .crate_type("bin") + .run(); + + let pdbutil_output = llvm::llvm_pdbutil() + .arg("dump") + .arg("-symbols") + .input("hotpatch_pdb.pdb") + .run() + .stdout_utf8(); + + llvm::llvm_filecheck().patterns("main.rs").stdin_buf(&pdbutil_output).run(); + } +}