Add x86 specific hotpatch flag to rustc

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -378,6 +378,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
         to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile"));
     }
 
+    // patchable-function is only implemented on x86 on LLVM
+    if cx.sess().opts.unstable_opts.hotpatch && cx.sess().target.is_x86() {
+        to_add.push(llvm::CreateAttrStringValue(
+            cx.llcx,
+            "patchable-function",
+            "prologue-short-redirect",
+        ));
+    }
+
     // FIXME: none of these functions interact with source level attributes.
     to_add.extend(frame_pointer_type_attr(cx));
     to_add.extend(function_return_attr(cx));

compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs

@@ -35,6 +35,7 @@ impl OwnedTargetMachine {
         emit_stack_size_section: bool,
         relax_elf_relocations: bool,
         use_init_array: bool,
+        use_hotpatch: bool,
         split_dwarf_file: &CStr,
         output_obj_file: &CStr,
         debug_info_compression: &CStr,
@@ -67,6 +68,7 @@ impl OwnedTargetMachine {
                 emit_stack_size_section,
                 relax_elf_relocations,
                 use_init_array,
+                use_hotpatch,
                 split_dwarf_file.as_ptr(),
                 output_obj_file.as_ptr(),
                 debug_info_compression.as_ptr(),

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -225,6 +225,11 @@ pub(crate) fn target_machine_factory(
     let use_init_array =
         !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
 
+    // this makes LLVM add a hotpatch flag in the codeview S_COMPILE3 record,
+    // which is required by linkers for the functionpadmin option
+    // aarch64 is always hotpatchable
+    let use_hotpatch = sess.opts.unstable_opts.hotpatch || sess.target.arch.contains("aarch64");
+
     let path_mapping = sess.source_map().path_mapping().clone();
 
     let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
@@ -297,6 +302,7 @@ pub(crate) fn target_machine_factory(
             emit_stack_size_section,
             relax_elf_relocations,
             use_init_array,
+            use_hotpatch,
             &split_dwarf_file,
             &output_obj_file,
             &debuginfo_compression,

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2170,6 +2170,7 @@ unsafe extern "C" {
         EmitStackSizeSection: bool,
         RelaxELFRelocations: bool,
         UseInitArray: bool,
+        UseHotpatch: bool,
         SplitDwarfFile: *const c_char,
         OutputObjFile: *const c_char,
         DebugInfoCompression: *const c_char,

compiler/rustc_interface/src/tests.rs

@@ -788,6 +788,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(fuel, Some(("abc".to_string(), 99)));
     tracked!(function_return, FunctionReturn::ThunkExtern);
     tracked!(function_sections, Some(false));
+    tracked!(hotpatch, true);
     tracked!(human_readable_cgu_names, true);
     tracked!(incremental_ignore_spans, true);
     tracked!(inline_in_all_cgus, Some(true));

compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

@@ -396,7 +396,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     bool FunctionSections, bool DataSections, bool UniqueSectionNames,
     bool TrapUnreachable, bool Singlethread, bool VerboseAsm,
     bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray,
-    const char *SplitDwarfFile, const char *OutputObjFile,
+    bool UseHotpatch, const char *SplitDwarfFile, const char *OutputObjFile,
     const char *DebugInfoCompression, bool UseEmulatedTls,
     const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
 
@@ -426,6 +426,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   // Always preserve comments that were written by the user
   Options.MCOptions.PreserveAsmComments = true;
   Options.MCOptions.ABIName = ABIStr;
+  Options.Hotpatch = UseHotpatch;
   if (SplitDwarfFile) {
     Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
   }

compiler/rustc_session/src/options.rs

@@ -1810,6 +1810,10 @@ options! {
         "explicitly enable the `cfg(target_thread_local)` directive"),
     hir_stats: bool = (false, parse_bool, [UNTRACKED],
         "print some statistics about AST and HIR (default: no)"),
+    hotpatch: bool = (false, parse_bool, [TRACKED],
+        "ensures hotpatching is always possible by ensuring that the first instruction of \
+        each function is at least two bytes, and no jump within the function goes to the first instruction. \
+        Should be combined with link-arg passing -functionpadmin to the linker. Currently only supported for x86 (default: false)"),
     human_readable_cgu_names: bool = (false, parse_bool, [TRACKED],
         "generate human-readable, predictable names for codegen units (default: no)"),
     identify_regions: bool = (false, parse_bool, [UNTRACKED],

compiler/rustc_target/src/spec/mod.rs

@@ -2070,6 +2070,9 @@ impl Target {
 
         Ok(dl)
     }
+    pub fn is_x86(&self) -> bool {
+        ["x86", "x86_64"].contains(&&self.arch[..])
+    }
 }
 
 pub trait HasTargetSpec {

src/tools/run-make-support/src/external_deps/llvm.rs

@@ -254,6 +254,13 @@ impl LlvmFilecheck {
         self
     }
 
+    /// Specify the prefix (without :) for patterns to match. By default, these patterns are prefixed with "CHECK:".
+    pub fn check_prefix(&mut self, prefix: &str) -> &mut Self {
+        self.cmd.arg("--check-prefix");
+        self.cmd.arg(prefix);
+        self
+    }
+
     /// `--input-file` option.
     pub fn input_file<P: AsRef<Path>>(&mut self, input_file: P) -> &mut Self {
         self.cmd.arg("--input-file");

tests/codegen/hotpatch.rs

@@ -0,0 +1,12 @@
+//@ revisions: x32 x64
+//@[x32] only-x86
+//@[x64] only-x86_64
+//@ compile-flags: -Z hotpatch
+
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn foo() {}
+
+// CHECK: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}} "patchable-function"="prologue-short-redirect" {{.*}}}

tests/run-make/hotpatch-unaffected/lib.rs

@@ -0,0 +1,25 @@
+// hotpatch has two requirements:
+// 1. the first instruction of a functin must be at least two bytes long
+// 2. there must not be a jump to the first instruction
+
+// the functions in this file already fulfill the conditions so hotpatch should not affect them
+
+// --------------------------------------------------------------------------------------------
+
+#[no_mangle]
+#[inline(never)]
+pub fn return_42() -> i32 {
+    42
+}
+
+// --------------------------------------------------------------------------------------------
+// This tailcall does not jump to the first instruction so hotpatch should leave it unaffected
+
+#[no_mangle]
+pub fn tailcall(a: i32) -> i32 {
+    if a > 10000 {
+        return a;
+    }
+
+    if a % 2 == 0 { tailcall(a / 2) } else { tailcall(a * 3 + 1) }
+}

tests/run-make/hotpatch-unaffected/rmake.rs

@@ -0,0 +1,51 @@
+// Check if hotpatch leaves the functions that are already hotpatchable untouched
+
+//@ revisions: x32 x64 aarch64
+//@[x32] only-x86
+//@[x64] only-x86_64
+//@[aarch64] only-aarch64
+
+// Reason: hotpatch is only implemented for X86 and aarch64
+
+use run_make_support::{assertion_helpers, llvm, rustc};
+
+fn main() {
+    fn base_rustc() -> rustc::Rustc {
+        let mut rustc = rustc();
+        rustc.input("lib.rs").crate_type("lib").opt_level("3");
+        rustc
+    }
+
+    fn dump_lib(libname: &str) -> String {
+        llvm::llvm_objdump()
+            .arg("--disassemble-symbols=return_42,tailcall")
+            .input(libname)
+            .run()
+            .stdout_utf8()
+    }
+
+    base_rustc().crate_name("regular").run();
+    let regular_dump = dump_lib("libregular.rlib");
+
+    base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run();
+    let hotpatch_dump = dump_lib("libhotpatch.rlib");
+
+    {
+        let mut lines_regular = regular_dump.lines();
+        let mut lines_hotpatch = hotpatch_dump.lines();
+
+        loop {
+            match (lines_regular.next(), lines_hotpatch.next()) {
+                (None, None) => break,
+                (Some(r), Some(h)) => {
+                    if r.contains("libregular.rlib") {
+                        assertion_helpers::assert_contains(h, "libhotpatch.rlib")
+                    } else {
+                        assertion_helpers::assert_equals(&r, &h)
+                    }
+                }
+                _ => panic!("expected files to have equal number of lines"),
+            }
+        }
+    }
+}

tests/run-make/hotpatch/lib.rs

@@ -0,0 +1,27 @@
+// hotpatch has two requirements:
+// 1) the first instruction of a functin must be at least two bytes long
+// 2) there must not be a jump to the first instruction
+
+// The LLVM attribute we use '"patchable-function", "prologue-short-redirect"' only ensures 1)
+// However in practice 2) rarely matters. Its rare that it occurs and the problems it caused can be
+// avoided by the hotpatch tool.
+// In this test we check if 1) is ensured by inserted nops as needed
+
+// ----------------------------------------------------------------------------------------------
+
+// empty_fn just returns. Note that 'ret' is a single byte instruction, but hotpatch requires
+// a two or more byte instructions to be at the start of the functions.
+// Preferably we would also tests a different single byte instruction,
+// but I was not able to find an example with another one byte intstruction.
+
+// check that if the first instruction is just a single byte, so our test is valid
+// CHECK-LABEL: <empty_fn>:
+// CHECK-NOT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}}
+
+// check that the first instruction is at least 2 bytes long
+// HOTPATCH-LABEL: <empty_fn>:
+// HOTPATCH-NEXT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}}
+
+#[no_mangle]
+#[inline(never)]
+pub fn empty_fn() {}

tests/run-make/hotpatch/rmake.rs

@@ -0,0 +1,44 @@
+// Check if hotpatch makes the functions hotpachable that were not
+// More details in lib.rs
+
+//@ revisions: x32 x64
+//@[x32] only-x86
+//@[x64] only-x86_64
+
+// Reason: hotpatch is only implemented for x86 and aarch64, but for aarch64 they
+// are always hotpatchable so we don't need to check it
+
+use run_make_support::{llvm, rustc};
+
+fn main() {
+    fn base_rustc() -> rustc::Rustc {
+        let mut rustc = rustc();
+        rustc.input("lib.rs").crate_type("lib").opt_level("3");
+        rustc
+    }
+
+    fn dump_lib(libname: &str) -> String {
+        llvm::llvm_objdump()
+            .arg("--disassemble-symbols=empty_fn")
+            .input(libname)
+            .run()
+            .stdout_utf8()
+    }
+
+    {
+        base_rustc().crate_name("regular").run();
+        let regular_dump = dump_lib("libregular.rlib");
+        llvm::llvm_filecheck().patterns("lib.rs").stdin_buf(regular_dump).run();
+    }
+
+    {
+        base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run();
+        let hotpatch_dump = dump_lib("libhotpatch.rlib");
+
+        llvm::llvm_filecheck()
+            .patterns("lib.rs")
+            .check_prefix("HOTPATCH")
+            .stdin_buf(hotpatch_dump)
+            .run();
+    }
+}

tests/run-make/hotpatch_pdb/main.rs

@@ -0,0 +1,6 @@
+// CHECK: S_OBJNAME{{.*}}hotpatch_pdb{{.*}}.o
+// CHECK: S_COMPILE3
+// CHECK-NOT: S_
+// CHECK: flags = {{.*}}hot patchable
+
+pub fn main() {}

tests/run-make/hotpatch_pdb/rmake.rs

@@ -0,0 +1,30 @@
+// Check if hotpatch flag is present in the Codeview.
+// This is need so linkers actually pad functions when given the functionpadmin arg.
+
+//@ revisions: x32 x64 aarch64
+//@[x32] only-x86
+//@[x64] only-x86_64
+//@[aarch64] only-aarch64
+
+// Reason: Hotpatch is only implemented for x86 and aarch64
+
+use run_make_support::{llvm, rustc};
+
+fn main() {
+    let output = rustc()
+        .input("main.rs")
+        .arg("-g")
+        .arg("-Zhotpatch")
+        .crate_name("hotpatch_pdb")
+        .crate_type("bin")
+        .run();
+
+    let pdbutil_output = llvm::llvm_pdbutil()
+        .arg("dump")
+        .arg("-symbols")
+        .input("hotpatch_pdb.pdb")
+        .run()
+        .stdout_utf8();
+
+    llvm::llvm_filecheck().patterns("main.rs").stdin_buf(&pdbutil_output).run();
+}