diff --git a/compiler/rustc_lint/src/reference_casting.rs b/compiler/rustc_lint/src/reference_casting.rs
index 9fed91f72628e..63d55a73a986d 100644
--- a/compiler/rustc_lint/src/reference_casting.rs
+++ b/compiler/rustc_lint/src/reference_casting.rs
@@ -199,6 +199,13 @@ fn is_cast_to_bigger_memory_layout<'tcx>(
     let e_alloc = cx.expr_or_init(e);
     let e_alloc =
         if let ExprKind::AddrOf(_, _, inner_expr) = e_alloc.kind { inner_expr } else { e_alloc };
+
+    // if the current expr looks like this `&mut expr[index]` then just looking
+    // at `expr[index]` won't give us the underlying allocation, so we just skip it
+    if let ExprKind::Index(..) = e_alloc.kind {
+        return None;
+    }
+
     let alloc_ty = cx.typeck_results().node_type(e_alloc.hir_id);
 
     // if we do not find it we bail out, as this may not be UB
diff --git a/tests/ui/lint/reference_casting.rs b/tests/ui/lint/reference_casting.rs
index d6897ab7b141a..3d0c36ca11823 100644
--- a/tests/ui/lint/reference_casting.rs
+++ b/tests/ui/lint/reference_casting.rs
@@ -247,6 +247,14 @@ unsafe fn bigger_layout() {
     unsafe fn from_ref(this: &i32) -> &i64 {
         &*(this as *const i32 as *const i64)
     }
+
+    // https://github.com/rust-lang/rust/issues/124685
+    unsafe fn slice_index(array: &mut [u8], offset: usize) {
+        let a1 = &mut array[offset];
+        let a2 = a1 as *mut u8;
+        let a3 = a2 as *mut u64;
+        unsafe { *a3 = 3 };
+    }
 }
 
 const RAW_PTR: *mut u8 = 1 as *mut u8;