Auto merge of rust-lang#138157 - scottmcm:inline-more-tiny-things, r=<try>

Allow more top-down inlining for single-BB callees

This means that things like `<usize as Step>::forward_unchecked` and `<PartialOrd for f32>::le` will inline even if
we've already done a bunch of inlining to find the calls to them.

Fixes rust-lang#138136

Draft as it's built atop rust-lang#138135, which adds a mir-opt test that's a nice demonstration of this.

Author: bors

compiler/rustc_mir_transform/src/inline.rs

@@ -26,6 +26,7 @@ use crate::{check_inline, util};
 
 pub(crate) mod cycle;
 
+const HISTORY_DEPTH_LIMIT: usize = 20;
 const TOP_DOWN_DEPTH_LIMIT: usize = 5;
 
 #[derive(Clone, Debug)]
@@ -128,10 +129,6 @@ trait Inliner<'tcx> {
         callee_attrs: &CodegenFnAttrs,
     ) -> Result<(), &'static str>;
 
-    // How many callsites in a body are we allowed to inline? We need to limit this in order
-    // to prevent super-linear growth in MIR size.
-    fn inline_limit_for_block(&self) -> Option<usize>;
-
     /// Called when inlining succeeds.
     fn on_inline_success(
         &mut self,
@@ -142,9 +139,6 @@ trait Inliner<'tcx> {
 
     /// Called when inlining failed or was not performed.
     fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str);
-
-    /// Called when the inline limit for a body is reached.
-    fn on_inline_limit_reached(&self) -> bool;
 }
 
 struct ForceInliner<'tcx> {
@@ -224,10 +218,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
         }
     }
 
-    fn inline_limit_for_block(&self) -> Option<usize> {
-        Some(usize::MAX)
-    }
-
     fn on_inline_success(
         &mut self,
         callsite: &CallSite<'tcx>,
@@ -261,10 +251,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
             justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
         });
     }
-
-    fn on_inline_limit_reached(&self) -> bool {
-        false
-    }
 }
 
 struct NormalInliner<'tcx> {
@@ -278,6 +264,10 @@ struct NormalInliner<'tcx> {
     /// The number of `DefId`s is finite, so checking history is enough
     /// to ensure that we do not loop endlessly while inlining.
     history: Vec<DefId>,
+    /// How many (multi-call) callsites have we inlined for the top-level call?
+    ///
+    /// We need to limit this in order to prevent super-linear growth in MIR size.
+    top_down_counter: usize,
     /// Indicates that the caller body has been modified.
     changed: bool,
     /// Indicates that the caller is #[inline] and just calls another function,
@@ -295,6 +285,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
             typing_env,
             def_id,
             history: Vec::new(),
+            top_down_counter: 0,
             changed: false,
             caller_is_inline_forwarder: matches!(
                 codegen_fn_attrs.inline,
@@ -351,6 +342,15 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
             return Err("body has errors");
         }
 
+        if callee_body.basic_blocks.len() > 1 {
+            if self.history.len() > HISTORY_DEPTH_LIMIT {
+                return Err("Too many blocks for deep history");
+            }
+            if self.top_down_counter > TOP_DOWN_DEPTH_LIMIT {
+                return Err("Too many blocks for more top-down inlining");
+            }
+        }
+
         let mut threshold = if self.caller_is_inline_forwarder {
             tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
         } else if tcx.cross_crate_inlinable(callsite.callee.def_id()) {
@@ -431,14 +431,6 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
         }
     }
 
-    fn inline_limit_for_block(&self) -> Option<usize> {
-        match self.history.len() {
-            0 => Some(usize::MAX),
-            1..=TOP_DOWN_DEPTH_LIMIT => Some(1),
-            _ => None,
-        }
-    }
-
     fn on_inline_success(
         &mut self,
         callsite: &CallSite<'tcx>,
@@ -447,13 +439,26 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
     ) {
         self.changed = true;
 
+        let new_calls_count = new_blocks
+            .clone()
+            .filter(|&bb| {
+                matches!(
+                    caller_body.basic_blocks[bb].terminator().kind,
+                    TerminatorKind::Call { .. },
+                )
+            })
+            .count();
+        if new_calls_count > 1 {
+            self.top_down_counter += 1;
+        }
+
         self.history.push(callsite.callee.def_id());
         process_blocks(self, caller_body, new_blocks);
         self.history.pop();
-    }
 
-    fn on_inline_limit_reached(&self) -> bool {
-        true
+        if self.history.is_empty() {
+            self.top_down_counter = 0;
+        }
     }
 
     fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {}
@@ -482,8 +487,6 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>(
     caller_body: &mut Body<'tcx>,
     blocks: Range<BasicBlock>,
 ) {
-    let Some(inline_limit) = inliner.inline_limit_for_block() else { return };
-    let mut inlined_count = 0;
     for bb in blocks {
         let bb_data = &caller_body[bb];
         if bb_data.is_cleanup {
@@ -505,13 +508,6 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>(
             Ok(new_blocks) => {
                 debug!("inlined {}", callsite.callee);
                 inliner.on_inline_success(&callsite, caller_body, new_blocks);
-
-                inlined_count += 1;
-                if inlined_count == inline_limit {
-                    if inliner.on_inline_limit_reached() {
-                        return;
-                    }
-                }
             }
         }
     }

library/core/src/cmp.rs

@@ -29,6 +29,7 @@ mod bytewise;
 pub(crate) use bytewise::BytewiseEq;
 
 use self::Ordering::*;
+use crate::ops::ControlFlow::{self, Break, Continue};
 
 /// Trait for comparisons using the equality operator.
 ///
@@ -1446,6 +1447,54 @@ pub macro PartialOrd($item:item) {
     /* compiler built-in */
 }
 
+/// Helpers for chaining together field PartialOrds into the full type's ordering.
+///
+/// If the two values are equal, returns `ControlFlow::Continue`.
+/// If the two values are not equal, returns `ControlFlow::Break(self OP other)`.
+///
+/// This allows simple types like `i32` and `f64` to just emit two comparisons
+/// directly, instead of needing to optimize the 3-way comparison.
+///
+/// Currently this is done using specialization, but it doesn't need that:
+/// it could be provided methods on `PartialOrd` instead and work fine.
+pub(crate) trait SpecChainingPartialOrd<Rhs>: PartialOrd<Rhs> {
+    fn spec_chain_lt(&self, other: &Rhs) -> ControlFlow<bool>;
+    fn spec_chain_le(&self, other: &Rhs) -> ControlFlow<bool>;
+    fn spec_chain_gt(&self, other: &Rhs) -> ControlFlow<bool>;
+    fn spec_chain_ge(&self, other: &Rhs) -> ControlFlow<bool>;
+}
+
+impl<T: PartialOrd<U>, U> SpecChainingPartialOrd<U> for T {
+    #[inline]
+    default fn spec_chain_lt(&self, other: &U) -> ControlFlow<bool> {
+        match PartialOrd::partial_cmp(self, other) {
+            Some(Equal) => Continue(()),
+            c => Break(c.is_some_and(Ordering::is_lt)),
+        }
+    }
+    #[inline]
+    default fn spec_chain_le(&self, other: &U) -> ControlFlow<bool> {
+        match PartialOrd::partial_cmp(self, other) {
+            Some(Equal) => Continue(()),
+            c => Break(c.is_some_and(Ordering::is_le)),
+        }
+    }
+    #[inline]
+    default fn spec_chain_gt(&self, other: &U) -> ControlFlow<bool> {
+        match PartialOrd::partial_cmp(self, other) {
+            Some(Equal) => Continue(()),
+            c => Break(c.is_some_and(Ordering::is_gt)),
+        }
+    }
+    #[inline]
+    default fn spec_chain_ge(&self, other: &U) -> ControlFlow<bool> {
+        match PartialOrd::partial_cmp(self, other) {
+            Some(Equal) => Continue(()),
+            c => Break(c.is_some_and(Ordering::is_ge)),
+        }
+    }
+}
+
 /// Compares and returns the minimum of two values.
 ///
 /// Returns the first argument if the comparison determines them to be equal.
@@ -1741,6 +1790,7 @@ where
 mod impls {
     use crate::cmp::Ordering::{self, Equal, Greater, Less};
     use crate::hint::unreachable_unchecked;
+    use crate::ops::ControlFlow::{self, Break, Continue};
 
     macro_rules! partial_eq_impl {
         ($($t:ty)*) => ($(
@@ -1779,6 +1829,36 @@ mod impls {
 
     eq_impl! { () bool char usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 }
 
+    macro_rules! chaining_impl {
+        ($t:ty) => {
+            // These implementations are the same for `Ord` or `PartialOrd` types
+            // because if either is NAN the `==` test will fail so we end up in
+            // the `Break` case and the comparison will correctly return `false`.
+            impl super::SpecChainingPartialOrd<$t> for $t {
+                #[inline]
+                fn spec_chain_lt(&self, other: &Self) -> ControlFlow<bool> {
+                    let (lhs, rhs) = (*self, *other);
+                    if lhs == rhs { Continue(()) } else { Break(lhs < rhs) }
+                }
+                #[inline]
+                fn spec_chain_le(&self, other: &Self) -> ControlFlow<bool> {
+                    let (lhs, rhs) = (*self, *other);
+                    if lhs == rhs { Continue(()) } else { Break(lhs <= rhs) }
+                }
+                #[inline]
+                fn spec_chain_gt(&self, other: &Self) -> ControlFlow<bool> {
+                    let (lhs, rhs) = (*self, *other);
+                    if lhs == rhs { Continue(()) } else { Break(lhs > rhs) }
+                }
+                #[inline]
+                fn spec_chain_ge(&self, other: &Self) -> ControlFlow<bool> {
+                    let (lhs, rhs) = (*self, *other);
+                    if lhs == rhs { Continue(()) } else { Break(lhs >= rhs) }
+                }
+            }
+        };
+    }
+
     macro_rules! partial_ord_impl {
         ($($t:ty)*) => ($(
             #[stable(feature = "rust1", since = "1.0.0")]
@@ -1801,6 +1881,8 @@ mod impls {
                 #[inline(always)]
                 fn gt(&self, other: &$t) -> bool { (*self) > (*other) }
             }
+
+            chaining_impl!($t);
         )*)
     }
 
@@ -1840,6 +1922,8 @@ mod impls {
                 fn gt(&self, other: &$t) -> bool { (*self) > (*other) }
             }
 
+            chaining_impl!($t);
+
             #[stable(feature = "rust1", since = "1.0.0")]
             impl Ord for $t {
                 #[inline]

library/core/src/tuple.rs

@@ -1,7 +1,9 @@
 // See core/src/primitive_docs.rs for documentation.
 
 use crate::cmp::Ordering::{self, *};
+use crate::cmp::SpecChainingPartialOrd;
 use crate::marker::{ConstParamTy_, StructuralPartialEq, UnsizedConstParamTy};
+use crate::ops::ControlFlow::{Break, Continue};
 
 // Recursive macro for implementing n-ary tuple functions and operations
 //
@@ -80,19 +82,19 @@ macro_rules! tuple_impls {
                 }
                 #[inline]
                 fn lt(&self, other: &($($T,)+)) -> bool {
-                    lexical_ord!(lt, Less, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
+                    lexical_ord!(lt, spec_chain_lt, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
                 }
                 #[inline]
                 fn le(&self, other: &($($T,)+)) -> bool {
-                    lexical_ord!(le, Less, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
+                    lexical_ord!(le, spec_chain_le, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
                 }
                 #[inline]
                 fn ge(&self, other: &($($T,)+)) -> bool {
-                    lexical_ord!(ge, Greater, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
+                    lexical_ord!(ge, spec_chain_ge, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
                 }
                 #[inline]
                 fn gt(&self, other: &($($T,)+)) -> bool {
-                    lexical_ord!(gt, Greater, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
+                    lexical_ord!(gt, spec_chain_gt, $( ${ignore($T)} self.${index()}, other.${index()} ),+)
                 }
             }
         }
@@ -171,15 +173,16 @@ macro_rules! maybe_tuple_doc {
 // `(a1, a2, a3) < (b1, b2, b3)` would be `lexical_ord!(lt, opt_is_lt, a1, b1,
 // a2, b2, a3, b3)` (and similarly for `lexical_cmp`)
 //
-// `$ne_rel` is only used to determine the result after checking that they're
-// not equal, so `lt` and `le` can both just use `Less`.
+// `$chain_rel` is the method from `SpecChainingPartialOrd` to use for all but the
+// final value, to produce better results for simple primitives.
 macro_rules! lexical_ord {
-    ($rel: ident, $ne_rel: ident, $a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {{
-        let c = PartialOrd::partial_cmp(&$a, &$b);
-        if c != Some(Equal) { c == Some($ne_rel) }
-        else { lexical_ord!($rel, $ne_rel, $($rest_a, $rest_b),+) }
+    ($rel: ident, $chain_rel: ident, $a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {{
+        match SpecChainingPartialOrd::$chain_rel(&$a, &$b) {
+            Break(val) => val,
+            Continue(()) => lexical_ord!($rel, $chain_rel, $($rest_a, $rest_b),+),
+        }
     }};
-    ($rel: ident, $ne_rel: ident, $a:expr, $b:expr) => {
+    ($rel: ident, $chain_rel: ident, $a:expr, $b:expr) => {
         // Use the specific method for the last element
         PartialOrd::$rel(&$a, &$b)
     };

tests/mir-opt/inline/exponential_runtime.rs

@@ -87,10 +87,15 @@ fn main() {
     // CHECK-LABEL: fn main(
     // CHECK-NOT: inlined
     // CHECK: (inlined <() as G>::call)
+    // CHECK-NOT: inlined
     // CHECK: (inlined <() as F>::call)
+    // CHECK-NOT: inlined
     // CHECK: (inlined <() as E>::call)
+    // CHECK-NOT: inlined
     // CHECK: (inlined <() as D>::call)
+    // CHECK-NOT: inlined
     // CHECK: (inlined <() as C>::call)
+    // CHECK-NOT: inlined
     // CHECK: (inlined <() as B>::call)
     // CHECK-NOT: inlined
     <() as G>::call();

tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff

@@ -6,6 +6,7 @@
       let _1: (!, !);
 +     let mut _2: fn() -> ! {sleep};
 +     let mut _7: ();
++     let mut _8: ();
 +     scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) {
 +         debug f => _2;
 +         let mut _3: &fn() -> ! {sleep};
@@ -17,6 +18,10 @@
 +             scope 3 {
 +                 debug b => _6;
 +             }
++             scope 6 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
++                 scope 7 (inlined sleep) {
++                 }
++             }
 +         }
 +         scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
 +             scope 5 (inlined sleep) {

tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff

@@ -6,6 +6,7 @@
       let _1: (!, !);
 +     let mut _2: fn() -> ! {sleep};
 +     let mut _8: ();
++     let mut _9: ();
 +     scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) {
 +         debug f => _2;
 +         let mut _3: &fn() -> ! {sleep};
@@ -18,6 +19,10 @@
 +             scope 3 {
 +                 debug b => _6;
 +             }
++             scope 6 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
++                 scope 7 (inlined sleep) {
++                 }
++             }
 +         }
 +         scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
 +             scope 5 (inlined sleep) {

tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff

@@ -55,10 +55,50 @@
 +                         let _26: ();
 +                         scope 9 {
 +                         }
++                         scope 12 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
++                         }
++                         scope 13 (inlined <std::future::Ready<()> as Future>::poll) {
++                             let mut _42: ();
++                             let mut _43: std::option::Option<()>;
++                             let mut _44: &mut std::option::Option<()>;
++                             let mut _45: &mut std::future::Ready<()>;
++                             let mut _46: &mut std::pin::Pin<&mut std::future::Ready<()>>;
++                             scope 14 (inlined <Pin<&mut std::future::Ready<()>> as DerefMut>::deref_mut) {
++                                 let mut _47: std::pin::Pin<&mut std::future::Ready<()>>;
++                                 scope 15 (inlined Pin::<&mut std::future::Ready<()>>::as_mut) {
++                                     let mut _48: &mut &mut std::future::Ready<()>;
++                                     scope 16 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
++                                     }
++                                     scope 18 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) {
++                                     }
++                                 }
++                                 scope 17 (inlined Pin::<&mut std::future::Ready<()>>::get_mut) {
++                                 }
++                             }
++                             scope 19 (inlined Option::<()>::take) {
++                                 let mut _49: std::option::Option<()>;
++                                 scope 20 (inlined std::mem::replace::<Option<()>>) {
++                                     scope 21 {
++                                         scope 23 (inlined std::ptr::write::<Option<()>>) {
++                                         }
++                                     }
++                                     scope 22 (inlined std::ptr::read::<Option<()>>) {
++                                     }
++                                 }
++                             }
++                             scope 24 (inlined #[track_caller] Option::<()>::expect) {
++                                 let mut _50: isize;
++                                 let mut _51: !;
++                                 scope 25 {
++                                 }
++                             }
++                         }
 +                     }
 +                     scope 10 (inlined ready::<()>) {
 +                         let mut _41: std::option::Option<()>;
 +                     }
++                     scope 11 (inlined <std::future::Ready<()> as IntoFuture>::into_future) {
++                     }
 +                 }
 +             }
           }
@@ -113,7 +153,7 @@
 +         StorageLive(_40);
 +         _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         _32 = discriminant((*_33));
-+         switchInt(move _32) -> [0: bb3, 1: bb13, 3: bb12, otherwise: bb8];
++         switchInt(move _32) -> [0: bb3, 1: bb10, 3: bb9, otherwise: bb5];
       }
 
 -     bb3: {
@@ -164,19 +204,16 @@
 +         _13 = std::future::Ready::<()>(move _41);
 +         StorageDead(_41);
 +         StorageDead(_14);
-+         _12 = <std::future::Ready<()> as IntoFuture>::into_future(move _13) -> [return: bb4, unwind unreachable];
-+     }
-+ 
-      bb4: {
--         StorageDead(_2);
--         return;
++         _12 = move _13;
 +         StorageDead(_13);
 +         _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         (((*_36) as variant#3).1: std::future::Ready<()>) = move _12;
-+         goto -> bb5;
++         goto -> bb4;
 +     }
 + 
-+     bb5: {
+      bb4: {
+-         StorageDead(_2);
+-         return;
 +         StorageLive(_17);
 +         StorageLive(_18);
 +         StorageLive(_19);
@@ -185,10 +222,7 @@
 +         _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>);
 +         _20 = &mut (*_21);
-+         _19 = Pin::<&mut std::future::Ready<()>>::new_unchecked(move _20) -> [return: bb6, unwind unreachable];
-+     }
-+ 
-+     bb6: {
++         _19 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _20 };
 +         StorageDead(_20);
 +         StorageLive(_22);
 +         StorageLive(_23);
@@ -197,21 +231,36 @@
 +         _23 = move _24;
 +         _22 = &mut (*_23);
 +         StorageDead(_24);
-+         _18 = <std::future::Ready<()> as Future>::poll(move _19, move _22) -> [return: bb7, unwind unreachable];
-+     }
-+ 
-+     bb7: {
-+         StorageDead(_22);
-+         StorageDead(_19);
-+         _25 = discriminant(_18);
-+         switchInt(move _25) -> [0: bb10, 1: bb9, otherwise: bb8];
++         StorageLive(_45);
++         StorageLive(_46);
++         StorageLive(_49);
++         StorageLive(_51);
++         StorageLive(_42);
++         StorageLive(_43);
++         StorageLive(_44);
++         _46 = &mut _19;
++         StorageLive(_47);
++         StorageLive(_48);
++         _48 = &mut (_19.0: &mut std::future::Ready<()>);
++         _45 = copy (_19.0: &mut std::future::Ready<()>);
++         StorageDead(_48);
++         _47 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _45 };
++         StorageDead(_47);
++         _44 = &mut ((*_45).0: std::option::Option<()>);
++         _49 = Option::<()>::None;
++         _43 = copy ((*_45).0: std::option::Option<()>);
++         ((*_45).0: std::option::Option<()>) = copy _49;
++         StorageDead(_44);
++         StorageLive(_50);
++         _50 = discriminant(_43);
++         switchInt(move _50) -> [0: bb11, 1: bb12, otherwise: bb5];
 +     }
 + 
-+     bb8: {
++     bb5: {
 +         unreachable;
 +     }
 + 
-+     bb9: {
++     bb6: {
 +         _17 = const ();
 +         StorageDead(_23);
 +         StorageDead(_21);
@@ -229,7 +278,7 @@
 +         goto -> bb2;
 +     }
 + 
-+     bb10: {
++     bb7: {
 +         StorageLive(_26);
 +         _26 = copy ((_18 as Ready).0: ());
 +         _30 = copy _26;
@@ -240,17 +289,17 @@
 +         StorageDead(_17);
 +         StorageDead(_12);
 +         _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb11, unwind unreachable];
++         drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb8, unwind unreachable];
 +     }
 + 
-+     bb11: {
++     bb8: {
 +         _7 = Poll::<()>::Ready(move _30);
 +         _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         discriminant((*_40)) = 1;
 +         goto -> bb2;
 +     }
 + 
-+     bb12: {
++     bb9: {
 +         StorageLive(_12);
 +         StorageLive(_28);
 +         StorageLive(_29);
@@ -259,11 +308,31 @@
 +         _31 = move _28;
 +         StorageDead(_28);
 +         _16 = const ();
-+         goto -> bb5;
++         goto -> bb4;
 +     }
 + 
-+     bb13: {
-+         assert(const false, "`async fn` resumed after completion") -> [success: bb13, unwind unreachable];
++     bb10: {
++         assert(const false, "`async fn` resumed after completion") -> [success: bb10, unwind unreachable];
++     }
++ 
++     bb11: {
++         _51 = option::expect_failed(const "`Ready` polled after completion") -> unwind unreachable;
++     }
++ 
++     bb12: {
++         _42 = move ((_43 as Some).0: ());
++         StorageDead(_50);
++         StorageDead(_43);
++         _18 = Poll::<()>::Ready(move _42);
++         StorageDead(_42);
++         StorageDead(_51);
++         StorageDead(_49);
++         StorageDead(_46);
++         StorageDead(_45);
++         StorageDead(_22);
++         StorageDead(_19);
++         _25 = discriminant(_18);
++         switchInt(move _25) -> [0: bb7, 1: bb6, otherwise: bb5];
       }
   }
 

tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff

@@ -57,10 +57,50 @@
 +                         let _26: ();
 +                         scope 9 {
 +                         }
++                         scope 12 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
++                         }
++                         scope 13 (inlined <std::future::Ready<()> as Future>::poll) {
++                             let mut _44: ();
++                             let mut _45: std::option::Option<()>;
++                             let mut _46: &mut std::option::Option<()>;
++                             let mut _47: &mut std::future::Ready<()>;
++                             let mut _48: &mut std::pin::Pin<&mut std::future::Ready<()>>;
++                             scope 14 (inlined <Pin<&mut std::future::Ready<()>> as DerefMut>::deref_mut) {
++                                 let mut _49: std::pin::Pin<&mut std::future::Ready<()>>;
++                                 scope 15 (inlined Pin::<&mut std::future::Ready<()>>::as_mut) {
++                                     let mut _50: &mut &mut std::future::Ready<()>;
++                                     scope 16 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
++                                     }
++                                     scope 18 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) {
++                                     }
++                                 }
++                                 scope 17 (inlined Pin::<&mut std::future::Ready<()>>::get_mut) {
++                                 }
++                             }
++                             scope 19 (inlined Option::<()>::take) {
++                                 let mut _51: std::option::Option<()>;
++                                 scope 20 (inlined std::mem::replace::<Option<()>>) {
++                                     scope 21 {
++                                         scope 23 (inlined std::ptr::write::<Option<()>>) {
++                                         }
++                                     }
++                                     scope 22 (inlined std::ptr::read::<Option<()>>) {
++                                     }
++                                 }
++                             }
++                             scope 24 (inlined #[track_caller] Option::<()>::expect) {
++                                 let mut _52: isize;
++                                 let mut _53: !;
++                                 scope 25 {
++                                 }
++                             }
++                         }
 +                     }
 +                     scope 10 (inlined ready::<()>) {
 +                         let mut _43: std::option::Option<()>;
 +                     }
++                     scope 11 (inlined <std::future::Ready<()> as IntoFuture>::into_future) {
++                     }
 +                 }
 +             }
           }
@@ -117,7 +157,7 @@
 +         StorageLive(_42);
 +         _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         _32 = discriminant((*_33));
-+         switchInt(move _32) -> [0: bb5, 1: bb22, 2: bb21, 3: bb20, otherwise: bb10];
++         switchInt(move _32) -> [0: bb5, 1: bb15, 2: bb14, 3: bb13, otherwise: bb7];
       }
 
 -     bb3: {
@@ -181,21 +221,16 @@
 +         _13 = std::future::Ready::<()>(move _43);
 +         StorageDead(_43);
 +         StorageDead(_14);
-+         _12 = <std::future::Ready<()> as IntoFuture>::into_future(move _13) -> [return: bb6, unwind: bb17];
-      }
-  
--     bb5 (cleanup): {
--         drop(_2) -> [return: bb6, unwind terminate(cleanup)];
-+     bb6: {
++         _12 = move _13;
 +         StorageDead(_13);
 +         _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         (((*_36) as variant#3).1: std::future::Ready<()>) = move _12;
-+         goto -> bb7;
++         goto -> bb6;
       }
 
--     bb6 (cleanup): {
--         resume;
-+     bb7: {
+-     bb5 (cleanup): {
+-         drop(_2) -> [return: bb6, unwind terminate(cleanup)];
++     bb6: {
 +         StorageLive(_17);
 +         StorageLive(_18);
 +         StorageLive(_19);
@@ -204,10 +239,7 @@
 +         _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>);
 +         _20 = &mut (*_21);
-+         _19 = Pin::<&mut std::future::Ready<()>>::new_unchecked(move _20) -> [return: bb8, unwind: bb15];
-+     }
-+ 
-+     bb8: {
++         _19 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _20 };
 +         StorageDead(_20);
 +         StorageLive(_22);
 +         StorageLive(_23);
@@ -216,21 +248,38 @@
 +         _23 = move _24;
 +         _22 = &mut (*_23);
 +         StorageDead(_24);
-+         _18 = <std::future::Ready<()> as Future>::poll(move _19, move _22) -> [return: bb9, unwind: bb14];
-+     }
-+ 
-+     bb9: {
-+         StorageDead(_22);
-+         StorageDead(_19);
-+         _25 = discriminant(_18);
-+         switchInt(move _25) -> [0: bb12, 1: bb11, otherwise: bb10];
-+     }
-+ 
-+     bb10: {
++         StorageLive(_47);
++         StorageLive(_48);
++         StorageLive(_51);
++         StorageLive(_53);
++         StorageLive(_44);
++         StorageLive(_45);
++         StorageLive(_46);
++         _48 = &mut _19;
++         StorageLive(_49);
++         StorageLive(_50);
++         _50 = &mut (_19.0: &mut std::future::Ready<()>);
++         _47 = copy (_19.0: &mut std::future::Ready<()>);
++         StorageDead(_50);
++         _49 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _47 };
++         StorageDead(_49);
++         _46 = &mut ((*_47).0: std::option::Option<()>);
++         _51 = Option::<()>::None;
++         _45 = copy ((*_47).0: std::option::Option<()>);
++         ((*_47).0: std::option::Option<()>) = copy _51;
++         StorageDead(_46);
++         StorageLive(_52);
++         _52 = discriminant(_45);
++         switchInt(move _52) -> [0: bb16, 1: bb17, otherwise: bb7];
+      }
+  
+-     bb6 (cleanup): {
+-         resume;
++     bb7: {
 +         unreachable;
 +     }
 + 
-+     bb11: {
++     bb8: {
 +         _17 = const ();
 +         StorageDead(_23);
 +         StorageDead(_21);
@@ -248,7 +297,7 @@
 +         goto -> bb4;
 +     }
 + 
-+     bb12: {
++     bb9: {
 +         StorageLive(_26);
 +         _26 = copy ((_18 as Ready).0: ());
 +         _30 = copy _26;
@@ -259,54 +308,35 @@
 +         StorageDead(_17);
 +         StorageDead(_12);
 +         _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb13, unwind: bb19];
++         drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb10, unwind: bb12];
 +     }
 + 
-+     bb13: {
++     bb10: {
 +         _7 = Poll::<()>::Ready(move _30);
 +         _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         discriminant((*_40)) = 1;
 +         goto -> bb4;
 +     }
 + 
-+     bb14 (cleanup): {
++     bb11 (cleanup): {
 +         StorageDead(_22);
 +         StorageDead(_19);
 +         StorageDead(_23);
-+         goto -> bb16;
-+     }
-+ 
-+     bb15 (cleanup): {
-+         StorageDead(_20);
-+         StorageDead(_19);
-+         goto -> bb16;
-+     }
-+ 
-+     bb16 (cleanup): {
 +         StorageDead(_21);
 +         StorageDead(_18);
 +         StorageDead(_17);
-+         goto -> bb18;
-+     }
-+ 
-+     bb17 (cleanup): {
-+         StorageDead(_13);
-+         goto -> bb18;
-+     }
-+ 
-+     bb18 (cleanup): {
 +         StorageDead(_12);
 +         _41 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         drop((((*_41) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb19, unwind terminate(cleanup)];
++         drop((((*_41) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb12, unwind terminate(cleanup)];
 +     }
 + 
-+     bb19 (cleanup): {
++     bb12 (cleanup): {
 +         _42 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
 +         discriminant((*_42)) = 2;
 +         goto -> bb2;
 +     }
 + 
-+     bb20: {
++     bb13: {
 +         StorageLive(_12);
 +         StorageLive(_28);
 +         StorageLive(_29);
@@ -315,15 +345,35 @@
 +         _31 = move _28;
 +         StorageDead(_28);
 +         _16 = const ();
-+         goto -> bb7;
++         goto -> bb6;
 +     }
 + 
-+     bb21: {
-+         assert(const false, "`async fn` resumed after panicking") -> [success: bb21, unwind: bb2];
++     bb14: {
++         assert(const false, "`async fn` resumed after panicking") -> [success: bb14, unwind: bb2];
 +     }
 + 
-+     bb22: {
-+         assert(const false, "`async fn` resumed after completion") -> [success: bb22, unwind: bb2];
++     bb15: {
++         assert(const false, "`async fn` resumed after completion") -> [success: bb15, unwind: bb2];
++     }
++ 
++     bb16: {
++         _53 = option::expect_failed(const "`Ready` polled after completion") -> bb11;
++     }
++ 
++     bb17: {
++         _44 = move ((_45 as Some).0: ());
++         StorageDead(_52);
++         StorageDead(_45);
++         _18 = Poll::<()>::Ready(move _44);
++         StorageDead(_44);
++         StorageDead(_53);
++         StorageDead(_51);
++         StorageDead(_48);
++         StorageDead(_47);
++         StorageDead(_22);
++         StorageDead(_19);
++         _25 = discriminant(_18);
++         switchInt(move _25) -> [0: bb9, 1: bb8, otherwise: bb7];
       }
   }
 

tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir

@@ -26,6 +26,18 @@ fn int_range(_1: usize, _2: usize) -> () {
                 let mut _12: usize;
                 scope 6 {
                     debug old => _11;
+                    scope 8 (inlined <usize as Step>::forward_unchecked) {
+                        debug start => _11;
+                        debug n => const 1_usize;
+                        scope 9 (inlined core::num::<impl usize>::unchecked_add) {
+                            debug self => _11;
+                            debug rhs => const 1_usize;
+                            scope 10 (inlined core::ub_checks::check_language_ub) {
+                                scope 11 (inlined core::ub_checks::check_language_ub::runtime) {
+                                }
+                            }
+                        }
+                    }
                 }
                 scope 7 (inlined std::cmp::impls::<impl PartialOrd for usize>::lt) {
                     debug self => _6;
@@ -50,7 +62,6 @@ fn int_range(_1: usize, _2: usize) -> () {
     bb1: {
         StorageLive(_13);
         _5 = &mut _4;
-        StorageLive(_11);
         StorageLive(_10);
         StorageLive(_6);
         _6 = &(_4.0: usize);
@@ -70,7 +81,6 @@ fn int_range(_1: usize, _2: usize) -> () {
         StorageDead(_7);
         StorageDead(_6);
         StorageDead(_10);
-        StorageDead(_11);
         StorageDead(_13);
         StorageDead(_4);
         return;
@@ -81,20 +91,16 @@ fn int_range(_1: usize, _2: usize) -> () {
         StorageDead(_6);
         _11 = copy (_4.0: usize);
         StorageLive(_12);
-        _12 = <usize as Step>::forward_unchecked(copy _11, const 1_usize) -> [return: bb4, unwind continue];
-    }
-
-    bb4: {
+        _12 = AddUnchecked(copy _11, const 1_usize);
         (_4.0: usize) = move _12;
         StorageDead(_12);
         _13 = Option::<usize>::Some(copy _11);
         StorageDead(_10);
-        StorageDead(_11);
         _14 = copy ((_13 as Some).0: usize);
-        _15 = opaque::<usize>(move _14) -> [return: bb5, unwind continue];
+        _15 = opaque::<usize>(move _14) -> [return: bb4, unwind continue];
     }
 
-    bb5: {
+    bb4: {
         StorageDead(_13);
         goto -> bb1;
     }

tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir

@@ -23,6 +23,14 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
                 let _7: u32;
                 let mut _8: u32;
                 scope 6 {
+                    scope 8 (inlined <u32 as Step>::forward_unchecked) {
+                        scope 9 (inlined core::num::<impl u32>::unchecked_add) {
+                            scope 10 (inlined core::ub_checks::check_language_ub) {
+                                scope 11 (inlined core::ub_checks::check_language_ub::runtime) {
+                                }
+                            }
+                        }
+                    }
                 }
                 scope 7 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) {
                     let mut _5: u32;
@@ -41,7 +49,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
 
     bb1: {
         StorageLive(_9);
-        StorageLive(_7);
         StorageLive(_6);
         StorageLive(_5);
         _5 = copy _4;
@@ -52,7 +59,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
 
     bb2: {
         StorageDead(_6);
-        StorageDead(_7);
         StorageDead(_9);
         StorageDead(_4);
         drop(_3) -> [return: bb3, unwind unreachable];
@@ -65,24 +71,20 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     bb4: {
         _7 = copy _4;
         StorageLive(_8);
-        _8 = <u32 as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind unreachable];
-    }
-
-    bb5: {
+        _8 = AddUnchecked(copy _7, const 1_u32);
         _4 = move _8;
         StorageDead(_8);
         _9 = Option::<u32>::Some(copy _7);
         StorageDead(_6);
-        StorageDead(_7);
         _10 = copy ((_9 as Some).0: u32);
         StorageLive(_11);
         _11 = &_3;
         StorageLive(_12);
         _12 = (copy _10,);
-        _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb6, unwind unreachable];
+        _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb5, unwind unreachable];
     }
 
-    bb6: {
+    bb5: {
         StorageDead(_12);
         StorageDead(_11);
         StorageDead(_9);

tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir

@@ -23,6 +23,14 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
                 let _7: u32;
                 let mut _8: u32;
                 scope 6 {
+                    scope 8 (inlined <u32 as Step>::forward_unchecked) {
+                        scope 9 (inlined core::num::<impl u32>::unchecked_add) {
+                            scope 10 (inlined core::ub_checks::check_language_ub) {
+                                scope 11 (inlined core::ub_checks::check_language_ub::runtime) {
+                                }
+                            }
+                        }
+                    }
                 }
                 scope 7 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) {
                     let mut _5: u32;
@@ -41,7 +49,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
 
     bb1: {
         StorageLive(_9);
-        StorageLive(_7);
         StorageLive(_6);
         StorageLive(_5);
         _5 = copy _4;
@@ -52,7 +59,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
 
     bb2: {
         StorageDead(_6);
-        StorageDead(_7);
         StorageDead(_9);
         StorageDead(_4);
         drop(_3) -> [return: bb3, unwind continue];
@@ -65,35 +71,31 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     bb4: {
         _7 = copy _4;
         StorageLive(_8);
-        _8 = <u32 as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind: bb7];
-    }
-
-    bb5: {
+        _8 = AddUnchecked(copy _7, const 1_u32);
         _4 = move _8;
         StorageDead(_8);
         _9 = Option::<u32>::Some(copy _7);
         StorageDead(_6);
-        StorageDead(_7);
         _10 = copy ((_9 as Some).0: u32);
         StorageLive(_11);
         _11 = &_3;
         StorageLive(_12);
         _12 = (copy _10,);
-        _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb6, unwind: bb7];
+        _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb5, unwind: bb6];
     }
 
-    bb6: {
+    bb5: {
         StorageDead(_12);
         StorageDead(_11);
         StorageDead(_9);
         goto -> bb1;
     }
 
-    bb7 (cleanup): {
-        drop(_3) -> [return: bb8, unwind terminate(cleanup)];
+    bb6 (cleanup): {
+        drop(_3) -> [return: bb7, unwind terminate(cleanup)];
     }
 
-    bb8 (cleanup): {
+    bb7 (cleanup): {
         resume;
     }
 }

tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-abort.mir

@@ -9,6 +9,14 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> {
             let _5: u32;
             let mut _6: u32;
             scope 3 {
+                scope 5 (inlined <u32 as Step>::forward_unchecked) {
+                    scope 6 (inlined core::num::<impl u32>::unchecked_add) {
+                        scope 7 (inlined core::ub_checks::check_language_ub) {
+                            scope 8 (inlined core::ub_checks::check_language_ub::runtime) {
+                            }
+                        }
+                    }
+                }
             }
             scope 4 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) {
                 let mut _2: u32;
@@ -18,7 +26,6 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> {
     }
 
     bb0: {
-        StorageLive(_5);
         StorageLive(_4);
         StorageLive(_2);
         _2 = copy ((*_1).0: u32);
@@ -32,25 +39,21 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> {
 
     bb1: {
         _0 = const Option::<u32>::None;
-        goto -> bb4;
+        goto -> bb3;
     }
 
     bb2: {
         _5 = copy ((*_1).0: u32);
         StorageLive(_6);
-        _6 = <u32 as Step>::forward_unchecked(copy _5, const 1_usize) -> [return: bb3, unwind unreachable];
-    }
-
-    bb3: {
+        _6 = AddUnchecked(copy _5, const 1_u32);
         ((*_1).0: u32) = move _6;
         StorageDead(_6);
         _0 = Option::<u32>::Some(copy _5);
-        goto -> bb4;
+        goto -> bb3;
     }
 
-    bb4: {
+    bb3: {
         StorageDead(_4);
-        StorageDead(_5);
         return;
     }
 }

tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-unwind.mir

@@ -9,6 +9,14 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> {
             let _5: u32;
             let mut _6: u32;
             scope 3 {
+                scope 5 (inlined <u32 as Step>::forward_unchecked) {
+                    scope 6 (inlined core::num::<impl u32>::unchecked_add) {
+                        scope 7 (inlined core::ub_checks::check_language_ub) {
+                            scope 8 (inlined core::ub_checks::check_language_ub::runtime) {
+                            }
+                        }
+                    }
+                }
             }
             scope 4 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) {
                 let mut _2: u32;
@@ -18,7 +26,6 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> {
     }
 
     bb0: {
-        StorageLive(_5);
         StorageLive(_4);
         StorageLive(_2);
         _2 = copy ((*_1).0: u32);
@@ -32,25 +39,21 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> {
 
     bb1: {
         _0 = const Option::<u32>::None;
-        goto -> bb4;
+        goto -> bb3;
     }
 
     bb2: {
         _5 = copy ((*_1).0: u32);
         StorageLive(_6);
-        _6 = <u32 as Step>::forward_unchecked(copy _5, const 1_usize) -> [return: bb3, unwind continue];
-    }
-
-    bb3: {
+        _6 = AddUnchecked(copy _5, const 1_u32);
         ((*_1).0: u32) = move _6;
         StorageDead(_6);
         _0 = Option::<u32>::Some(copy _5);
-        goto -> bb4;
+        goto -> bb3;
     }
 
-    bb4: {
+    bb3: {
         StorageDead(_4);
-        StorageDead(_5);
         return;
     }
 }

tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir

@@ -28,6 +28,14 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
                 let _7: usize;
                 let mut _8: usize;
                 scope 7 {
+                    scope 9 (inlined <usize as Step>::forward_unchecked) {
+                        scope 10 (inlined core::num::<impl usize>::unchecked_add) {
+                            scope 11 (inlined core::ub_checks::check_language_ub) {
+                                scope 12 (inlined core::ub_checks::check_language_ub::runtime) {
+                                }
+                            }
+                        }
+                    }
                 }
                 scope 8 (inlined std::cmp::impls::<impl PartialOrd for usize>::lt) {
                     let mut _5: usize;
@@ -47,7 +55,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb1: {
         StorageLive(_9);
-        StorageLive(_7);
         StorageLive(_6);
         StorageLive(_5);
         _5 = copy _4;
@@ -58,7 +65,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb2: {
         StorageDead(_6);
-        StorageDead(_7);
         StorageDead(_9);
         StorageDead(_4);
         drop(_2) -> [return: bb3, unwind unreachable];
@@ -71,30 +77,26 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     bb4: {
         _7 = copy _4;
         StorageLive(_8);
-        _8 = <usize as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind unreachable];
-    }
-
-    bb5: {
+        _8 = AddUnchecked(copy _7, const 1_usize);
         _4 = move _8;
         StorageDead(_8);
         _9 = Option::<usize>::Some(copy _7);
         StorageDead(_6);
-        StorageDead(_7);
         _10 = copy ((_9 as Some).0: usize);
         _11 = Lt(copy _10, copy _3);
-        assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb6, unwind unreachable];
+        assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind unreachable];
     }
 
-    bb6: {
+    bb5: {
         _12 = &(*_1)[_10];
         StorageLive(_13);
         _13 = &_2;
         StorageLive(_14);
         _14 = (copy _10, copy _12);
-        _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb7, unwind unreachable];
+        _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb6, unwind unreachable];
     }
 
-    bb7: {
+    bb6: {
         StorageDead(_14);
         StorageDead(_13);
         StorageDead(_9);

tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir

@@ -28,6 +28,14 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
                 let _7: usize;
                 let mut _8: usize;
                 scope 7 {
+                    scope 9 (inlined <usize as Step>::forward_unchecked) {
+                        scope 10 (inlined core::num::<impl usize>::unchecked_add) {
+                            scope 11 (inlined core::ub_checks::check_language_ub) {
+                                scope 12 (inlined core::ub_checks::check_language_ub::runtime) {
+                                }
+                            }
+                        }
+                    }
                 }
                 scope 8 (inlined std::cmp::impls::<impl PartialOrd for usize>::lt) {
                     let mut _5: usize;
@@ -47,7 +55,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb1: {
         StorageLive(_9);
-        StorageLive(_7);
         StorageLive(_6);
         StorageLive(_5);
         _5 = copy _4;
@@ -58,7 +65,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb2: {
         StorageDead(_6);
-        StorageDead(_7);
         StorageDead(_9);
         StorageDead(_4);
         drop(_2) -> [return: bb3, unwind continue];
@@ -71,41 +77,37 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     bb4: {
         _7 = copy _4;
         StorageLive(_8);
-        _8 = <usize as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind: bb8];
-    }
-
-    bb5: {
+        _8 = AddUnchecked(copy _7, const 1_usize);
         _4 = move _8;
         StorageDead(_8);
         _9 = Option::<usize>::Some(copy _7);
         StorageDead(_6);
-        StorageDead(_7);
         _10 = copy ((_9 as Some).0: usize);
         _11 = Lt(copy _10, copy _3);
-        assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb6, unwind: bb8];
+        assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind: bb7];
     }
 
-    bb6: {
+    bb5: {
         _12 = &(*_1)[_10];
         StorageLive(_13);
         _13 = &_2;
         StorageLive(_14);
         _14 = (copy _10, copy _12);
-        _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb7, unwind: bb8];
+        _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb6, unwind: bb7];
     }
 
-    bb7: {
+    bb6: {
         StorageDead(_14);
         StorageDead(_13);
         StorageDead(_9);
         goto -> bb1;
     }
 
-    bb8 (cleanup): {
-        drop(_2) -> [return: bb9, unwind terminate(cleanup)];
+    bb7 (cleanup): {
+        drop(_2) -> [return: bb8, unwind terminate(cleanup)];
     }
 
-    bb9 (cleanup): {
+    bb8 (cleanup): {
         resume;
     }
 }

tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir

@@ -0,0 +1,70 @@
+// MIR for `demo_ge_partial` after PreCodegen
+
+fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool {
+    debug a => _1;
+    debug b => _2;
+    let mut _0: bool;
+    scope 1 (inlined std::cmp::impls::<impl PartialOrd for &(f32, f32)>::le) {
+        scope 2 (inlined core::tuple::<impl PartialOrd for (f32, f32)>::le) {
+            let mut _7: std::ops::ControlFlow<bool>;
+            let _8: bool;
+            scope 3 {
+            }
+            scope 4 (inlined std::cmp::impls::<impl std::cmp::SpecChainingPartialOrd<f32> for f32>::spec_chain_le) {
+                let mut _3: f32;
+                let mut _4: f32;
+                let mut _5: bool;
+                let mut _6: bool;
+                scope 5 {
+                }
+            }
+            scope 6 (inlined std::cmp::impls::<impl PartialOrd for f32>::le) {
+                let mut _9: f32;
+                let mut _10: f32;
+            }
+        }
+    }
+
+    bb0: {
+        StorageLive(_7);
+        StorageLive(_3);
+        StorageLive(_4);
+        _3 = copy ((*_1).0: f32);
+        _4 = copy ((*_2).0: f32);
+        StorageLive(_5);
+        _5 = Eq(copy _3, copy _4);
+        switchInt(move _5) -> [0: bb1, otherwise: bb2];
+    }
+
+    bb1: {
+        StorageLive(_6);
+        _6 = Le(copy _3, copy _4);
+        _7 = ControlFlow::<bool>::Break(move _6);
+        StorageDead(_6);
+        StorageDead(_5);
+        StorageDead(_4);
+        StorageDead(_3);
+        _8 = copy ((_7 as Break).0: bool);
+        _0 = copy _8;
+        goto -> bb3;
+    }
+
+    bb2: {
+        StorageDead(_5);
+        StorageDead(_4);
+        StorageDead(_3);
+        StorageLive(_9);
+        _9 = copy ((*_1).1: f32);
+        StorageLive(_10);
+        _10 = copy ((*_2).1: f32);
+        _0 = Le(move _9, move _10);
+        StorageDead(_10);
+        StorageDead(_9);
+        goto -> bb3;
+    }
+
+    bb3: {
+        StorageDead(_7);
+        return;
+    }
+}

tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir

@@ -0,0 +1,70 @@
+// MIR for `demo_le_total` after PreCodegen
+
+fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool {
+    debug a => _1;
+    debug b => _2;
+    let mut _0: bool;
+    scope 1 (inlined std::cmp::impls::<impl PartialOrd for &(u16, i16)>::le) {
+        scope 2 (inlined core::tuple::<impl PartialOrd for (u16, i16)>::le) {
+            let mut _7: std::ops::ControlFlow<bool>;
+            let _8: bool;
+            scope 3 {
+            }
+            scope 4 (inlined std::cmp::impls::<impl std::cmp::SpecChainingPartialOrd<u16> for u16>::spec_chain_le) {
+                let mut _3: u16;
+                let mut _4: u16;
+                let mut _5: bool;
+                let mut _6: bool;
+                scope 5 {
+                }
+            }
+            scope 6 (inlined std::cmp::impls::<impl PartialOrd for i16>::le) {
+                let mut _9: i16;
+                let mut _10: i16;
+            }
+        }
+    }
+
+    bb0: {
+        StorageLive(_7);
+        StorageLive(_3);
+        StorageLive(_4);
+        _3 = copy ((*_1).0: u16);
+        _4 = copy ((*_2).0: u16);
+        StorageLive(_5);
+        _5 = Eq(copy _3, copy _4);
+        switchInt(move _5) -> [0: bb1, otherwise: bb2];
+    }
+
+    bb1: {
+        StorageLive(_6);
+        _6 = Le(copy _3, copy _4);
+        _7 = ControlFlow::<bool>::Break(move _6);
+        StorageDead(_6);
+        StorageDead(_5);
+        StorageDead(_4);
+        StorageDead(_3);
+        _8 = copy ((_7 as Break).0: bool);
+        _0 = copy _8;
+        goto -> bb3;
+    }
+
+    bb2: {
+        StorageDead(_5);
+        StorageDead(_4);
+        StorageDead(_3);
+        StorageLive(_9);
+        _9 = copy ((*_1).1: i16);
+        StorageLive(_10);
+        _10 = copy ((*_2).1: i16);
+        _0 = Le(move _9, move _10);
+        StorageDead(_10);
+        StorageDead(_9);
+        goto -> bb3;
+    }
+
+    bb3: {
+        StorageDead(_7);
+        return;
+    }
+}

tests/mir-opt/pre-codegen/tuple_ord.rs

@@ -0,0 +1,16 @@
+//@ compile-flags: -O -Zmir-opt-level=2 -Cdebuginfo=0
+//@ needs-unwind
+
+#![crate_type = "lib"]
+
+// EMIT_MIR tuple_ord.demo_le_total.PreCodegen.after.mir
+pub fn demo_le_total(a: &(u16, i16), b: &(u16, i16)) -> bool {
+    // CHECK-LABEL: demo_le_total
+    a <= b
+}
+
+// EMIT_MIR tuple_ord.demo_ge_partial.PreCodegen.after.mir
+pub fn demo_ge_partial(a: &(f32, f32), b: &(f32, f32)) -> bool {
+    // CHECK-LABEL: demo_ge_partial
+    a <= b
+}