Pre-compute MIR CFG caches for borrowck and other analyses

compiler/rustc_borrowck/src/renumber.rs

@@ -21,10 +21,10 @@ pub(crate) fn renumber_mir<'tcx>(
     let mut renumberer = RegionRenumberer { infcx };
 
     for body in promoted.iter_mut() {
-        renumberer.visit_body(body);
+        renumberer.visit_body_preserves_cfg(body);
     }
 
-    renumberer.visit_body(body);
+    renumberer.visit_body_preserves_cfg(body);
 }
 
 // The fields are used only for debugging output in `sccs_info`.

compiler/rustc_middle/src/mir/basic_blocks.rs

@@ -1,4 +1,4 @@
-use std::sync::OnceLock;
+use std::sync::{Arc, OnceLock};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph;
@@ -15,7 +15,8 @@ use crate::mir::{BasicBlock, BasicBlockData, START_BLOCK, Terminator, Terminator
 #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable, TypeFoldable, TypeVisitable)]
 pub struct BasicBlocks<'tcx> {
     basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
-    cache: Cache,
+    /// Use an `Arc` so we can share the cache when we clone the MIR body, as borrowck does.
+    cache: Arc<Cache>,
 }
 
 // Typically 95%+ of basic blocks have 4 or fewer predecessors.
@@ -49,9 +50,10 @@ struct Cache {
 impl<'tcx> BasicBlocks<'tcx> {
     #[inline]
     pub fn new(basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>) -> Self {
-        BasicBlocks { basic_blocks, cache: Cache::default() }
+        BasicBlocks { basic_blocks, cache: Arc::new(Cache::default()) }
     }
 
+    #[inline]
     pub fn dominators(&self) -> &Dominators<BasicBlock> {
         self.cache.dominators.get_or_init(|| dominators(self))
     }
@@ -142,7 +144,14 @@ impl<'tcx> BasicBlocks<'tcx> {
     /// All other methods that allow you to mutate the basic blocks also call this method
     /// themselves, thereby avoiding any risk of accidentally cache invalidation.
     pub fn invalidate_cfg_cache(&mut self) {
-        self.cache = Cache::default();
+        if let Some(cache) = Arc::get_mut(&mut self.cache) {
+            // If we only have a single reference to this cache, clear it.
+            *cache = Cache::default();
+        } else {
+            // If we have several references to this cache, overwrite the pointer itself so other
+            // users can continue to use their (valid) cache.
+            self.cache = Arc::new(Cache::default());
+        }
     }
 }
 

compiler/rustc_mir_transform/src/simplify.rs

@@ -74,7 +74,11 @@ impl SimplifyCfg {
 }
 
 pub(super) fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    CfgSimplifier::new(tcx, body).simplify();
+    if CfgSimplifier::new(tcx, body).simplify() {
+        // `simplify` returns that it changed something. We must invalidate the CFG caches as they
+        // are not consistent with the modified CFG any more.
+        body.basic_blocks.invalidate_cfg_cache();
+    }
     remove_dead_blocks(body);
 
     // FIXME: Should probably be moved into some kind of pass manager
@@ -121,19 +125,24 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
         // Preserve `SwitchInt` reads on built and analysis MIR, or if `-Zmir-preserve-ub`.
         let preserve_switch_reads = matches!(body.phase, MirPhase::Built | MirPhase::Analysis(_))
             || tcx.sess.opts.unstable_opts.mir_preserve_ub;
-        let basic_blocks = body.basic_blocks_mut();
+        // Do not clear caches yet. The caller to `simplify` will do it if anything changed.
+        let basic_blocks = body.basic_blocks.as_mut_preserves_cfg();
 
         CfgSimplifier { preserve_switch_reads, basic_blocks, pred_count }
     }
 
-    fn simplify(mut self) {
+    /// Returns whether we actually simplified anything. In that case, the caller *must* invalidate
+    /// the CFG caches of the MIR body.
+    #[must_use]
+    fn simplify(mut self) -> bool {
         self.strip_nops();
 
         // Vec of the blocks that should be merged. We store the indices here, instead of the
         // statements itself to avoid moving the (relatively) large statements twice.
         // We do not push the statements directly into the target block (`bb`) as that is slower
         // due to additional reallocations
         let mut merged_blocks = Vec::new();
+        let mut outer_changed = false;
         loop {
             let mut changed = false;
 
@@ -177,7 +186,11 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
             if !changed {
                 break;
             }
+
+            outer_changed = true;
         }
+
+        outer_changed
     }
 
     /// This function will return `None` if