Auto merge of #138759 - scottmcm:operand-builder, r=<try>

Allow `enum` and `union` literals to also create SSA values

Today, `Some(x)` always goes through an `alloca`, even in trivial cases where the niching means the constructor doesn't even change the value.

For example, <https://rust.godbolt.org/z/6KG6PqoYz>
```rust
pub fn demo(r: &i32) -> Option<&i32> {
    Some(r)
}
```
currently emits the IR
```llvm
define align 4 ptr `@demo(ptr` align 4 %r) unnamed_addr {
start:
  %_0 = alloca [8 x i8], align 8
  store ptr %r, ptr %_0, align 8
  %0 = load ptr, ptr %_0, align 8
  ret ptr %0
}
```
but with this PR it becomes just
```llvm
define align 4 ptr `@demo(ptr` align 4 %r) unnamed_addr {
start:
  ret ptr %r
}
```
(Of course the optimizer can clean that up, but it'd be nice if it didn't have to -- especially in debug where it doesn't run.  This is like #123886, but that only handled non-simd `struct`s -- this PR generalizes it to all non-simd ADTs.)

There's two commits you can review independently:
1. The first is simplifying how the aggregate handling works.  Past-me wrote something overly complicated, needing arrayvecs and zipping, depending on a careful iteration order of the fields, and fragile enough that even for just structs it needed extra double-checks to make sure it even made the right variant.  It's replaced with something far more direct that works just like `extract_field`: use the offset to put it in exactly the correct immediate in the `OperandValue`.  This doesn't support anything new, just refactors -- including moving some things off `FunctionCx` that had no reason to be there.  (I have no idea why my past self put them there.)
2. The second extends that work to support more ADTs.  That means handing variants other than `FIRST_VARIANT`, handling the active field for unions, refactoring the discriminant code so the Place and Operand parts can share the calculation, etc.

Author: bors

Cargo.lock

@@ -3404,11 +3404,9 @@ name = "rustc_codegen_ssa"
 version = "0.0.0"
 dependencies = [
  "ar_archive_writer",
- "arrayvec",
  "bitflags",
  "bstr",
  "cc",
- "either",
  "itertools",
  "libc",
  "object 0.36.7",

compiler/rustc_codegen_ssa/Cargo.toml

@@ -6,13 +6,11 @@ edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
 ar_archive_writer = "0.4.2"
-arrayvec = { version = "0.7", default-features = false }
 bitflags = "2.4.1"
 bstr = "1.11.3"
 # Pinned so `cargo update` bumps don't cause breakage. Please also update the
 # `cc` in `rustc_llvm` if you update the `cc` here.
 cc = "=1.2.16"
-either = "1.5.0"
 itertools = "0.12"
 pathdiff = "0.2.0"
 regex = "1.4"

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -1,14 +1,15 @@
 use std::fmt;
 
-use arrayvec::ArrayVec;
-use either::Either;
 use rustc_abi as abi;
-use rustc_abi::{Align, BackendRepr, FIRST_VARIANT, Primitive, Size, TagEncoding, Variants};
+use rustc_abi::{
+    Align, BackendRepr, FIRST_VARIANT, FieldIdx, Primitive, Size, TagEncoding, VariantIdx, Variants,
+};
 use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range};
 use rustc_middle::mir::{self, ConstValue};
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::{bug, span_bug};
+use rustc_session::config::OptLevel;
 use tracing::{debug, instrument};
 
 use super::place::{PlaceRef, PlaceValue};
@@ -62,31 +63,6 @@ pub enum OperandValue<V> {
 }
 
 impl<V: CodegenObject> OperandValue<V> {
-    /// If this is ZeroSized/Immediate/Pair, return an array of the 0/1/2 values.
-    /// If this is Ref, return the place.
-    #[inline]
-    pub(crate) fn immediates_or_place(self) -> Either<ArrayVec<V, 2>, PlaceValue<V>> {
-        match self {
-            OperandValue::ZeroSized => Either::Left(ArrayVec::new()),
-            OperandValue::Immediate(a) => Either::Left(ArrayVec::from_iter([a])),
-            OperandValue::Pair(a, b) => Either::Left([a, b].into()),
-            OperandValue::Ref(p) => Either::Right(p),
-        }
-    }
-
-    /// Given an array of 0/1/2 immediate values, return ZeroSized/Immediate/Pair.
-    #[inline]
-    pub(crate) fn from_immediates(immediates: ArrayVec<V, 2>) -> Self {
-        let mut it = immediates.into_iter();
-        let Some(a) = it.next() else {
-            return OperandValue::ZeroSized;
-        };
-        let Some(b) = it.next() else {
-            return OperandValue::Immediate(a);
-        };
-        OperandValue::Pair(a, b)
-    }
-
     /// Treat this value as a pointer and return the data pointer and
     /// optional metadata as backend values.
     ///
@@ -559,6 +535,123 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             }
         }
     }
+
+    pub(crate) fn builder(layout: TyAndLayout<'tcx>) -> OperandRef<'tcx, Result<V, abi::Scalar>> {
+        let val = match layout.backend_repr {
+            BackendRepr::Memory { .. } if layout.is_zst() => OperandValue::ZeroSized,
+            BackendRepr::Scalar(s) => OperandValue::Immediate(Err(s)),
+            BackendRepr::ScalarPair(a, b) => OperandValue::Pair(Err(a), Err(b)),
+            _ => bug!("Cannot use type in operand builder: {layout:?}"),
+        };
+        OperandRef { val, layout }
+    }
+
+    pub(crate) fn supports_builder(layout: TyAndLayout<'tcx>) -> bool {
+        match layout.backend_repr {
+            BackendRepr::Memory { .. } if layout.is_zst() => true,
+            BackendRepr::Scalar(_) | BackendRepr::ScalarPair(_, _) => true,
+            _ => false,
+        }
+    }
+}
+
+impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> {
+    pub(crate) fn insert_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+        &mut self,
+        bx: &mut Bx,
+        v: VariantIdx,
+        f: FieldIdx,
+        operand: OperandRef<'tcx, V>,
+    ) {
+        let (expect_zst, is_zero_offset) = if let abi::FieldsShape::Primitive = self.layout.fields {
+            // Don't ask for field layout for primitives, because that will panic.
+            if !self.layout.uninhabited {
+                // Real primitives only have one variant, but weird types like
+                // `Result<!, !>` turn out to also be "Primitive", and dead code
+                // like `Err(never)` needs to not ICE.
+                assert_eq!(v, FIRST_VARIANT);
+            }
+            let first_field = f == FieldIdx::ZERO;
+            (self.layout.is_zst() || !first_field, first_field)
+        } else {
+            let variant_layout = self.layout.for_variant(bx.cx(), v);
+            let field_layout = variant_layout.field(bx.cx(), f.as_usize());
+            let field_offset = variant_layout.fields.offset(f.as_usize());
+            (field_layout.is_zst(), field_offset == Size::ZERO)
+        };
+
+        let mut update = |tgt: &mut Result<V, abi::Scalar>, src, from_scalar| {
+            let from_bty = bx.cx().type_from_scalar(from_scalar);
+            let to_scalar = tgt.unwrap_err();
+            let to_bty = bx.cx().type_from_scalar(to_scalar);
+            let imm = transmute_immediate(bx, src, from_scalar, from_bty, to_scalar, to_bty);
+            *tgt = Ok(imm);
+        };
+
+        match (operand.val, operand.layout.backend_repr) {
+            (OperandValue::ZeroSized, _) if expect_zst => {}
+            (OperandValue::Immediate(v), BackendRepr::Scalar(from_scalar)) => match &mut self.val {
+                OperandValue::Immediate(val @ Err(_)) if is_zero_offset => {
+                    update(val, v, from_scalar);
+                }
+                OperandValue::Pair(fst @ Err(_), _) if is_zero_offset => {
+                    update(fst, v, from_scalar);
+                }
+                OperandValue::Pair(_, snd @ Err(_)) if !is_zero_offset => {
+                    update(snd, v, from_scalar);
+                }
+                _ => bug!("Tried to insert {operand:?} into {v:?}.{f:?} of {self:?}"),
+            },
+            (OperandValue::Pair(a, b), BackendRepr::ScalarPair(from_sa, from_sb)) => {
+                match &mut self.val {
+                    OperandValue::Pair(fst @ Err(_), snd @ Err(_)) => {
+                        update(fst, a, from_sa);
+                        update(snd, b, from_sb);
+                    }
+                    _ => bug!("Tried to insert {operand:?} into {v:?}.{f:?} of {self:?}"),
+                }
+            }
+            _ => bug!("Unsupported operand {operand:?} inserting into {v:?}.{f:?} of {self:?}"),
+        }
+    }
+
+    pub(super) fn insert_imm(&mut self, f: FieldIdx, imm: V) {
+        let field_offset = self.layout.fields.offset(f.as_usize());
+        let is_zero_offset = field_offset == Size::ZERO;
+        match &mut self.val {
+            OperandValue::Immediate(val @ Err(_)) if is_zero_offset => {
+                *val = Ok(imm);
+            }
+            OperandValue::Pair(fst @ Err(_), _) if is_zero_offset => {
+                *fst = Ok(imm);
+            }
+            OperandValue::Pair(_, snd @ Err(_)) if !is_zero_offset => {
+                *snd = Ok(imm);
+            }
+            _ => bug!("Tried to insert {imm:?} into field {f:?} of {self:?}"),
+        }
+    }
+
+    pub fn finalize(&self, cx: &impl CodegenMethods<'tcx, Value = V>) -> OperandRef<'tcx, V> {
+        let OperandRef { val, layout } = *self;
+
+        let unwrap = |r: Result<V, abi::Scalar>| match r {
+            Ok(v) => v,
+            Err(s) if s.is_uninit_valid() => {
+                let bty = cx.type_from_scalar(s);
+                cx.const_undef(bty)
+            }
+            Err(_) => bug!("OperandRef::finalize called while fields are missing {self:?}"),
+        };
+
+        let val = match val {
+            OperandValue::ZeroSized => OperandValue::ZeroSized,
+            OperandValue::Immediate(v) => OperandValue::Immediate(unwrap(v)),
+            OperandValue::Pair(a, b) => OperandValue::Pair(unwrap(a), unwrap(b)),
+            OperandValue::Ref(_) => bug!(),
+        };
+        OperandRef { val, layout }
+    }
 }
 
 impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
@@ -808,3 +901,93 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
     }
 }
+
+/// Transmutes one of the immediates from an [`OperandValue::Immediate`]
+/// or an [`OperandValue::Pair`] to an immediate of the target type.
+///
+/// `to_backend_ty` must be the *non*-immediate backend type (so it will be
+/// `i8`, not `i1`, for `bool`-like types.)
+pub(super) fn transmute_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+    bx: &mut Bx,
+    mut imm: Bx::Value,
+    from_scalar: abi::Scalar,
+    from_backend_ty: Bx::Type,
+    to_scalar: abi::Scalar,
+    to_backend_ty: Bx::Type,
+) -> Bx::Value {
+    assert_eq!(from_scalar.size(bx.cx()), to_scalar.size(bx.cx()));
+
+    // While optimizations will remove no-op transmutes, they might still be
+    // there in debug or things that aren't no-op in MIR because they change
+    // the Rust type but not the underlying layout/niche.
+    if from_scalar == to_scalar && from_backend_ty == to_backend_ty {
+        return imm;
+    }
+
+    use abi::Primitive::*;
+    imm = bx.from_immediate(imm);
+
+    // If we have a scalar, we must already know its range. Either
+    //
+    // 1) It's a parameter with `range` parameter metadata,
+    // 2) It's something we `load`ed with `!range` metadata, or
+    // 3) After a transmute we `assume`d the range (see below).
+    //
+    // That said, last time we tried removing this, it didn't actually help
+    // the rustc-perf results, so might as well keep doing it
+    // <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182>
+    assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
+
+    imm = match (from_scalar.primitive(), to_scalar.primitive()) {
+        (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty),
+        (Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty),
+        (Int(..), Pointer(..)) => bx.ptradd(bx.const_null(bx.type_ptr()), imm),
+        (Pointer(..), Int(..)) => {
+            // FIXME: this exposes the provenance, which shouldn't be necessary.
+            bx.ptrtoint(imm, to_backend_ty)
+        }
+        (Float(_), Pointer(..)) => {
+            let int_imm = bx.bitcast(imm, bx.cx().type_isize());
+            bx.ptradd(bx.const_null(bx.type_ptr()), int_imm)
+        }
+        (Pointer(..), Float(_)) => {
+            // FIXME: this exposes the provenance, which shouldn't be necessary.
+            let int_imm = bx.ptrtoint(imm, bx.cx().type_isize());
+            bx.bitcast(int_imm, to_backend_ty)
+        }
+    };
+
+    // This `assume` remains important for cases like (a conceptual)
+    //    transmute::<u32, NonZeroU32>(x) == 0
+    // since it's never passed to something with parameter metadata (especially
+    // after MIR inlining) so the only way to tell the backend about the
+    // constraint that the `transmute` introduced is to `assume` it.
+    assume_scalar_range(bx, imm, to_scalar, to_backend_ty);
+
+    imm = bx.to_immediate_scalar(imm, to_scalar);
+    imm
+}
+
+pub(super) fn assume_scalar_range<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+    bx: &mut Bx,
+    imm: Bx::Value,
+    scalar: abi::Scalar,
+    backend_ty: Bx::Type,
+) {
+    if matches!(bx.cx().sess().opts.optimize, OptLevel::No) || scalar.is_always_valid(bx.cx()) {
+        return;
+    }
+
+    match scalar.primitive() {
+        abi::Primitive::Int(..) => {
+            let range = scalar.valid_range(bx.cx());
+            bx.assume_integer_range(imm, backend_ty, range);
+        }
+        abi::Primitive::Pointer(abi::AddressSpace::DATA)
+            if !scalar.valid_range(bx.cx()).contains(0) =>
+        {
+            bx.assume_nonnull(imm);
+        }
+        abi::Primitive::Pointer(..) | abi::Primitive::Float(..) => {}
+    }
+}

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -1,4 +1,6 @@
-use rustc_abi::{Align, BackendRepr, FieldsShape, Size, TagEncoding, VariantIdx, Variants};
+use rustc_abi::{
+    Align, BackendRepr, FieldIdx, FieldsShape, Size, TagEncoding, VariantIdx, Variants,
+};
 use rustc_middle::mir::PlaceTy;
 use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
@@ -239,53 +241,17 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         bx: &mut Bx,
         variant_index: VariantIdx,
     ) {
-        if self.layout.for_variant(bx.cx(), variant_index).is_uninhabited() {
-            // We play it safe by using a well-defined `abort`, but we could go for immediate UB
-            // if that turns out to be helpful.
-            bx.abort();
-            return;
-        }
-        match self.layout.variants {
-            Variants::Empty => unreachable!("we already handled uninhabited types"),
-            Variants::Single { index } => assert_eq!(index, variant_index),
-
-            Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => {
-                let ptr = self.project_field(bx, tag_field);
-                let to =
-                    self.layout.ty.discriminant_for_variant(bx.tcx(), variant_index).unwrap().val;
-                bx.store_to_place(
-                    bx.cx().const_uint_big(bx.cx().backend_type(ptr.layout), to),
-                    ptr.val,
-                );
+        match codegen_tagged_field_value(bx.cx(), variant_index, self.layout) {
+            Err(UninhabitedVariantError) => {
+                // We play it safe by using a well-defined `abort`, but we could go for immediate UB
+                // if that turns out to be helpful.
+                bx.abort();
             }
-            Variants::Multiple {
-                tag_encoding:
-                    TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
-                tag_field,
-                ..
-            } => {
-                if variant_index != untagged_variant {
-                    let niche = self.project_field(bx, tag_field);
-                    let niche_llty = bx.cx().immediate_backend_type(niche.layout);
-                    let BackendRepr::Scalar(scalar) = niche.layout.backend_repr else {
-                        bug!("expected a scalar placeref for the niche");
-                    };
-                    // We are supposed to compute `niche_value.wrapping_add(niche_start)` wrapping
-                    // around the `niche`'s type.
-                    // The easiest way to do that is to do wrapping arithmetic on `u128` and then
-                    // masking off any extra bits that occur because we did the arithmetic with too many bits.
-                    let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
-                    let niche_value = (niche_value as u128).wrapping_add(niche_start);
-                    let niche_value = niche_value & niche.layout.size.unsigned_int_max();
-
-                    let niche_llval = bx.cx().scalar_to_backend(
-                        Scalar::from_uint(niche_value, niche.layout.size),
-                        scalar,
-                        niche_llty,
-                    );
-                    OperandValue::Immediate(niche_llval).store(bx, niche);
-                }
+            Ok(Some((tag_field, imm))) => {
+                let tag_place = self.project_field(bx, tag_field.as_usize());
+                OperandValue::Immediate(imm).store(bx, tag_place);
             }
+            Ok(None) => {}
         }
     }
 
@@ -471,3 +437,73 @@ fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let offset = bx.and(neg_value, align_minus_1);
     bx.add(value, offset)
 }
+
+/// Calculates the value that needs to be stored to mark the discriminant.
+///
+/// This might be `None` for a `struct` or a niched variant (like `Some(&3)`).
+///
+/// If it's `Some`, it returns the value to store and the field in which to
+/// store it. Note that this value is *not* the same as the discriminant, in
+/// general, as it might be a niche value or have a different size.
+///
+/// It might also be an `Err` because the variant is uninhabited.
+pub(super) fn codegen_tagged_field_value<'tcx, V>(
+    cx: &impl CodegenMethods<'tcx, Value = V>,
+    variant_index: VariantIdx,
+    layout: TyAndLayout<'tcx>,
+) -> Result<Option<(FieldIdx, V)>, UninhabitedVariantError> {
+    // By checking uninhabited-ness first we don't need to worry about types
+    // like `(u32, !)` which are single-variant but weird.
+    if layout.for_variant(cx, variant_index).is_uninhabited() {
+        return Err(UninhabitedVariantError);
+    }
+
+    Ok(match layout.variants {
+        Variants::Empty => unreachable!("we already handled uninhabited types"),
+        Variants::Single { index } => {
+            assert_eq!(index, variant_index);
+            None
+        }
+
+        Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => {
+            let discr = layout.ty.discriminant_for_variant(cx.tcx(), variant_index);
+            let to = discr.unwrap().val;
+            let tag_layout = layout.field(cx, tag_field);
+            let tag_llty = cx.immediate_backend_type(tag_layout);
+            let imm = cx.const_uint_big(tag_llty, to);
+            Some((FieldIdx::from_usize(tag_field), imm))
+        }
+        Variants::Multiple {
+            tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
+            tag_field,
+            ..
+        } => {
+            if variant_index != untagged_variant {
+                let niche_layout = layout.field(cx, tag_field);
+                let niche_llty = cx.immediate_backend_type(niche_layout);
+                let BackendRepr::Scalar(scalar) = niche_layout.backend_repr else {
+                    bug!("expected a scalar placeref for the niche");
+                };
+                // We are supposed to compute `niche_value.wrapping_add(niche_start)` wrapping
+                // around the `niche`'s type.
+                // The easiest way to do that is to do wrapping arithmetic on `u128` and then
+                // masking off any extra bits that occur because we did the arithmetic with too many bits.
+                let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
+                let niche_value = (niche_value as u128).wrapping_add(niche_start);
+                let niche_value = niche_value & niche_layout.size.unsigned_int_max();
+
+                let niche_llval = cx.scalar_to_backend(
+                    Scalar::from_uint(niche_value, niche_layout.size),
+                    scalar,
+                    niche_llty,
+                );
+                Some((FieldIdx::from_usize(tag_field), niche_llval))
+            } else {
+                None
+            }
+        }
+    })
+}
+
+#[derive(Debug)]
+pub(super) struct UninhabitedVariantError;

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -1,7 +1,6 @@
 use std::assert_matches::assert_matches;
 
-use arrayvec::ArrayVec;
-use rustc_abi::{self as abi, FIRST_VARIANT, FieldIdx};
+use rustc_abi::{self as abi, FIRST_VARIANT};
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
@@ -10,8 +9,8 @@ use rustc_session::config::OptLevel;
 use rustc_span::{DUMMY_SP, Span};
 use tracing::{debug, instrument};
 
-use super::operand::{OperandRef, OperandValue};
-use super::place::PlaceRef;
+use super::operand::{OperandRef, OperandValue, assume_scalar_range, transmute_immediate};
+use super::place::{PlaceRef, codegen_tagged_field_value};
 use super::{FunctionCx, LocalRef};
 use crate::common::IntPredicate;
 use crate::traits::*;
@@ -261,7 +260,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 {
                     let from_backend_ty = bx.backend_type(operand.layout);
                     let to_backend_ty = bx.backend_type(cast);
-                    Some(OperandValue::Immediate(self.transmute_immediate(
+                    Some(OperandValue::Immediate(transmute_immediate(
                         bx,
                         imm,
                         from_scalar,
@@ -286,8 +285,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     let out_a_ibty = bx.scalar_pair_element_backend_type(cast, 0, false);
                     let out_b_ibty = bx.scalar_pair_element_backend_type(cast, 1, false);
                     Some(OperandValue::Pair(
-                        self.transmute_immediate(bx, imm_a, in_a, in_a_ibty, out_a, out_a_ibty),
-                        self.transmute_immediate(bx, imm_b, in_b, in_b_ibty, out_b, out_b_ibty),
+                        transmute_immediate(bx, imm_a, in_a, in_a_ibty, out_a, out_a_ibty),
+                        transmute_immediate(bx, imm_b, in_b, in_b_ibty, out_b, out_b_ibty),
                     ))
                 } else {
                     None
@@ -301,7 +300,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ///
     /// Returns `None` if the cast is not possible.
     fn cast_immediate(
-        &self,
         bx: &mut Bx,
         mut imm: Bx::Value,
         from_scalar: abi::Scalar,
@@ -315,7 +313,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // valid ranges. For example, `char`s are passed as just `i32`, with no
         // way for LLVM to know that they're 0x10FFFF at most. Thus we assume
         // the range of the input value too, not just the output range.
-        self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
+        assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
 
         imm = match (from_scalar.primitive(), to_scalar.primitive()) {
             (Int(_, is_signed), Int(..)) => bx.intcast(imm, to_backend_ty, is_signed),
@@ -348,98 +346,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         Some(imm)
     }
 
-    /// Transmutes one of the immediates from an [`OperandValue::Immediate`]
-    /// or an [`OperandValue::Pair`] to an immediate of the target type.
-    ///
-    /// `to_backend_ty` must be the *non*-immediate backend type (so it will be
-    /// `i8`, not `i1`, for `bool`-like types.)
-    fn transmute_immediate(
-        &self,
-        bx: &mut Bx,
-        mut imm: Bx::Value,
-        from_scalar: abi::Scalar,
-        from_backend_ty: Bx::Type,
-        to_scalar: abi::Scalar,
-        to_backend_ty: Bx::Type,
-    ) -> Bx::Value {
-        assert_eq!(from_scalar.size(self.cx), to_scalar.size(self.cx));
-
-        // While optimizations will remove no-op transmutes, they might still be
-        // there in debug or things that aren't no-op in MIR because they change
-        // the Rust type but not the underlying layout/niche.
-        if from_scalar == to_scalar && from_backend_ty == to_backend_ty {
-            return imm;
-        }
-
-        use abi::Primitive::*;
-        imm = bx.from_immediate(imm);
-
-        // If we have a scalar, we must already know its range. Either
-        //
-        // 1) It's a parameter with `range` parameter metadata,
-        // 2) It's something we `load`ed with `!range` metadata, or
-        // 3) After a transmute we `assume`d the range (see below).
-        //
-        // That said, last time we tried removing this, it didn't actually help
-        // the rustc-perf results, so might as well keep doing it
-        // <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182>
-        self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
-
-        imm = match (from_scalar.primitive(), to_scalar.primitive()) {
-            (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty),
-            (Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty),
-            (Int(..), Pointer(..)) => bx.ptradd(bx.const_null(bx.type_ptr()), imm),
-            (Pointer(..), Int(..)) => {
-                // FIXME: this exposes the provenance, which shouldn't be necessary.
-                bx.ptrtoint(imm, to_backend_ty)
-            }
-            (Float(_), Pointer(..)) => {
-                let int_imm = bx.bitcast(imm, bx.cx().type_isize());
-                bx.ptradd(bx.const_null(bx.type_ptr()), int_imm)
-            }
-            (Pointer(..), Float(_)) => {
-                // FIXME: this exposes the provenance, which shouldn't be necessary.
-                let int_imm = bx.ptrtoint(imm, bx.cx().type_isize());
-                bx.bitcast(int_imm, to_backend_ty)
-            }
-        };
-
-        // This `assume` remains important for cases like (a conceptual)
-        //    transmute::<u32, NonZeroU32>(x) == 0
-        // since it's never passed to something with parameter metadata (especially
-        // after MIR inlining) so the only way to tell the backend about the
-        // constraint that the `transmute` introduced is to `assume` it.
-        self.assume_scalar_range(bx, imm, to_scalar, to_backend_ty);
-
-        imm = bx.to_immediate_scalar(imm, to_scalar);
-        imm
-    }
-
-    fn assume_scalar_range(
-        &self,
-        bx: &mut Bx,
-        imm: Bx::Value,
-        scalar: abi::Scalar,
-        backend_ty: Bx::Type,
-    ) {
-        if matches!(self.cx.sess().opts.optimize, OptLevel::No) || scalar.is_always_valid(self.cx) {
-            return;
-        }
-
-        match scalar.primitive() {
-            abi::Primitive::Int(..) => {
-                let range = scalar.valid_range(self.cx);
-                bx.assume_integer_range(imm, backend_ty, range);
-            }
-            abi::Primitive::Pointer(abi::AddressSpace::DATA)
-                if !scalar.valid_range(self.cx).contains(0) =>
-            {
-                bx.assume_nonnull(imm);
-            }
-            abi::Primitive::Pointer(..) | abi::Primitive::Float(..) => {}
-        }
-    }
-
     pub(crate) fn codegen_rvalue_unsized(
         &mut self,
         bx: &mut Bx,
@@ -578,7 +484,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             bug!("Found {cast_kind:?} for operand {cast:?}");
                         };
 
-                        self.cast_immediate(bx, imm, from_scalar, from_backend_ty, to_scalar, to_backend_ty)
+                        Self::cast_immediate(bx, imm, from_scalar, from_backend_ty, to_scalar, to_backend_ty)
                             .map(OperandValue::Immediate)
                             .unwrap_or_else(|| {
                                 bug!("Unsupported cast of {operand:?} to {cast:?}");
@@ -776,45 +682,42 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
             mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand),
             mir::Rvalue::Repeat(..) => bug!("{rvalue:?} in codegen_rvalue_operand"),
-            mir::Rvalue::Aggregate(_, ref fields) => {
+            mir::Rvalue::Aggregate(ref kind, ref fields) => {
+                let (variant_index, active_field_index) = match **kind {
+                    mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
+                        (variant_index, active_field_index)
+                    }
+                    _ => (FIRST_VARIANT, None),
+                };
+
                 let ty = rvalue.ty(self.mir, self.cx.tcx());
                 let ty = self.monomorphize(ty);
                 let layout = self.cx.layout_of(ty);
 
-                // `rvalue_creates_operand` has arranged that we only get here if
-                // we can build the aggregate immediate from the field immediates.
-                let mut inputs = ArrayVec::<Bx::Value, 2>::new();
-                let mut input_scalars = ArrayVec::<abi::Scalar, 2>::new();
-                for field_idx in layout.fields.index_by_increasing_offset() {
-                    let field_idx = FieldIdx::from_usize(field_idx);
-                    let op = self.codegen_operand(bx, &fields[field_idx]);
-                    let values = op.val.immediates_or_place().left_or_else(|p| {
-                        bug!("Field {field_idx:?} is {p:?} making {layout:?}");
-                    });
-                    let scalars = self.value_kind(op.layout).scalars().unwrap();
-                    assert_eq!(values.len(), scalars.len());
-                    inputs.extend(values);
-                    input_scalars.extend(scalars);
+                let mut builder = OperandRef::builder(layout);
+                for (field_idx, field) in fields.iter_enumerated() {
+                    let op = self.codegen_operand(bx, field);
+                    let fi = active_field_index.unwrap_or(field_idx);
+                    builder.insert_field(bx, variant_index, fi, op);
                 }
 
-                let output_scalars = self.value_kind(layout).scalars().unwrap();
-                itertools::izip!(&mut inputs, input_scalars, output_scalars).for_each(
-                    |(v, in_s, out_s)| {
-                        if in_s != out_s {
-                            // We have to be really careful about bool here, because
-                            // `(bool,)` stays i1 but `Cell<bool>` becomes i8.
-                            *v = bx.from_immediate(*v);
-                            *v = bx.to_immediate_scalar(*v, out_s);
+                let tag_result = codegen_tagged_field_value(self.cx, variant_index, layout);
+                match tag_result {
+                    Err(super::place::UninhabitedVariantError) => {
+                        // Like codegen_set_discr we use a sound abort, but could
+                        // potentially `unreachable` or just return the poison for
+                        // more optimizability, if that turns out to be helpful.
+                        bx.abort();
+                        let val = OperandValue::poison(bx, layout);
+                        OperandRef { val, layout }
+                    }
+                    Ok(maybe_tag_value) => {
+                        if let Some((tag_field, tag_imm)) = maybe_tag_value {
+                            builder.insert_imm(tag_field, tag_imm);
                         }
-                    },
-                );
-
-                let val = OperandValue::from_immediates(inputs);
-                assert!(
-                    val.is_expected_variant_for_type(self.cx, layout),
-                    "Made wrong variant {val:?} for type {layout:?}",
-                );
-                OperandRef { val, layout }
+                        builder.finalize(bx.cx())
+                    }
+                }
             }
             mir::Rvalue::ShallowInitBox(ref operand, content_ty) => {
                 let operand = self.codegen_operand(bx, operand);
@@ -1145,19 +1048,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     mir::AggregateKind::RawPtr(..) => true,
                     mir::AggregateKind::Array(..) => false,
                     mir::AggregateKind::Tuple => true,
-                    mir::AggregateKind::Adt(def_id, ..) => {
-                        let adt_def = self.cx.tcx().adt_def(def_id);
-                        adt_def.is_struct() && !adt_def.repr().simd()
-                    }
+                    mir::AggregateKind::Adt(..) => true,
                     mir::AggregateKind::Closure(..) => true,
                     // FIXME: Can we do this for simple coroutines too?
                     mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false,
                 };
                 allowed_kind && {
-                let ty = rvalue.ty(self.mir, self.cx.tcx());
-                let ty = self.monomorphize(ty);
+                    let ty = rvalue.ty(self.mir, self.cx.tcx());
+                    let ty = self.monomorphize(ty);
                     let layout = self.cx.spanned_layout_of(ty, span);
-                    !self.cx.is_backend_ref(layout)
+                    OperandRef::<Bx::Value>::supports_builder(layout)
                 }
             }
         }
@@ -1200,14 +1100,3 @@ enum OperandValueKind {
     Pair(abi::Scalar, abi::Scalar),
     ZeroSized,
 }
-
-impl OperandValueKind {
-    fn scalars(self) -> Option<ArrayVec<abi::Scalar, 2>> {
-        Some(match self {
-            OperandValueKind::ZeroSized => ArrayVec::new(),
-            OperandValueKind::Immediate(a) => ArrayVec::from_iter([a]),
-            OperandValueKind::Pair(a, b) => [a, b].into(),
-            OperandValueKind::Ref => return None,
-        })
-    }
-}

compiler/rustc_codegen_ssa/src/traits/type_.rs

@@ -1,4 +1,4 @@
-use rustc_abi::{AddressSpace, Float, Integer, Reg};
+use rustc_abi::{AddressSpace, Float, Integer, Primitive, Reg, Scalar};
 use rustc_middle::bug;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, TyAndLayout};
@@ -73,6 +73,24 @@ pub trait DerivedTypeCodegenMethods<'tcx>:
         }
     }
 
+    fn type_from_primitive(&self, p: Primitive) -> Self::Type {
+        use Primitive::*;
+        match p {
+            Int(i, _) => self.type_from_integer(i),
+            Float(f) => self.type_from_float(f),
+            Pointer(address_space) => self.type_ptr_ext(address_space),
+        }
+    }
+
+    fn type_from_scalar(&self, s: Scalar) -> Self::Type {
+        // `MaybeUninit` being `repr(transparent)` somewhat implies that the type
+        // of a scalar has to be the type of its primitive (which is true in LLVM,
+        // where noundef is a parameter attribute or metadata) but if we ever get
+        // a backend where that's no longer true, every use of this will need to
+        // to carefully scrutinized and re-evaluated.
+        self.type_from_primitive(s.primitive())
+    }
+
     fn type_needs_drop(&self, ty: Ty<'tcx>) -> bool {
         ty.needs_drop(self.tcx(), self.typing_env())
     }

tests/codegen/align-struct.rs

@@ -13,9 +13,11 @@ pub struct Nested64 {
     d: i8,
 }
 
+// This has the extra field in B to ensure it's not ScalarPair,
+// and thus that the test actually emits it via memory, not `insertvalue`.
 pub enum Enum4 {
     A(i32),
-    B(i32),
+    B(i32, i32),
 }
 
 pub enum Enum64 {
@@ -52,7 +54,7 @@ pub fn nested64(a: Align64, b: i32, c: i32, d: i8) -> Nested64 {
 // CHECK-LABEL: @enum4
 #[no_mangle]
 pub fn enum4(a: i32) -> Enum4 {
-    // CHECK: %e4 = alloca [8 x i8], align 4
+    // CHECK: %e4 = alloca [12 x i8], align 4
     let e4 = Enum4::A(a);
     e4
 }

tests/codegen/common_prim_int_ptr.rs

@@ -11,9 +11,9 @@
 #[no_mangle]
 pub fn insert_int(x: usize) -> Result<usize, Box<()>> {
     // CHECK: start:
-    // CHECK-NEXT: inttoptr i{{[0-9]+}} %x to ptr
-    // CHECK-NEXT: insertvalue
-    // CHECK-NEXT: ret { i{{[0-9]+}}, ptr }
+    // CHECK-NEXT: %[[WO_PROV:.+]] = getelementptr i8, ptr null, [[USIZE:i[0-9]+]] %x
+    // CHECK-NEXT: %[[R:.+]] = insertvalue { [[USIZE]], ptr } { [[USIZE]] 0, ptr poison }, ptr %[[WO_PROV]], 1
+    // CHECK-NEXT: ret { [[USIZE]], ptr } %[[R]]
     Ok(x)
 }
 

tests/codegen/enum/enum-aggregate.rs

@@ -0,0 +1,94 @@
+//@ compile-flags: -Copt-level=0 -Cno-prepopulate-passes
+//@ min-llvm-version: 19
+//@ only-64bit
+
+#![crate_type = "lib"]
+
+use std::cmp::Ordering;
+use std::num::NonZero;
+use std::ptr::NonNull;
+
+#[no_mangle]
+fn make_some_bool(x: bool) -> Option<bool> {
+    // CHECK-LABEL: i8 @make_some_bool(i1 zeroext %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[WIDER:.+]] = zext i1 %x to i8
+    // CHECK-NEXT: ret i8 %[[WIDER]]
+    Some(x)
+}
+
+#[no_mangle]
+fn make_none_bool() -> Option<bool> {
+    // CHECK-LABEL: i8 @make_none_bool()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i8 2
+    None
+}
+
+#[no_mangle]
+fn make_some_ordering(x: Ordering) -> Option<Ordering> {
+    // CHECK-LABEL: i8 @make_some_ordering(i8 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i8 %x
+    Some(x)
+}
+
+#[no_mangle]
+fn make_some_u16(x: u16) -> Option<u16> {
+    // CHECK-LABEL: { i16, i16 } @make_some_u16(i16 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { i16, i16 } { i16 1, i16 poison }, i16 %x, 1
+    // CHECK-NEXT: ret { i16, i16 } %0
+    Some(x)
+}
+
+#[no_mangle]
+fn make_none_u16() -> Option<u16> {
+    // CHECK-LABEL: { i16, i16 } @make_none_u16()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret { i16, i16 } { i16 0, i16 undef }
+    None
+}
+
+#[no_mangle]
+fn make_some_nzu32(x: NonZero<u32>) -> Option<NonZero<u32>> {
+    // CHECK-LABEL: i32 @make_some_nzu32(i32 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i32 %x
+    Some(x)
+}
+
+#[no_mangle]
+fn make_ok_ptr(x: NonNull<u16>) -> Result<NonNull<u16>, usize> {
+    // CHECK-LABEL: { i64, ptr } @make_ok_ptr(ptr %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { i64, ptr } { i64 0, ptr poison }, ptr %x, 1
+    // CHECK-NEXT: ret { i64, ptr } %0
+    Ok(x)
+}
+
+#[no_mangle]
+fn make_ok_int(x: usize) -> Result<usize, NonNull<u16>> {
+    // CHECK-LABEL: { i64, ptr } @make_ok_int(i64 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[NOPROV:.+]] = getelementptr i8, ptr null, i64 %x
+    // CHECK-NEXT: %[[R:.+]] = insertvalue { i64, ptr } { i64 0, ptr poison }, ptr %[[NOPROV]], 1
+    // CHECK-NEXT: ret { i64, ptr } %[[R]]
+    Ok(x)
+}
+
+#[no_mangle]
+fn make_some_ref(x: &u16) -> Option<&u16> {
+    // CHECK-LABEL: ptr @make_some_ref(ptr align 2 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr %x
+    Some(x)
+}
+
+#[no_mangle]
+fn make_none_ref<'a>() -> Option<&'a u16> {
+    // CHECK-LABEL: ptr @make_none_ref()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr null
+    None
+}

tests/codegen/set-discriminant-invalid.rs

@@ -16,10 +16,9 @@ impl IntoError<Error> for Api {
     type Source = ApiError;
     // CHECK-LABEL: @into_error
     // CHECK: llvm.trap()
-    // Also check the next two instructions to make sure we do not match against `trap`
+    // Also check the next instruction to make sure we do not match against `trap`
     // elsewhere in the code.
-    // CHECK-NEXT: load
-    // CHECK-NEXT: ret
+    // CHECK-NEXT: ret i8 poison
     #[no_mangle]
     fn into_error(self, error: Self::Source) -> Error {
         Error::Api { source: error }

tests/codegen/union-aggregate.rs

@@ -0,0 +1,85 @@
+//@ compile-flags: -Copt-level=0 -Cno-prepopulate-passes
+//@ min-llvm-version: 19
+//@ only-64bit
+
+#![crate_type = "lib"]
+#![feature(transparent_unions)]
+
+#[repr(transparent)]
+union MU<T: Copy> {
+    uninit: (),
+    value: T,
+}
+
+use std::cmp::Ordering;
+use std::num::NonZero;
+use std::ptr::NonNull;
+
+#[no_mangle]
+fn make_mu_bool(x: bool) -> MU<bool> {
+    // CHECK-LABEL: i8 @make_mu_bool(i1 zeroext %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[WIDER:.+]] = zext i1 %x to i8
+    // CHECK-NEXT: ret i8 %[[WIDER]]
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_bool_uninit() -> MU<bool> {
+    // CHECK-LABEL: i8 @make_mu_bool_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret i8 undef
+    MU { uninit: () }
+}
+
+#[no_mangle]
+fn make_mu_ref(x: &u16) -> MU<&u16> {
+    // CHECK-LABEL: ptr @make_mu_ref(ptr align 2 %x)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr %x
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_ref_uninit<'a>() -> MU<&'a u16> {
+    // CHECK-LABEL: ptr @make_mu_ref_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret ptr undef
+    MU { uninit: () }
+}
+
+#[no_mangle]
+fn make_mu_str(x: &str) -> MU<&str> {
+    // CHECK-LABEL: { ptr, i64 } @make_mu_str(ptr align 1 %x.0, i64 %x.1)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { ptr, i64 } poison, ptr %x.0, 0
+    // CHECK-NEXT: %1 = insertvalue { ptr, i64 } %0, i64 %x.1, 1
+    // CHECK-NEXT: ret { ptr, i64 } %1
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_str_uninit<'a>() -> MU<&'a str> {
+    // CHECK-LABEL: { ptr, i64 } @make_mu_str_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret { ptr, i64 } undef
+    MU { uninit: () }
+}
+
+#[no_mangle]
+fn make_mu_pair(x: (u8, u32)) -> MU<(u8, u32)> {
+    // CHECK-LABEL: { i8, i32 } @make_mu_pair(i8 %x.0, i32 %x.1)
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %0 = insertvalue { i8, i32 } poison, i8 %x.0, 0
+    // CHECK-NEXT: %1 = insertvalue { i8, i32 } %0, i32 %x.1, 1
+    // CHECK-NEXT: ret { i8, i32 } %1
+    MU { value: x }
+}
+
+#[no_mangle]
+fn make_mu_pair_uninit() -> MU<(u8, u32)> {
+    // CHECK-LABEL: { i8, i32 } @make_mu_pair_uninit()
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: ret { i8, i32 } undef
+    MU { uninit: () }
+}