diff --git a/compiler/rustc_ast/src/expand/autodiff_attrs.rs b/compiler/rustc_ast/src/expand/autodiff_attrs.rs
index c8ec185ee5e29..f01c781f46c65 100644
--- a/compiler/rustc_ast/src/expand/autodiff_attrs.rs
+++ b/compiler/rustc_ast/src/expand/autodiff_attrs.rs
@@ -77,6 +77,17 @@ pub struct AutoDiffAttrs {
/// e.g. in the [JAX
/// Documentation](https://jax.readthedocs.io/en/latest/_tutorials/advanced-autodiff.html#how-it-s-made-two-foundational-autodiff-functions).
pub mode: DiffMode,
+ /// A user-provided, batching width. If not given, we will default to 1 (no batching).
+ /// Calling a differentiated, non-batched function through a loop 100 times is equivalent to:
+ /// - Calling the function 50 times with a batch size of 2
+ /// - Calling the function 25 times with a batch size of 4,
+ /// etc. A batched function takes more (or longer) arguments, and might be able to benefit from
+ /// cache locality, better re-usal of primal values, and other optimizations.
+ /// We will (before LLVM's vectorizer runs) just generate most LLVM-IR instructions `width`
+ /// times, so this massively increases code size. As such, values like 1024 are unlikely to
+ /// work. We should consider limiting this to u8 or u16, but will leave it at u32 for
+ /// experiments for now and focus on documenting the implications of a large width.
+ pub width: u32,
pub ret_activity: DiffActivity,
pub input_activity: Vec<DiffActivity>,
}
@@ -222,6 +233,7 @@ impl AutoDiffAttrs {
pub const fn error() -> Self {
AutoDiffAttrs {
mode: DiffMode::Error,
+ width: 0,
ret_activity: DiffActivity::None,
input_activity: Vec::new(),
}
@@ -229,6 +241,7 @@ impl AutoDiffAttrs {
pub fn source() -> Self {
AutoDiffAttrs {
mode: DiffMode::Source,
+ width: 0,
ret_activity: DiffActivity::None,
input_activity: Vec::new(),
}
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 28f596ac0926d..958a6917dff87 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -676,12 +676,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
let ty =
self.lower_ty(ty, ImplTraitContext::Disallowed(ImplTraitPosition::StaticTy));
let safety = self.lower_safety(*safety, hir::Safety::Unsafe);
-
- // njn: where for this?
if define_opaque.is_some() {
self.dcx().span_err(i.span, "foreign statics cannot define opaque types");
}
-
(ident, hir::ForeignItemKind::Static(ty, *mutability, safety))
}
ForeignItemKind::TyAlias(box TyAlias { ident, .. }) => {
diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl
index 3f03834f8d781..603dc90bafca2 100644
--- a/compiler/rustc_builtin_macros/messages.ftl
+++ b/compiler/rustc_builtin_macros/messages.ftl
@@ -79,6 +79,7 @@ builtin_macros_autodiff_ret_activity = invalid return activity {$act} in {$mode}
builtin_macros_autodiff_ty_activity = {$act} can not be used for this type
builtin_macros_autodiff_unknown_activity = did not recognize Activity: `{$act}`
+builtin_macros_autodiff_width = autodiff width must fit u32, but is {$width}
builtin_macros_bad_derive_target = `derive` may only be applied to `struct`s, `enum`s and `union`s
.label = not applicable here
.label2 = not a `struct`, `enum` or `union`
diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs
index 8937d35d53aed..7f99f75b2b9df 100644
--- a/compiler/rustc_builtin_macros/src/autodiff.rs
+++ b/compiler/rustc_builtin_macros/src/autodiff.rs
@@ -12,12 +12,12 @@ mod llvm_enzyme {
valid_ty_for_activity,
};
use rustc_ast::ptr::P;
- use rustc_ast::token::{Token, TokenKind};
+ use rustc_ast::token::{Lit, LitKind, Token, TokenKind};
use rustc_ast::tokenstream::*;
use rustc_ast::visit::AssocCtxt::*;
use rustc_ast::{
- self as ast, AssocItemKind, BindingMode, FnRetTy, FnSig, Generics, ItemKind, MetaItemInner,
- PatKind, TyKind,
+ self as ast, AssocItemKind, BindingMode, ExprKind, FnRetTy, FnSig, Generics, ItemKind,
+ MetaItemInner, PatKind, QSelf, TyKind,
};
use rustc_expand::base::{Annotatable, ExtCtxt};
use rustc_span::{Ident, Span, Symbol, kw, sym};
@@ -45,6 +45,16 @@ mod llvm_enzyme {
}
}
fn first_ident(x: &MetaItemInner) -> rustc_span::Ident {
+ if let Some(l) = x.lit() {
+ match l.kind {
+ ast::LitKind::Int(val, _) => {
+ // get an Ident from a lit
+ return rustc_span::Ident::from_str(val.get().to_string().as_str());
+ }
+ _ => {}
+ }
+ }
+
let segments = &x.meta_item().unwrap().path.segments;
assert!(segments.len() == 1);
segments[0].ident
@@ -54,6 +64,14 @@ mod llvm_enzyme {
first_ident(x).name.to_string()
}
+ fn width(x: &MetaItemInner) -> Option<u128> {
+ let lit = x.lit()?;
+ match lit.kind {
+ ast::LitKind::Int(x, _) => Some(x.get()),
+ _ => return None,
+ }
+ }
+
pub(crate) fn from_ast(
ecx: &mut ExtCtxt<'_>,
meta_item: &ThinVec<MetaItemInner>,
@@ -65,9 +83,32 @@ mod llvm_enzyme {
dcx.emit_err(errors::AutoDiffInvalidMode { span: meta_item[1].span(), mode });
return AutoDiffAttrs::error();
};
+
+ // Now we check, whether the user wants autodiff in batch/vector mode, or scalar mode.
+ // If he doesn't specify an integer (=width), we default to scalar mode, thus width=1.
+ let mut first_activity = 2;
+
+ let width = if let [_, _, x, ..] = &meta_item[..]
+ && let Some(x) = width(x)
+ {
+ first_activity = 3;
+ match x.try_into() {
+ Ok(x) => x,
+ Err(_) => {
+ dcx.emit_err(errors::AutoDiffInvalidWidth {
+ span: meta_item[2].span(),
+ width: x,
+ });
+ return AutoDiffAttrs::error();
+ }
+ }
+ } else {
+ 1
+ };
+
let mut activities: Vec<DiffActivity> = vec![];
let mut errors = false;
- for x in &meta_item[2..] {
+ for x in &meta_item[first_activity..] {
let activity_str = name(&x);
let res = DiffActivity::from_str(&activity_str);
match res {
@@ -98,7 +139,20 @@ mod llvm_enzyme {
(&DiffActivity::None, activities.as_slice())
};
- AutoDiffAttrs { mode, ret_activity: *ret_activity, input_activity: input_activity.to_vec() }
+ AutoDiffAttrs {
+ mode,
+ width,
+ ret_activity: *ret_activity,
+ input_activity: input_activity.to_vec(),
+ }
+ }
+
+ fn meta_item_inner_to_ts(t: &MetaItemInner, ts: &mut Vec<TokenTree>) {
+ let comma: Token = Token::new(TokenKind::Comma, Span::default());
+ let val = first_ident(t);
+ let t = Token::from_ast_ident(val);
+ ts.push(TokenTree::Token(t, Spacing::Joint));
+ ts.push(TokenTree::Token(comma.clone(), Spacing::Alone));
}
/// We expand the autodiff macro to generate a new placeholder function which passes
@@ -195,27 +249,49 @@ mod llvm_enzyme {
// create TokenStream from vec elemtents:
// meta_item doesn't have a .tokens field
- let comma: Token = Token::new(TokenKind::Comma, Span::default());
let mut ts: Vec<TokenTree> = vec![];
if meta_item_vec.len() < 2 {
// At the bare minimum, we need a fnc name and a mode, even for a dummy function with no
// input and output args.
dcx.emit_err(errors::AutoDiffMissingConfig { span: item.span() });
return vec![item];
+ }
+
+ meta_item_inner_to_ts(&meta_item_vec[1], &mut ts);
+
+ // Now, if the user gave a width (vector aka batch-mode ad), then we copy it.
+ // If it is not given, we default to 1 (scalar mode).
+ let start_position;
+ let kind: LitKind = LitKind::Integer;
+ let symbol;
+ if meta_item_vec.len() >= 3
+ && let Some(width) = width(&meta_item_vec[2])
+ {
+ start_position = 3;
+ symbol = Symbol::intern(&width.to_string());
} else {
- for t in meta_item_vec.clone()[1..].iter() {
- let val = first_ident(t);
- let t = Token::from_ast_ident(val);
- ts.push(TokenTree::Token(t, Spacing::Joint));
- ts.push(TokenTree::Token(comma.clone(), Spacing::Alone));
- }
+ start_position = 2;
+ symbol = sym::integer(1);
}
+ let l: Lit = Lit { kind, symbol, suffix: None };
+ let t = Token::new(TokenKind::Literal(l), Span::default());
+ let comma = Token::new(TokenKind::Comma, Span::default());
+ ts.push(TokenTree::Token(t, Spacing::Joint));
+ ts.push(TokenTree::Token(comma.clone(), Spacing::Alone));
+
+ for t in meta_item_vec.clone()[start_position..].iter() {
+ meta_item_inner_to_ts(t, &mut ts);
+ }
+
if !has_ret {
// We don't want users to provide a return activity if the function doesn't return anything.
// For simplicity, we just add a dummy token to the end of the list.
let t = Token::new(TokenKind::Ident(sym::None, false.into()), Span::default());
ts.push(TokenTree::Token(t, Spacing::Joint));
+ ts.push(TokenTree::Token(comma, Spacing::Alone));
}
+ // We remove the last, trailing comma.
+ ts.pop();
let ts: TokenStream = TokenStream::from_iter(ts);
let x: AutoDiffAttrs = from_ast(ecx, &meta_item_vec, has_ret);
@@ -470,6 +546,8 @@ mod llvm_enzyme {
return body;
}
+ // Everything from here onwards just tries to fullfil the return type. Fun!
+
// having an active-only return means we'll drop the original return type.
// So that can be treated identical to not having one in the first place.
let primal_ret = has_ret(&sig.decl.output) && !x.has_active_only_ret();
@@ -497,86 +575,65 @@ mod llvm_enzyme {
return body;
}
- let mut exprs = ThinVec::<P<ast::Expr>>::new();
- if primal_ret {
- // We have both primal ret and active floats.
- // primal ret is first, by construction.
- exprs.push(primal_call);
- }
-
- // Now construct default placeholder for each active float.
- // Is there something nicer than f32::default() and f64::default()?
+ let mut exprs: P<ast::Expr> = primal_call.clone();
let d_ret_ty = match d_sig.decl.output {
FnRetTy::Ty(ref ty) => ty.clone(),
FnRetTy::Default(span) => {
panic!("Did not expect Default ret ty: {:?}", span);
}
};
- let mut d_ret_ty = match d_ret_ty.kind.clone() {
- TyKind::Tup(ref tys) => tys.clone(),
- TyKind::Path(_, rustc_ast::Path { segments, .. }) => {
- if let [segment] = &segments[..]
- && segment.args.is_none()
- {
- let id = vec![segments[0].ident];
- let kind = TyKind::Path(None, ecx.path(span, id));
- let ty = P(rustc_ast::Ty { kind, id: ast::DUMMY_NODE_ID, span, tokens: None });
- thin_vec![ty]
- } else {
- panic!("Expected tuple or simple path return type");
- }
- }
- _ => {
- // We messed up construction of d_sig
- panic!("Did not expect non-tuple ret ty: {:?}", d_ret_ty);
- }
- };
-
- if x.mode.is_fwd() && x.ret_activity == DiffActivity::Dual {
- assert!(d_ret_ty.len() == 2);
- // both should be identical, by construction
- let arg = d_ret_ty[0].kind.is_simple_path().unwrap();
- let arg2 = d_ret_ty[1].kind.is_simple_path().unwrap();
- assert!(arg == arg2);
- let sl: Vec<Symbol> = vec![arg, kw::Default];
- let tmp = ecx.def_site_path(&sl);
- let default_call_expr = ecx.expr_path(ecx.path(span, tmp));
- let default_call_expr = ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]);
- exprs.push(default_call_expr);
- } else if x.mode.is_rev() {
- if primal_ret {
- // We have extra handling above for the primal ret
- d_ret_ty = d_ret_ty[1..].to_vec().into();
- }
- for arg in d_ret_ty.iter() {
- let arg = arg.kind.is_simple_path().unwrap();
- let sl: Vec<Symbol> = vec![arg, kw::Default];
- let tmp = ecx.def_site_path(&sl);
- let default_call_expr = ecx.expr_path(ecx.path(span, tmp));
+ if x.mode.is_fwd() {
+ // Fwd mode is easy. If the return activity is Const, we support arbitrary types.
+ // Otherwise, we only support a scalar, a pair of scalars, or an array of scalars.
+ // We checked that (on a best-effort base) in the preceding gen_enzyme_decl function.
+ // In all three cases, we can return `std::hint::black_box(<T>::default())`.
+ if x.ret_activity == DiffActivity::Const {
+ // Here we call the primal function, since our dummy function has the same return
+ // type due to the Const return activity.
+ exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]);
+ } else {
+ let q = QSelf { ty: d_ret_ty.clone(), path_span: span, position: 0 };
+ let y =
+ ExprKind::Path(Some(P(q)), ecx.path_ident(span, Ident::from_str("default")));
+ let default_call_expr = ecx.expr(span, y);
let default_call_expr =
ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]);
- exprs.push(default_call_expr);
- }
- }
-
- let ret: P<ast::Expr>;
- match &exprs[..] {
- [] => {
- assert!(!has_ret(&d_sig.decl.output));
- // We don't have to match the return type.
- return body;
- }
- [arg] => {
- ret = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![arg.clone()]);
+ exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![default_call_expr]);
}
- args => {
- let ret_tuple: P<ast::Expr> = ecx.expr_tuple(span, args.into());
- ret = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![ret_tuple]);
+ } else if x.mode.is_rev() {
+ if x.width == 1 {
+ // We either have `-> ArbitraryType` or `-> (ArbitraryType, repeated_float_scalars)`.
+ match d_ret_ty.kind {
+ TyKind::Tup(ref args) => {
+ // We have a tuple return type. We need to create a tuple of the same size
+ // and fill it with default values.
+ let mut exprs2 = thin_vec![exprs];
+ for arg in args.iter().skip(1) {
+ let arg = arg.kind.is_simple_path().unwrap();
+ let sl: Vec<Symbol> = vec![arg, kw::Default];
+ let tmp = ecx.def_site_path(&sl);
+ let default_call_expr = ecx.expr_path(ecx.path(span, tmp));
+ let default_call_expr =
+ ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]);
+ exprs2.push(default_call_expr);
+ }
+ exprs = ecx.expr_tuple(new_decl_span, exprs2);
+ }
+ _ => {
+ // Interestingly, even the `-> ArbitraryType` case
+ // ends up getting matched and handled correctly above,
+ // so we don't have to handle any other case for now.
+ panic!("Unsupported return type: {:?}", d_ret_ty);
+ }
+ }
}
+ exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]);
+ } else {
+ unreachable!("Unsupported mode: {:?}", x.mode);
}
- assert!(has_ret(&d_sig.decl.output));
- body.stmts.push(ecx.stmt_expr(ret));
+
+ body.stmts.push(ecx.stmt_expr(exprs));
body
}
@@ -684,50 +741,55 @@ mod llvm_enzyme {
match activity {
DiffActivity::Active => {
act_ret.push(arg.ty.clone());
+ // if width =/= 1, then push [arg.ty; width] to act_ret
}
DiffActivity::ActiveOnly => {
// We will add the active scalar to the return type.
// This is handled later.
}
DiffActivity::Duplicated | DiffActivity::DuplicatedOnly => {
- let mut shadow_arg = arg.clone();
- // We += into the shadow in reverse mode.
- shadow_arg.ty = P(assure_mut_ref(&arg.ty));
- let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
- ident.name
- } else {
- debug!("{:#?}", &shadow_arg.pat);
- panic!("not an ident?");
- };
- let name: String = format!("d{}", old_name);
- new_inputs.push(name.clone());
- let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
- shadow_arg.pat = P(ast::Pat {
- id: ast::DUMMY_NODE_ID,
- kind: PatKind::Ident(BindingMode::NONE, ident, None),
- span: shadow_arg.pat.span,
- tokens: shadow_arg.pat.tokens.clone(),
- });
- d_inputs.push(shadow_arg);
+ for i in 0..x.width {
+ let mut shadow_arg = arg.clone();
+ // We += into the shadow in reverse mode.
+ shadow_arg.ty = P(assure_mut_ref(&arg.ty));
+ let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
+ ident.name
+ } else {
+ debug!("{:#?}", &shadow_arg.pat);
+ panic!("not an ident?");
+ };
+ let name: String = format!("d{}_{}", old_name, i);
+ new_inputs.push(name.clone());
+ let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
+ shadow_arg.pat = P(ast::Pat {
+ id: ast::DUMMY_NODE_ID,
+ kind: PatKind::Ident(BindingMode::NONE, ident, None),
+ span: shadow_arg.pat.span,
+ tokens: shadow_arg.pat.tokens.clone(),
+ });
+ d_inputs.push(shadow_arg.clone());
+ }
}
DiffActivity::Dual | DiffActivity::DualOnly => {
- let mut shadow_arg = arg.clone();
- let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
- ident.name
- } else {
- debug!("{:#?}", &shadow_arg.pat);
- panic!("not an ident?");
- };
- let name: String = format!("b{}", old_name);
- new_inputs.push(name.clone());
- let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
- shadow_arg.pat = P(ast::Pat {
- id: ast::DUMMY_NODE_ID,
- kind: PatKind::Ident(BindingMode::NONE, ident, None),
- span: shadow_arg.pat.span,
- tokens: shadow_arg.pat.tokens.clone(),
- });
- d_inputs.push(shadow_arg);
+ for i in 0..x.width {
+ let mut shadow_arg = arg.clone();
+ let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
+ ident.name
+ } else {
+ debug!("{:#?}", &shadow_arg.pat);
+ panic!("not an ident?");
+ };
+ let name: String = format!("b{}_{}", old_name, i);
+ new_inputs.push(name.clone());
+ let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
+ shadow_arg.pat = P(ast::Pat {
+ id: ast::DUMMY_NODE_ID,
+ kind: PatKind::Ident(BindingMode::NONE, ident, None),
+ span: shadow_arg.pat.span,
+ tokens: shadow_arg.pat.tokens.clone(),
+ });
+ d_inputs.push(shadow_arg.clone());
+ }
}
DiffActivity::Const => {
// Nothing to do here.
@@ -783,23 +845,48 @@ mod llvm_enzyme {
d_decl.inputs = d_inputs.into();
if x.mode.is_fwd() {
+ let ty = match d_decl.output {
+ FnRetTy::Ty(ref ty) => ty.clone(),
+ FnRetTy::Default(span) => {
+ // We want to return std::hint::black_box(()).
+ let kind = TyKind::Tup(ThinVec::new());
+ let ty = P(rustc_ast::Ty { kind, id: ast::DUMMY_NODE_ID, span, tokens: None });
+ d_decl.output = FnRetTy::Ty(ty.clone());
+ assert!(matches!(x.ret_activity, DiffActivity::None));
+ // this won't be used below, so any type would be fine.
+ ty
+ }
+ };
+
if let DiffActivity::Dual = x.ret_activity {
- let ty = match d_decl.output {
- FnRetTy::Ty(ref ty) => ty.clone(),
- FnRetTy::Default(span) => {
- panic!("Did not expect Default ret ty: {:?}", span);
- }
+ let kind = if x.width == 1 {
+ // Dual can only be used for f32/f64 ret.
+ // In that case we return now a tuple with two floats.
+ TyKind::Tup(thin_vec![ty.clone(), ty.clone()])
+ } else {
+ // We have to return [T; width+1], +1 for the primal return.
+ let anon_const = rustc_ast::AnonConst {
+ id: ast::DUMMY_NODE_ID,
+ value: ecx.expr_usize(span, 1 + x.width as usize),
+ };
+ TyKind::Array(ty.clone(), anon_const)
};
- // Dual can only be used for f32/f64 ret.
- // In that case we return now a tuple with two floats.
- let kind = TyKind::Tup(thin_vec![ty.clone(), ty.clone()]);
let ty = P(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None });
d_decl.output = FnRetTy::Ty(ty);
}
if let DiffActivity::DualOnly = x.ret_activity {
// No need to change the return type,
- // we will just return the shadow in place
- // of the primal return.
+ // we will just return the shadow in place of the primal return.
+ // However, if we have a width > 1, then we don't return -> T, but -> [T; width]
+ if x.width > 1 {
+ let anon_const = rustc_ast::AnonConst {
+ id: ast::DUMMY_NODE_ID,
+ value: ecx.expr_usize(span, x.width as usize),
+ };
+ let kind = TyKind::Array(ty.clone(), anon_const);
+ let ty = P(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None });
+ d_decl.output = FnRetTy::Ty(ty);
+ }
}
}
diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs
index 30597944124cb..4bbe212f4296c 100644
--- a/compiler/rustc_builtin_macros/src/errors.rs
+++ b/compiler/rustc_builtin_macros/src/errors.rs
@@ -202,6 +202,14 @@ mod autodiff {
pub(crate) mode: String,
}
+ #[derive(Diagnostic)]
+ #[diag(builtin_macros_autodiff_width)]
+ pub(crate) struct AutoDiffInvalidWidth {
+ #[primary_span]
+ pub(crate) span: Span,
+ pub(crate) width: u128,
+ }
+
#[derive(Diagnostic)]
#[diag(builtin_macros_autodiff)]
pub(crate) struct AutoDiffInvalidApplication {
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index f083cfbd7d306..a8b49e9552c30 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -610,6 +610,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff], module: &mut ModuleCodegen<
}
// We handle this below
config::AutoDiff::PrintModAfter => {}
+ // We handle this below
+ config::AutoDiff::PrintModFinal => {}
// This is required and already checked
config::AutoDiff::Enable => {}
}
@@ -657,14 +659,20 @@ pub(crate) fn run_pass_manager(
}
if cfg!(llvm_enzyme) && enable_ad {
+ // This is the post-autodiff IR, mainly used for testing and educational purposes.
+ if config.autodiff.contains(&config::AutoDiff::PrintModAfter) {
+ unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) };
+ }
+
let opt_stage = llvm::OptStage::FatLTO;
let stage = write::AutodiffStage::PostAD;
unsafe {
write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?;
}
- // This is the final IR, so people should be able to inspect the optimized autodiff output.
- if config.autodiff.contains(&config::AutoDiff::PrintModAfter) {
+ // This is the final IR, so people should be able to inspect the optimized autodiff output,
+ // for manual inspection.
+ if config.autodiff.contains(&config::AutoDiff::PrintModFinal) {
unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) };
}
}
diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
index 7cd4ee539d875..7d264ba4d00c8 100644
--- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
+++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
@@ -3,8 +3,10 @@ use std::ptr;
use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivity, DiffMode};
use rustc_codegen_ssa::ModuleCodegen;
use rustc_codegen_ssa::back::write::ModuleConfig;
-use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _;
+use rustc_codegen_ssa::common::TypeKind;
+use rustc_codegen_ssa::traits::BaseTypeCodegenMethods;
use rustc_errors::FatalError;
+use rustc_middle::bug;
use tracing::{debug, trace};
use crate::back::write::llvm_err;
@@ -18,21 +20,42 @@ use crate::value::Value;
use crate::{CodegenContext, LlvmCodegenBackend, ModuleLlvm, attributes, llvm};
fn get_params(fnc: &Value) -> Vec<&Value> {
+ let param_num = llvm::LLVMCountParams(fnc) as usize;
+ let mut fnc_args: Vec<&Value> = vec![];
+ fnc_args.reserve(param_num);
unsafe {
- let param_num = llvm::LLVMCountParams(fnc) as usize;
- let mut fnc_args: Vec<&Value> = vec![];
- fnc_args.reserve(param_num);
llvm::LLVMGetParams(fnc, fnc_args.as_mut_ptr());
fnc_args.set_len(param_num);
- fnc_args
}
+ fnc_args
}
+fn has_sret(fnc: &Value) -> bool {
+ let num_args = llvm::LLVMCountParams(fnc) as usize;
+ if num_args == 0 {
+ false
+ } else {
+ unsafe { llvm::LLVMRustHasAttributeAtIndex(fnc, 0, llvm::AttributeKind::StructRet) }
+ }
+}
+
+// When we call the `__enzyme_autodiff` or `__enzyme_fwddiff` function, we need to pass all the
+// original inputs, as well as metadata and the additional shadow arguments.
+// This function matches the arguments from the outer function to the inner enzyme call.
+//
+// This function also considers that Rust level arguments not always match the llvm-ir level
+// arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on
+// llvm-ir level. The number of activities matches the number of Rust level arguments, so we
+// need to match those.
+// FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it
+// using iterators and peek()?
fn match_args_from_caller_to_enzyme<'ll>(
cx: &SimpleCx<'ll>,
+ width: u32,
args: &mut Vec<&'ll llvm::Value>,
inputs: &[DiffActivity],
outer_args: &[&'ll llvm::Value],
+ has_sret: bool,
) {
debug!("matching autodiff arguments");
// We now handle the issue that Rust level arguments not always match the llvm-ir level
@@ -44,6 +67,14 @@ fn match_args_from_caller_to_enzyme<'ll>(
let mut outer_pos: usize = 0;
let mut activity_pos = 0;
+ if has_sret {
+ // Then the first outer arg is the sret pointer. Enzyme doesn't know about sret, so the
+ // inner function will still return something. We increase our outer_pos by one,
+ // and once we're done with all other args we will take the return of the inner call and
+ // update the sret pointer with it
+ outer_pos = 1;
+ }
+
let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap();
let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap();
let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap();
@@ -92,23 +123,20 @@ fn match_args_from_caller_to_enzyme<'ll>(
// (..., metadata! enzyme_dup, ptr, ptr, int1, ...).
// FIXME(ZuseZ4): We will upstream a safety check later which asserts that
// int2 >= int1, which means the shadow vector is large enough to store the gradient.
- assert!(unsafe {
- llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Integer
- });
- let next_outer_arg2 = outer_args[outer_pos + 2];
- let next_outer_ty2 = cx.val_ty(next_outer_arg2);
- assert!(unsafe {
- llvm::LLVMRustGetTypeKind(next_outer_ty2) == llvm::TypeKind::Pointer
- });
- let next_outer_arg3 = outer_args[outer_pos + 3];
- let next_outer_ty3 = cx.val_ty(next_outer_arg3);
- assert!(unsafe {
- llvm::LLVMRustGetTypeKind(next_outer_ty3) == llvm::TypeKind::Integer
- });
- args.push(next_outer_arg2);
+ assert_eq!(cx.type_kind(next_outer_ty), TypeKind::Integer);
+
+ for i in 0..(width as usize) {
+ let next_outer_arg2 = outer_args[outer_pos + 2 * (i + 1)];
+ let next_outer_ty2 = cx.val_ty(next_outer_arg2);
+ assert_eq!(cx.type_kind(next_outer_ty2), TypeKind::Pointer);
+ let next_outer_arg3 = outer_args[outer_pos + 2 * (i + 1) + 1];
+ let next_outer_ty3 = cx.val_ty(next_outer_arg3);
+ assert_eq!(cx.type_kind(next_outer_ty3), TypeKind::Integer);
+ args.push(next_outer_arg2);
+ }
args.push(cx.get_metadata_value(enzyme_const));
args.push(next_outer_arg);
- outer_pos += 4;
+ outer_pos += 2 + 2 * width as usize;
activity_pos += 2;
} else {
// A duplicated pointer will have the following two outer_fn arguments:
@@ -116,15 +144,19 @@ fn match_args_from_caller_to_enzyme<'ll>(
// (..., metadata! enzyme_dup, ptr, ptr, ...).
if matches!(diff_activity, DiffActivity::Duplicated | DiffActivity::DuplicatedOnly)
{
- assert!(
- unsafe { llvm::LLVMRustGetTypeKind(next_outer_ty) }
- == llvm::TypeKind::Pointer
- );
+ assert_eq!(cx.type_kind(next_outer_ty), TypeKind::Pointer);
}
// In the case of Dual we don't have assumptions, e.g. f32 would be valid.
args.push(next_outer_arg);
outer_pos += 2;
activity_pos += 1;
+
+ // Now, if width > 1, we need to account for that
+ for _ in 1..width {
+ let next_outer_arg = outer_args[outer_pos];
+ args.push(next_outer_arg);
+ outer_pos += 1;
+ }
}
} else {
// We do not differentiate with resprect to this argument.
@@ -135,6 +167,76 @@ fn match_args_from_caller_to_enzyme<'ll>(
}
}
+// On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input
+// arguments. We do however need to declare them with their correct return type.
+// We already figured the correct return type out in our frontend, when generating the outer_fn,
+// so we can now just go ahead and use that. This is not always trivial, e.g. because sret.
+// Beyond sret, this article describes our challenges nicely:
+// <https://yorickpeterse.com/articles/the-mess-that-is-handling-structure-arguments-and-returns-in-llvm/>
+// I.e. (i32, f32) will get merged into i64, but we don't handle that yet.
+fn compute_enzyme_fn_ty<'ll>(
+ cx: &SimpleCx<'ll>,
+ attrs: &AutoDiffAttrs,
+ fn_to_diff: &'ll Value,
+ outer_fn: &'ll Value,
+) -> &'ll llvm::Type {
+ let fn_ty = cx.get_type_of_global(outer_fn);
+ let mut ret_ty = cx.get_return_type(fn_ty);
+
+ let has_sret = has_sret(outer_fn);
+
+ if has_sret {
+ // Now we don't just forward the return type, so we have to figure it out based on the
+ // primal return type, in combination with the autodiff settings.
+ let fn_ty = cx.get_type_of_global(fn_to_diff);
+ let inner_ret_ty = cx.get_return_type(fn_ty);
+
+ let void_ty = unsafe { llvm::LLVMVoidTypeInContext(cx.llcx) };
+ if inner_ret_ty == void_ty {
+ // This indicates that even the inner function has an sret.
+ // Right now I only look for an sret in the outer function.
+ // This *probably* needs some extra handling, but I never ran
+ // into such a case. So I'll wait for user reports to have a test case.
+ bug!("sret in inner function");
+ }
+
+ if attrs.width == 1 {
+ todo!("Handle sret for scalar ad");
+ } else {
+ // First we check if we also have to deal with the primal return.
+ match attrs.mode {
+ DiffMode::Forward => match attrs.ret_activity {
+ DiffActivity::Dual => {
+ let arr_ty =
+ unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64 + 1) };
+ ret_ty = arr_ty;
+ }
+ DiffActivity::DualOnly => {
+ let arr_ty =
+ unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64) };
+ ret_ty = arr_ty;
+ }
+ DiffActivity::Const => {
+ todo!("Not sure, do we need to do something here?");
+ }
+ _ => {
+ bug!("unreachable");
+ }
+ },
+ DiffMode::Reverse => {
+ todo!("Handle sret for reverse mode");
+ }
+ _ => {
+ bug!("unreachable");
+ }
+ }
+ }
+ }
+
+ // LLVM can figure out the input types on it's own, so we take a shortcut here.
+ unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) }
+}
+
/// When differentiating `fn_to_diff`, take a `outer_fn` and generate another
/// function with expected naming and calling conventions[^1] which will be
/// discovered by the enzyme LLVM pass and its body populated with the differentiated
@@ -197,17 +299,9 @@ fn generate_enzyme_call<'ll>(
// }
// ```
unsafe {
- // On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input
- // arguments. We do however need to declare them with their correct return type.
- // We already figured the correct return type out in our frontend, when generating the outer_fn,
- // so we can now just go ahead and use that. FIXME(ZuseZ4): This doesn't handle sret yet.
- let fn_ty = llvm::LLVMGlobalGetValueType(outer_fn);
- let ret_ty = llvm::LLVMGetReturnType(fn_ty);
-
- // LLVM can figure out the input types on it's own, so we take a shortcut here.
- let enzyme_ty = llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True);
+ let enzyme_ty = compute_enzyme_fn_ty(cx, &attrs, fn_to_diff, outer_fn);
- //FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and
+ // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and
// think a bit more about what should go here.
let cc = llvm::LLVMGetFunctionCallConv(outer_fn);
let ad_fn = declare_simple_fn(
@@ -240,14 +334,27 @@ fn generate_enzyme_call<'ll>(
if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) {
args.push(cx.get_metadata_value(enzyme_primal_ret));
}
+ if attrs.width > 1 {
+ let enzyme_width = cx.create_metadata("enzyme_width".to_string()).unwrap();
+ args.push(cx.get_metadata_value(enzyme_width));
+ args.push(cx.get_const_i64(attrs.width as u64));
+ }
+ let has_sret = has_sret(outer_fn);
let outer_args: Vec<&llvm::Value> = get_params(outer_fn);
- match_args_from_caller_to_enzyme(&cx, &mut args, &attrs.input_activity, &outer_args);
+ match_args_from_caller_to_enzyme(
+ &cx,
+ attrs.width,
+ &mut args,
+ &attrs.input_activity,
+ &outer_args,
+ has_sret,
+ );
let call = builder.call(enzyme_ty, ad_fn, &args, None);
// This part is a bit iffy. LLVM requires that a call to an inlineable function has some
- // metadata attachted to it, but we just created this code oota. Given that the
+ // metadata attached to it, but we just created this code oota. Given that the
// differentiated function already has partly confusing metadata, and given that this
// affects nothing but the auttodiff IR, we take a shortcut and just steal metadata from the
// dummy code which we inserted at a higher level.
@@ -268,7 +375,22 @@ fn generate_enzyme_call<'ll>(
// Now that we copied the metadata, get rid of dummy code.
llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst);
- if cx.val_ty(call) == cx.type_void() {
+ if cx.val_ty(call) == cx.type_void() || has_sret {
+ if has_sret {
+ // This is what we already have in our outer_fn (shortened):
+ // define void @_foo(ptr <..> sret([32 x i8]) initializes((0, 32)) %0, <...>) {
+ // %7 = call [4 x double] (...) @__enzyme_fwddiff_foo(ptr @square, metadata !"enzyme_width", i64 4, <...>)
+ // <Here we are, we want to add the following two lines>
+ // store [4 x double] %7, ptr %0, align 8
+ // ret void
+ // }
+
+ // now store the result of the enzyme call into the sret pointer.
+ let sret_ptr = outer_args[0];
+ let call_ty = cx.val_ty(call);
+ assert_eq!(cx.type_kind(call_ty), TypeKind::Array);
+ llvm::LLVMBuildStore(&builder.llbuilder, call, sret_ptr);
+ }
builder.ret_void();
} else {
builder.ret(call);
@@ -300,8 +422,7 @@ pub(crate) fn differentiate<'ll>(
if !diff_items.is_empty()
&& !cgcx.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable)
{
- let dcx = cgcx.create_dcx();
- return Err(dcx.handle().emit_almost_fatal(AutoDiffWithoutEnable));
+ return Err(diag_handler.handle().emit_almost_fatal(AutoDiffWithoutEnable));
}
// Before dumping the module, we want all the TypeTrees to become part of the module.
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index 7675e75338a3e..bf81eb648f8ae 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -430,7 +430,7 @@ impl<'ll> CodegenCx<'ll, '_> {
let val_llty = self.val_ty(v);
let g = self.get_static_inner(def_id, val_llty);
- let llty = llvm::LLVMGlobalGetValueType(g);
+ let llty = self.get_type_of_global(g);
let g = if val_llty == llty {
g
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index f7b096ff976a6..3be8cd5f6ac21 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -8,6 +8,7 @@ use std::str;
use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx};
use rustc_codegen_ssa::back::versioned_llvm_target;
use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh};
+use rustc_codegen_ssa::common::TypeKind;
use rustc_codegen_ssa::errors as ssa_errors;
use rustc_codegen_ssa::traits::*;
use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN};
@@ -38,7 +39,7 @@ use crate::debuginfo::metadata::apply_vcall_visibility_metadata;
use crate::llvm::Metadata;
use crate::type_::Type;
use crate::value::Value;
-use crate::{attributes, coverageinfo, debuginfo, llvm, llvm_util};
+use crate::{attributes, common, coverageinfo, debuginfo, llvm, llvm_util};
/// `TyCtxt` (and related cache datastructures) can't be move between threads.
/// However, there are various cx related functions which we want to be available to the builder and
@@ -643,7 +644,18 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
llvm::set_section(g, c"llvm.metadata");
}
}
-
+impl<'ll> SimpleCx<'ll> {
+ pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type {
+ assert_eq!(self.type_kind(ty), TypeKind::Function);
+ unsafe { llvm::LLVMGetReturnType(ty) }
+ }
+ pub(crate) fn get_type_of_global(&self, val: &'ll Value) -> &'ll Type {
+ unsafe { llvm::LLVMGlobalGetValueType(val) }
+ }
+ pub(crate) fn val_ty(&self, v: &'ll Value) -> &'ll Type {
+ common::val_ty(v)
+ }
+}
impl<'ll> SimpleCx<'ll> {
pub(crate) fn new(
llmod: &'ll llvm::Module,
@@ -660,6 +672,13 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
llvm::LLVMMetadataAsValue(self.llcx(), metadata)
}
+ // FIXME(autodiff): We should split `ConstCodegenMethods` to pull the reusable parts
+ // onto a trait that is also implemented for GenericCx.
+ pub(crate) fn get_const_i64(&self, n: u64) -> &'ll Value {
+ let ty = unsafe { llvm::LLVMInt64TypeInContext(self.llcx()) };
+ unsafe { llvm::LLVMConstInt(ty, n, llvm::False) }
+ }
+
pub(crate) fn get_function(&self, name: &str) -> Option<&'ll Value> {
let name = SmallCStr::new(name);
unsafe { llvm::LLVMGetNamedFunction((**self).borrow().llmod, name.as_ptr()) }
diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs
index 79e4cc8aa7744..a9b3bdf7344be 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs
@@ -4,7 +4,7 @@
use libc::{c_char, c_uint};
use super::MetadataKindId;
-use super::ffi::{BasicBlock, Metadata, Module, Type, Value};
+use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value};
use crate::llvm::Bool;
#[link(name = "llvm-wrapper", kind = "static")]
@@ -17,6 +17,8 @@ unsafe extern "C" {
pub(crate) fn LLVMRustEraseInstFromParent(V: &Value);
pub(crate) fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value;
pub(crate) fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool;
+ pub(crate) fn LLVMRustHasAttributeAtIndex(V: &Value, i: c_uint, Kind: AttributeKind) -> bool;
+ pub(crate) fn LLVMRustGetArrayNumElements(Ty: &Type) -> u64;
}
unsafe extern "C" {
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 3ce3761944b3a..9ff04f729030c 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -1180,7 +1180,7 @@ unsafe extern "C" {
// Operations on parameters
pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;
- pub(crate) fn LLVMCountParams(Fn: &Value) -> c_uint;
+ pub(crate) safe fn LLVMCountParams(Fn: &Value) -> c_uint;
pub(crate) fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value;
// Operations on basic blocks
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 8a184fc0bef6d..ddb6118898334 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -2,7 +2,7 @@ use std::str::FromStr;
use rustc_abi::ExternAbi;
use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode};
-use rustc_ast::{MetaItem, MetaItemInner, attr};
+use rustc_ast::{LitKind, MetaItem, MetaItemInner, attr};
use rustc_attr_parsing::ReprAttr::ReprAlign;
use rustc_attr_parsing::{AttributeKind, InlineAttr, InstructionSetAttr, OptimizeAttr};
use rustc_data_structures::fx::FxHashMap;
@@ -805,8 +805,8 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
return Some(AutoDiffAttrs::source());
}
- let [mode, input_activities @ .., ret_activity] = &list[..] else {
- span_bug!(attr.span(), "rustc_autodiff attribute must contain mode and activities");
+ let [mode, width_meta, input_activities @ .., ret_activity] = &list[..] else {
+ span_bug!(attr.span(), "rustc_autodiff attribute must contain mode, width and activities");
};
let mode = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = mode {
p1.segments.first().unwrap().ident
@@ -823,6 +823,30 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
}
};
+ let width: u32 = match width_meta {
+ MetaItemInner::MetaItem(MetaItem { path: p1, .. }) => {
+ let w = p1.segments.first().unwrap().ident;
+ match w.as_str().parse() {
+ Ok(val) => val,
+ Err(_) => {
+ span_bug!(w.span, "rustc_autodiff width should fit u32");
+ }
+ }
+ }
+ MetaItemInner::Lit(lit) => {
+ if let LitKind::Int(val, _) = lit.kind {
+ match val.get().try_into() {
+ Ok(val) => val,
+ Err(_) => {
+ span_bug!(lit.span, "rustc_autodiff width should fit u32");
+ }
+ }
+ } else {
+ span_bug!(lit.span, "rustc_autodiff width should be an integer");
+ }
+ }
+ };
+
// First read the ret symbol from the attribute
let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = ret_activity {
p1.segments.first().unwrap().ident
@@ -860,7 +884,7 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
}
}
- Some(AutoDiffAttrs { mode, ret_activity, input_activity: arg_activities })
+ Some(AutoDiffAttrs { mode, width, ret_activity, input_activity: arg_activities })
}
pub(crate) fn provide(providers: &mut Providers) {
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 21b9542d0e1e8..6fb01bf415acc 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -64,6 +64,7 @@ use rustc_session::lint::{Lint, LintId};
use rustc_session::output::{CRATE_TYPES, collect_crate_types, invalid_output_for_target};
use rustc_session::{EarlyDiagCtxt, Session, config, filesearch};
use rustc_span::FileName;
+use rustc_span::def_id::LOCAL_CRATE;
use rustc_target::json::ToJson;
use rustc_target::spec::{Target, TargetTuple};
use time::OffsetDateTime;
@@ -392,14 +393,10 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
}
fn dump_feature_usage_metrics(tcxt: TyCtxt<'_>, metrics_dir: &Path) {
- let output_filenames = tcxt.output_filenames(());
- let mut metrics_file_name = std::ffi::OsString::from("unstable_feature_usage_metrics-");
- let mut metrics_path = output_filenames.with_directory_and_extension(metrics_dir, "json");
- let metrics_file_stem =
- metrics_path.file_name().expect("there should be a valid default output filename");
- metrics_file_name.push(metrics_file_stem);
- metrics_path.pop();
- metrics_path.push(metrics_file_name);
+ let hash = tcxt.crate_hash(LOCAL_CRATE);
+ let crate_name = tcxt.crate_name(LOCAL_CRATE);
+ let metrics_file_name = format!("unstable_feature_usage_metrics-{crate_name}-{hash}.json");
+ let metrics_path = metrics_dir.join(metrics_file_name);
if let Err(error) = tcxt.features().dump_feature_usage_metrics(metrics_path) {
// FIXME(yaahc): once metrics can be enabled by default we will want "failure to emit
// default metrics" to only produce a warning when metrics are enabled by default and emit
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index 53df59930f4fd..32e6da446d72e 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -384,6 +384,12 @@ static inline void AddAttributes(T *t, unsigned Index, LLVMAttributeRef *Attrs,
t->setAttributes(PALNew);
}
+extern "C" bool LLVMRustHasAttributeAtIndex(LLVMValueRef Fn, unsigned Index,
+ LLVMRustAttributeKind RustAttr) {
+ Function *F = unwrap<Function>(Fn);
+ return F->hasParamAttribute(Index, fromRust(RustAttr));
+}
+
extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index,
LLVMAttributeRef *Attrs,
size_t AttrsLen) {
@@ -636,6 +642,10 @@ static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) {
}
}
+extern "C" uint64_t LLVMRustGetArrayNumElements(LLVMTypeRef Ty) {
+ return unwrap(Ty)->getArrayNumElements();
+}
+
extern "C" LLVMValueRef
LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen,
char *Constraints, size_t ConstraintsLen,
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 1efe36a0910b8..162ca1f4af850 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -1807,10 +1807,15 @@ impl<'tcx> TyCtxt<'tcx> {
// - needs_metadata: for putting into crate metadata.
// - instrument_coverage: for putting into coverage data (see
// `hash_mir_source`).
+ // - metrics_dir: metrics use the strict version hash in the filenames
+ // for dumped metrics files to prevent overwriting distinct metrics
+ // for similar source builds (may change in the future, this is part
+ // of the proof of concept impl for the metrics initiative project goal)
cfg!(debug_assertions)
|| self.sess.opts.incremental.is_some()
|| self.needs_metadata()
|| self.sess.instrument_coverage()
+ || self.sess.opts.unstable_opts.metrics_dir.is_some()
}
#[inline]
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 392a1c1057abb..3b0861a9942a6 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -512,6 +512,14 @@ impl<'a> Parser<'a> {
self
}
+ #[inline]
+ fn with_recovery<T>(&mut self, recovery: Recovery, f: impl FnOnce(&mut Self) -> T) -> T {
+ let old = mem::replace(&mut self.recovery, recovery);
+ let res = f(self);
+ self.recovery = old;
+ res
+ }
+
/// Whether the parser is allowed to recover from broken code.
///
/// If this returns false, recovering broken code into valid code (especially if this recovery does lookahead)
@@ -770,7 +778,14 @@ impl<'a> Parser<'a> {
&& match_mv_kind(mv_kind)
{
self.bump();
- let res = f(self).expect("failed to reparse {mv_kind:?}");
+
+ // Recovery is disabled when parsing macro arguments, so it must
+ // also be disabled when reparsing pasted macro arguments,
+ // otherwise we get inconsistent results (e.g. #137874).
+ let res = self.with_recovery(Recovery::Forbidden, |this| {
+ f(this).expect("failed to reparse {mv_kind:?}")
+ });
+
if let token::CloseDelim(delim) = self.token.kind
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
&& match_mv_kind(mv_kind)
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 1f18950feac35..56b3fe2ab4cb1 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -237,10 +237,12 @@ pub enum AutoDiff {
PrintPerf,
/// Print intermediate IR generation steps
PrintSteps,
- /// Print the whole module, before running opts.
+ /// Print the module, before running autodiff.
PrintModBefore,
- /// Print the module after Enzyme differentiated everything.
+ /// Print the module after running autodiff.
PrintModAfter,
+ /// Print the module after running autodiff and optimizations.
+ PrintModFinal,
/// Enzyme's loose type debug helper (can cause incorrect gradients!!)
/// Usable in cases where Enzyme errors with `can not deduce type of X`.
@@ -1425,10 +1427,12 @@ pub fn build_target_config(
}
target
}
- Err(e) => early_dcx.early_fatal(format!(
- "Error loading target specification: {e}. \
- Run `rustc --print target-list` for a list of built-in targets"
- )),
+ Err(e) => {
+ let mut err =
+ early_dcx.early_struct_fatal(format!("error loading target specification: {e}"));
+ err.help("run `rustc --print target-list` for a list of built-in targets");
+ err.emit();
+ }
}
}
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 4f544d2c16b70..5ed8cc178861d 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -711,7 +711,7 @@ mod desc {
pub(crate) const parse_list: &str = "a space-separated list of strings";
pub(crate) const parse_list_with_polarity: &str =
"a comma-separated list of strings, with elements beginning with + or -";
- pub(crate) const parse_autodiff: &str = "a comma separated list of settings: `Enable`, `PrintSteps`, `PrintTA`, `PrintAA`, `PrintPerf`, `PrintModBefore`, `PrintModAfter`, `LooseTypes`, `Inline`";
+ pub(crate) const parse_autodiff: &str = "a comma separated list of settings: `Enable`, `PrintSteps`, `PrintTA`, `PrintAA`, `PrintPerf`, `PrintModBefore`, `PrintModAfter`, `PrintModFinal`, `LooseTypes`, `Inline`";
pub(crate) const parse_comma_list: &str = "a comma-separated list of strings";
pub(crate) const parse_opt_comma_list: &str = parse_comma_list;
pub(crate) const parse_number: &str = "a number";
@@ -1359,6 +1359,7 @@ pub mod parse {
"PrintSteps" => AutoDiff::PrintSteps,
"PrintModBefore" => AutoDiff::PrintModBefore,
"PrintModAfter" => AutoDiff::PrintModAfter,
+ "PrintModFinal" => AutoDiff::PrintModFinal,
"LooseTypes" => AutoDiff::LooseTypes,
"Inline" => AutoDiff::Inline,
_ => {
@@ -2093,6 +2094,7 @@ options! {
`=PrintSteps`
`=PrintModBefore`
`=PrintModAfter`
+ `=PrintModFinal`
`=LooseTypes`
`=Inline`
Multiple options can be combined with commas."),
diff --git a/compiler/rustc_smir/src/rustc_internal/mod.rs b/compiler/rustc_smir/src/rustc_internal/mod.rs
index ad38ea228bf53..30e9b2e72f735 100644
--- a/compiler/rustc_smir/src/rustc_internal/mod.rs
+++ b/compiler/rustc_smir/src/rustc_internal/mod.rs
@@ -147,6 +147,14 @@ impl<'tcx> Tables<'tcx> {
stable_mir::ty::CoroutineWitnessDef(self.create_def_id(did))
}
+ pub fn assoc_def(&mut self, did: DefId) -> stable_mir::ty::AssocDef {
+ stable_mir::ty::AssocDef(self.create_def_id(did))
+ }
+
+ pub fn opaque_def(&mut self, did: DefId) -> stable_mir::ty::OpaqueDef {
+ stable_mir::ty::OpaqueDef(self.create_def_id(did))
+ }
+
pub fn prov(&mut self, aid: AllocId) -> stable_mir::ty::Prov {
stable_mir::ty::Prov(self.create_alloc_id(aid))
}
diff --git a/compiler/rustc_smir/src/rustc_smir/context.rs b/compiler/rustc_smir/src/rustc_smir/context.rs
index aa1921fc8e784..322e86147c19b 100644
--- a/compiler/rustc_smir/src/rustc_smir/context.rs
+++ b/compiler/rustc_smir/src/rustc_smir/context.rs
@@ -822,6 +822,21 @@ impl<'tcx> Context for TablesWrapper<'tcx> {
let ty = un_op.internal(&mut *tables, tcx).ty(tcx, arg_internal);
ty.stable(&mut *tables)
}
+
+ fn associated_items(&self, def_id: stable_mir::DefId) -> stable_mir::AssocItems {
+ let mut tables = self.0.borrow_mut();
+ let tcx = tables.tcx;
+ let def_id = tables[def_id];
+ let assoc_items = if tcx.is_trait_alias(def_id) {
+ Vec::new()
+ } else {
+ tcx.associated_item_def_ids(def_id)
+ .iter()
+ .map(|did| tcx.associated_item(*did).stable(&mut *tables))
+ .collect()
+ };
+ assoc_items
+ }
}
pub(crate) struct TablesWrapper<'tcx>(pub RefCell<Tables<'tcx>>);
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
index aa0eac628dd0f..8309809d7b51c 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
@@ -890,3 +890,63 @@ impl<'tcx> Stable<'tcx> for rustc_session::cstore::ForeignModule {
}
}
}
+
+impl<'tcx> Stable<'tcx> for ty::AssocKind {
+ type T = stable_mir::ty::AssocKind;
+
+ fn stable(&self, _tables: &mut Tables<'_>) -> Self::T {
+ use stable_mir::ty::AssocKind;
+ match self {
+ ty::AssocKind::Const => AssocKind::Const,
+ ty::AssocKind::Fn => AssocKind::Fn,
+ ty::AssocKind::Type => AssocKind::Type,
+ }
+ }
+}
+
+impl<'tcx> Stable<'tcx> for ty::AssocItemContainer {
+ type T = stable_mir::ty::AssocItemContainer;
+
+ fn stable(&self, _tables: &mut Tables<'_>) -> Self::T {
+ use stable_mir::ty::AssocItemContainer;
+ match self {
+ ty::AssocItemContainer::Trait => AssocItemContainer::Trait,
+ ty::AssocItemContainer::Impl => AssocItemContainer::Impl,
+ }
+ }
+}
+
+impl<'tcx> Stable<'tcx> for ty::AssocItem {
+ type T = stable_mir::ty::AssocItem;
+
+ fn stable(&self, tables: &mut Tables<'_>) -> Self::T {
+ stable_mir::ty::AssocItem {
+ def_id: tables.assoc_def(self.def_id),
+ name: self.name.to_string(),
+ kind: self.kind.stable(tables),
+ container: self.container.stable(tables),
+ trait_item_def_id: self.trait_item_def_id.map(|did| tables.assoc_def(did)),
+ fn_has_self_parameter: self.fn_has_self_parameter,
+ opt_rpitit_info: self.opt_rpitit_info.map(|rpitit| rpitit.stable(tables)),
+ }
+ }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ImplTraitInTraitData {
+ type T = stable_mir::ty::ImplTraitInTraitData;
+
+ fn stable(&self, tables: &mut Tables<'_>) -> Self::T {
+ use stable_mir::ty::ImplTraitInTraitData;
+ match self {
+ ty::ImplTraitInTraitData::Trait { fn_def_id, opaque_def_id } => {
+ ImplTraitInTraitData::Trait {
+ fn_def_id: tables.fn_def(*fn_def_id),
+ opaque_def_id: tables.opaque_def(*opaque_def_id),
+ }
+ }
+ ty::ImplTraitInTraitData::Impl { fn_def_id } => {
+ ImplTraitInTraitData::Impl { fn_def_id: tables.fn_def(*fn_def_id) }
+ }
+ }
+ }
+}
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 981dded2601a3..98db0af30c299 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -3516,7 +3516,7 @@ impl Target {
Err("the `i586-pc-windows-msvc` target has been removed. Use the `i686-pc-windows-msvc` target instead.\n\
Windows 10 (the minimum required OS version) requires a CPU baseline of at least i686 so you can safely switch".into())
} else {
- Err(format!("Could not find specification for target {target_tuple:?}"))
+ Err(format!("could not find specification for target {target_tuple:?}"))
}
}
TargetTuple::TargetJson { ref contents, .. } => {
diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs
index e82c957c34ea6..46154da36ca07 100644
--- a/compiler/stable_mir/src/compiler_interface.rs
+++ b/compiler/stable_mir/src/compiler_interface.rs
@@ -18,8 +18,8 @@ use crate::ty::{
TraitDef, Ty, TyConst, TyConstId, TyKind, UintTy, VariantDef,
};
use crate::{
- Crate, CrateItem, CrateItems, CrateNum, DefId, Error, Filename, ImplTraitDecls, ItemKind,
- Symbol, TraitDecls, mir,
+ AssocItems, Crate, CrateItem, CrateItems, CrateNum, DefId, Error, Filename, ImplTraitDecls,
+ ItemKind, Symbol, TraitDecls, mir,
};
/// This trait defines the interface between stable_mir and the Rust compiler.
@@ -251,6 +251,9 @@ pub trait Context {
/// Get the resulting type of unary operation.
fn unop_ty(&self, un_op: UnOp, arg: Ty) -> Ty;
+
+ /// Get all associated items of a definition.
+ fn associated_items(&self, def_id: DefId) -> AssocItems;
}
// A thread local variable that stores a pointer to the tables mapping between TyCtxt
diff --git a/compiler/stable_mir/src/crate_def.rs b/compiler/stable_mir/src/crate_def.rs
index 2577c281ca4f2..75228135e4cb3 100644
--- a/compiler/stable_mir/src/crate_def.rs
+++ b/compiler/stable_mir/src/crate_def.rs
@@ -4,7 +4,7 @@
use serde::Serialize;
use crate::ty::{GenericArgs, Span, Ty};
-use crate::{Crate, Symbol, with};
+use crate::{AssocItems, Crate, Symbol, with};
/// A unique identification number for each item accessible for the current compilation unit.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize)]
@@ -103,6 +103,14 @@ pub trait CrateDefType: CrateDef {
}
}
+/// A trait for retrieving all items from a definition within a crate.
+pub trait CrateDefItems: CrateDef {
+ /// Retrieve all associated items from a definition.
+ fn associated_items(&self) -> AssocItems {
+ with(|cx| cx.associated_items(self.def_id()))
+ }
+}
+
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Attribute {
value: String,
@@ -158,3 +166,9 @@ macro_rules! crate_def_with_ty {
impl CrateDefType for $name {}
};
}
+
+macro_rules! impl_crate_def_items {
+ ( $name:ident $(;)? ) => {
+ impl CrateDefItems for $name {}
+ };
+}
diff --git a/compiler/stable_mir/src/lib.rs b/compiler/stable_mir/src/lib.rs
index 70d42dfbfcb9a..df90d3e5a0841 100644
--- a/compiler/stable_mir/src/lib.rs
+++ b/compiler/stable_mir/src/lib.rs
@@ -23,11 +23,11 @@ use std::{fmt, io};
use serde::Serialize;
use crate::compiler_interface::with;
-pub use crate::crate_def::{CrateDef, CrateDefType, DefId};
+pub use crate::crate_def::{CrateDef, CrateDefItems, CrateDefType, DefId};
pub use crate::error::*;
use crate::mir::mono::StaticDef;
use crate::mir::{Body, Mutability};
-use crate::ty::{FnDef, ForeignModuleDef, ImplDef, IndexedVal, Span, TraitDef, Ty};
+use crate::ty::{AssocItem, FnDef, ForeignModuleDef, ImplDef, IndexedVal, Span, TraitDef, Ty};
pub mod abi;
#[macro_use]
@@ -71,6 +71,9 @@ pub type TraitDecls = Vec<TraitDef>;
/// A list of impl trait decls.
pub type ImplTraitDecls = Vec<ImplDef>;
+/// A list of associated items.
+pub type AssocItems = Vec<AssocItem>;
+
/// Holds information about a crate.
#[derive(Clone, PartialEq, Eq, Debug, Serialize)]
pub struct Crate {
diff --git a/compiler/stable_mir/src/mir/pretty.rs b/compiler/stable_mir/src/mir/pretty.rs
index 8278afb7a2f17..65d9f20f0a3b1 100644
--- a/compiler/stable_mir/src/mir/pretty.rs
+++ b/compiler/stable_mir/src/mir/pretty.rs
@@ -9,7 +9,7 @@ use super::{AggregateKind, AssertMessage, BinOp, BorrowKind, FakeBorrowKind, Ter
use crate::mir::{
Operand, Place, RawPtrKind, Rvalue, StatementKind, UnwindAction, VarDebugInfoContents,
};
-use crate::ty::{AdtKind, IndexedVal, MirConst, Ty, TyConst};
+use crate::ty::{AdtKind, AssocKind, IndexedVal, MirConst, Ty, TyConst};
use crate::{Body, CrateDef, Mutability, with};
impl Display for Ty {
@@ -18,6 +18,16 @@ impl Display for Ty {
}
}
+impl Display for AssocKind {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ match self {
+ AssocKind::Fn => write!(f, "method"),
+ AssocKind::Const => write!(f, "associated const"),
+ AssocKind::Type => write!(f, "associated type"),
+ }
+ }
+}
+
impl Debug for Place {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
with(|ctx| write!(f, "{}", ctx.place_pretty(self)))
diff --git a/compiler/stable_mir/src/ty.rs b/compiler/stable_mir/src/ty.rs
index b857a735b7259..25ec4a440d6ec 100644
--- a/compiler/stable_mir/src/ty.rs
+++ b/compiler/stable_mir/src/ty.rs
@@ -6,7 +6,7 @@ use serde::Serialize;
use super::mir::{Body, Mutability, Safety};
use super::{DefId, Error, Symbol, with};
use crate::abi::{FnAbi, Layout};
-use crate::crate_def::{CrateDef, CrateDefType};
+use crate::crate_def::{CrateDef, CrateDefItems, CrateDefType};
use crate::mir::alloc::{AllocId, read_target_int, read_target_uint};
use crate::mir::mono::StaticDef;
use crate::target::MachineInfo;
@@ -910,6 +910,10 @@ crate_def! {
pub TraitDef;
}
+impl_crate_def_items! {
+ TraitDef;
+}
+
impl TraitDef {
pub fn declaration(trait_def: &TraitDef) -> TraitDecl {
with(|cx| cx.trait_decl(trait_def))
@@ -932,6 +936,10 @@ crate_def! {
pub ImplDef;
}
+impl_crate_def_items! {
+ ImplDef;
+}
+
impl ImplDef {
/// Retrieve information about this implementation.
pub fn trait_impl(&self) -> ImplTrait {
@@ -1555,3 +1563,60 @@ index_impl!(Span);
pub struct VariantIdx(usize);
index_impl!(VariantIdx);
+
+crate_def! {
+ /// Hold infomation about an Opaque definition, particularly useful in `RPITIT`.
+ #[derive(Serialize)]
+ pub OpaqueDef;
+}
+
+crate_def! {
+ #[derive(Serialize)]
+ pub AssocDef;
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+pub struct AssocItem {
+ pub def_id: AssocDef,
+ pub name: Symbol,
+ pub kind: AssocKind,
+ pub container: AssocItemContainer,
+
+ /// If this is an item in an impl of a trait then this is the `DefId` of
+ /// the associated item on the trait that this implements.
+ pub trait_item_def_id: Option<AssocDef>,
+
+ /// Whether this is a method with an explicit self
+ /// as its first parameter, allowing method calls.
+ pub fn_has_self_parameter: bool,
+
+ /// `Some` if the associated item (an associated type) comes from the
+ /// return-position `impl Trait` in trait desugaring. The `ImplTraitInTraitData`
+ /// provides additional information about its source.
+ pub opt_rpitit_info: Option<ImplTraitInTraitData>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+pub enum AssocKind {
+ Const,
+ Fn,
+ Type,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+pub enum AssocItemContainer {
+ Trait,
+ Impl,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Serialize)]
+pub enum ImplTraitInTraitData {
+ Trait { fn_def_id: FnDef, opaque_def_id: OpaqueDef },
+ Impl { fn_def_id: FnDef },
+}
+
+impl AssocItem {
+ pub fn is_impl_trait_in_trait(&self) -> bool {
+ self.opt_rpitit_info.is_some()
+ }
+}
diff --git a/library/alloctests/tests/lib.rs b/library/alloctests/tests/lib.rs
index 46c11ea150bf8..f1f4cc6f93bbd 100644
--- a/library/alloctests/tests/lib.rs
+++ b/library/alloctests/tests/lib.rs
@@ -63,6 +63,7 @@ mod fmt;
mod heap;
mod linked_list;
mod misc_tests;
+mod num;
mod rc;
mod slice;
mod sort;
diff --git a/library/alloctests/tests/num.rs b/library/alloctests/tests/num.rs
new file mode 100644
index 0000000000000..3c76e68c60640
--- /dev/null
+++ b/library/alloctests/tests/num.rs
@@ -0,0 +1,69 @@
+use std::fmt::{Debug, Display};
+use std::str::FromStr;
+
+fn assert_nb<Int: ToString + FromStr + Debug + Display + Eq>(value: Int) {
+ let s = value.to_string();
+ let s2 = format!("s: {}.", value);
+
+ assert_eq!(format!("s: {s}."), s2);
+ let Ok(ret) = Int::from_str(&s) else {
+ panic!("failed to convert into to string");
+ };
+ assert_eq!(ret, value);
+}
+
+macro_rules! uint_to_s {
+ ($($fn_name:ident, $int:ident,)+) => {
+ $(
+ #[test]
+ fn $fn_name() {
+ assert_nb::<$int>($int::MIN);
+ assert_nb::<$int>($int::MAX);
+ assert_nb::<$int>(1);
+ assert_nb::<$int>($int::MIN / 2);
+ assert_nb::<$int>($int::MAX / 2);
+ }
+ )+
+ }
+}
+macro_rules! int_to_s {
+ ($($fn_name:ident, $int:ident,)+) => {
+ $(
+ #[test]
+ fn $fn_name() {
+ assert_nb::<$int>($int::MIN);
+ assert_nb::<$int>($int::MAX);
+ assert_nb::<$int>(1);
+ assert_nb::<$int>(0);
+ assert_nb::<$int>(-1);
+ assert_nb::<$int>($int::MIN / 2);
+ assert_nb::<$int>($int::MAX / 2);
+ }
+ )+
+ }
+}
+
+int_to_s!(
+ test_i8_to_string,
+ i8,
+ test_i16_to_string,
+ i16,
+ test_i32_to_string,
+ i32,
+ test_i64_to_string,
+ i64,
+ test_i128_to_string,
+ i128,
+);
+uint_to_s!(
+ test_u8_to_string,
+ u8,
+ test_u16_to_string,
+ u16,
+ test_u32_to_string,
+ u32,
+ test_u64_to_string,
+ u64,
+ test_u128_to_string,
+ u128,
+);
diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs
index 30fd2d7815f51..0c8e1495c62d4 100644
--- a/library/core/src/fmt/mod.rs
+++ b/library/core/src/fmt/mod.rs
@@ -743,6 +743,7 @@ impl<'a> Arguments<'a> {
#[unstable(feature = "fmt_internals", reason = "internal to standard library", issue = "none")]
#[must_use]
#[inline]
+ #[doc(hidden)]
pub fn as_statically_known_str(&self) -> Option<&'static str> {
let s = self.as_str();
if core::intrinsics::is_val_statically_known(s.is_some()) { s } else { None }
diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs
index 81e59a1f349ec..afd6192d7c473 100644
--- a/library/core/src/intrinsics/mod.rs
+++ b/library/core/src/intrinsics/mod.rs
@@ -2475,35 +2475,35 @@ pub unsafe fn float_to_int_unchecked<Float: Copy, Int: Copy>(value: Float) -> In
/// Float addition that allows optimizations based on algebraic rules.
///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_add`], [`f32::algebraic_add`], [`f64::algebraic_add`] and [`f128::algebraic_add`].
#[rustc_nounwind]
#[rustc_intrinsic]
pub fn fadd_algebraic<T: Copy>(a: T, b: T) -> T;
/// Float subtraction that allows optimizations based on algebraic rules.
///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_sub`], [`f32::algebraic_sub`], [`f64::algebraic_sub`] and [`f128::algebraic_sub`].
#[rustc_nounwind]
#[rustc_intrinsic]
pub fn fsub_algebraic<T: Copy>(a: T, b: T) -> T;
/// Float multiplication that allows optimizations based on algebraic rules.
///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_mul`], [`f32::algebraic_mul`], [`f64::algebraic_mul`] and [`f128::algebraic_mul`].
#[rustc_nounwind]
#[rustc_intrinsic]
pub fn fmul_algebraic<T: Copy>(a: T, b: T) -> T;
/// Float division that allows optimizations based on algebraic rules.
///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_div`], [`f32::algebraic_div`], [`f64::algebraic_div`] and [`f128::algebraic_div`].
#[rustc_nounwind]
#[rustc_intrinsic]
pub fn fdiv_algebraic<T: Copy>(a: T, b: T) -> T;
/// Float remainder that allows optimizations based on algebraic rules.
///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_rem`], [`f32::algebraic_rem`], [`f64::algebraic_rem`] and [`f128::algebraic_rem`].
#[rustc_nounwind]
#[rustc_intrinsic]
pub fn frem_algebraic<T: Copy>(a: T, b: T) -> T;
diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs
index b17190971c3e8..08c34e852da41 100644
--- a/library/core/src/num/f128.rs
+++ b/library/core/src/num/f128.rs
@@ -1362,4 +1362,54 @@ impl f128 {
// SAFETY: this is actually a safe intrinsic
unsafe { intrinsics::copysignf128(self, sign) }
}
+
+ /// Float addition that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_add(self, rhs: f128) -> f128 {
+ intrinsics::fadd_algebraic(self, rhs)
+ }
+
+ /// Float subtraction that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_sub(self, rhs: f128) -> f128 {
+ intrinsics::fsub_algebraic(self, rhs)
+ }
+
+ /// Float multiplication that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_mul(self, rhs: f128) -> f128 {
+ intrinsics::fmul_algebraic(self, rhs)
+ }
+
+ /// Float division that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_div(self, rhs: f128) -> f128 {
+ intrinsics::fdiv_algebraic(self, rhs)
+ }
+
+ /// Float remainder that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_rem(self, rhs: f128) -> f128 {
+ intrinsics::frem_algebraic(self, rhs)
+ }
}
diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs
index d20677f43b417..a33e5f5301469 100644
--- a/library/core/src/num/f16.rs
+++ b/library/core/src/num/f16.rs
@@ -1338,4 +1338,54 @@ impl f16 {
// SAFETY: this is actually a safe intrinsic
unsafe { intrinsics::copysignf16(self, sign) }
}
+
+ /// Float addition that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_add(self, rhs: f16) -> f16 {
+ intrinsics::fadd_algebraic(self, rhs)
+ }
+
+ /// Float subtraction that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_sub(self, rhs: f16) -> f16 {
+ intrinsics::fsub_algebraic(self, rhs)
+ }
+
+ /// Float multiplication that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_mul(self, rhs: f16) -> f16 {
+ intrinsics::fmul_algebraic(self, rhs)
+ }
+
+ /// Float division that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_div(self, rhs: f16) -> f16 {
+ intrinsics::fdiv_algebraic(self, rhs)
+ }
+
+ /// Float remainder that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_rem(self, rhs: f16) -> f16 {
+ intrinsics::frem_algebraic(self, rhs)
+ }
}
diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs
index 53373584d5551..e473fac03935a 100644
--- a/library/core/src/num/f32.rs
+++ b/library/core/src/num/f32.rs
@@ -1504,4 +1504,54 @@ impl f32 {
// SAFETY: this is actually a safe intrinsic
unsafe { intrinsics::copysignf32(self, sign) }
}
+
+ /// Float addition that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_add(self, rhs: f32) -> f32 {
+ intrinsics::fadd_algebraic(self, rhs)
+ }
+
+ /// Float subtraction that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_sub(self, rhs: f32) -> f32 {
+ intrinsics::fsub_algebraic(self, rhs)
+ }
+
+ /// Float multiplication that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_mul(self, rhs: f32) -> f32 {
+ intrinsics::fmul_algebraic(self, rhs)
+ }
+
+ /// Float division that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_div(self, rhs: f32) -> f32 {
+ intrinsics::fdiv_algebraic(self, rhs)
+ }
+
+ /// Float remainder that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_rem(self, rhs: f32) -> f32 {
+ intrinsics::frem_algebraic(self, rhs)
+ }
}
diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs
index ca28b40bb3adc..6522a80b0b7e8 100644
--- a/library/core/src/num/f64.rs
+++ b/library/core/src/num/f64.rs
@@ -1503,4 +1503,54 @@ impl f64 {
// SAFETY: this is actually a safe intrinsic
unsafe { intrinsics::copysignf64(self, sign) }
}
+
+ /// Float addition that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_add(self, rhs: f64) -> f64 {
+ intrinsics::fadd_algebraic(self, rhs)
+ }
+
+ /// Float subtraction that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_sub(self, rhs: f64) -> f64 {
+ intrinsics::fsub_algebraic(self, rhs)
+ }
+
+ /// Float multiplication that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_mul(self, rhs: f64) -> f64 {
+ intrinsics::fmul_algebraic(self, rhs)
+ }
+
+ /// Float division that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_div(self, rhs: f64) -> f64 {
+ intrinsics::fdiv_algebraic(self, rhs)
+ }
+
+ /// Float remainder that allows optimizations based on algebraic rules.
+ ///
+ /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+ #[must_use = "method returns a new number and does not mutate the original value"]
+ #[unstable(feature = "float_algebraic", issue = "136469")]
+ #[inline]
+ pub fn algebraic_rem(self, rhs: f64) -> f64 {
+ intrinsics::frem_algebraic(self, rhs)
+ }
}
diff --git a/library/core/src/primitive_docs.rs b/library/core/src/primitive_docs.rs
index ba4c849837e74..369bf18c2b9f7 100644
--- a/library/core/src/primitive_docs.rs
+++ b/library/core/src/primitive_docs.rs
@@ -1313,6 +1313,51 @@ mod prim_f16 {}
/// | `wasm32`, `wasm64` | If all input NaNs are quiet with all-zero payload: None.<br> Otherwise: all possible payloads. |
///
/// For targets not in this table, all payloads are possible.
+///
+/// # Algebraic operators
+///
+/// Algebraic operators of the form `a.algebraic_*(b)` allow the compiler to optimize
+/// floating point operations using all the usual algebraic properties of real numbers --
+/// despite the fact that those properties do *not* hold on floating point numbers.
+/// This can give a great performance boost since it may unlock vectorization.
+///
+/// The exact set of optimizations is unspecified but typically allows combining operations,
+/// rearranging series of operations based on mathematical properties, converting between division
+/// and reciprocal multiplication, and disregarding the sign of zero. This means that the results of
+/// elementary operations may have undefined precision, and "non-mathematical" values
+/// such as NaN, +/-Inf, or -0.0 may behave in unexpected ways, but these operations
+/// will never cause undefined behavior.
+///
+/// Because of the unpredictable nature of compiler optimizations, the same inputs may produce
+/// different results even within a single program run. **Unsafe code must not rely on any property
+/// of the return value for soundness.** However, implementations will generally do their best to
+/// pick a reasonable tradeoff between performance and accuracy of the result.
+///
+/// For example:
+///
+/// ```
+/// # #![feature(float_algebraic)]
+/// # #![allow(unused_assignments)]
+/// # let mut x: f32 = 0.0;
+/// # let a: f32 = 1.0;
+/// # let b: f32 = 2.0;
+/// # let c: f32 = 3.0;
+/// # let d: f32 = 4.0;
+/// x = a.algebraic_add(b).algebraic_add(c).algebraic_add(d);
+/// ```
+///
+/// May be rewritten as:
+///
+/// ```
+/// # #![allow(unused_assignments)]
+/// # let mut x: f32 = 0.0;
+/// # let a: f32 = 1.0;
+/// # let b: f32 = 2.0;
+/// # let c: f32 = 3.0;
+/// # let d: f32 = 4.0;
+/// x = a + b + c + d; // As written
+/// x = (a + c) + (b + d); // Reordered to shorten critical path and enable vectorization
+/// ```
#[stable(feature = "rust1", since = "1.0.0")]
mod prim_f32 {}
diff --git a/library/core/src/unicode/unicode_data.rs b/library/core/src/unicode/unicode_data.rs
index 4655d35e9c437..25b9c6e0e0e94 100644
--- a/library/core/src/unicode/unicode_data.rs
+++ b/library/core/src/unicode/unicode_data.rs
@@ -47,45 +47,78 @@ const fn bitset_search<
(word & (1 << (needle % 64) as u64)) != 0
}
-fn decode_prefix_sum(short_offset_run_header: u32) -> u32 {
- short_offset_run_header & ((1 << 21) - 1)
-}
+#[repr(transparent)]
+struct ShortOffsetRunHeader(u32);
+
+impl ShortOffsetRunHeader {
+ const fn new(start_index: usize, prefix_sum: u32) -> Self {
+ assert!(start_index < (1 << 11));
+ assert!(prefix_sum < (1 << 21));
+
+ Self((start_index as u32) << 21 | prefix_sum)
+ }
-fn decode_length(short_offset_run_header: u32) -> usize {
- (short_offset_run_header >> 21) as usize
+ #[inline]
+ const fn start_index(&self) -> usize {
+ (self.0 >> 21) as usize
+ }
+
+ #[inline]
+ const fn prefix_sum(&self) -> u32 {
+ self.0 & ((1 << 21) - 1)
+ }
}
+/// # Safety
+///
+/// - The last element of `short_offset_runs` must be greater than `std::char::MAX`.
+/// - The start indices of all elements in `short_offset_runs` must be less than `OFFSETS`.
#[inline(always)]
-fn skip_search<const SOR: usize, const OFFSETS: usize>(
- needle: u32,
- short_offset_runs: &[u32; SOR],
+unsafe fn skip_search<const SOR: usize, const OFFSETS: usize>(
+ needle: char,
+ short_offset_runs: &[ShortOffsetRunHeader; SOR],
offsets: &[u8; OFFSETS],
) -> bool {
- // Note that this *cannot* be past the end of the array, as the last
- // element is greater than std::char::MAX (the largest possible needle).
- //
- // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct
- // location cannot be past it, so Err(idx) != length either.
- //
- // This means that we can avoid bounds checking for the accesses below, too.
+ let needle = needle as u32;
+
let last_idx =
- match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) {
+ match short_offset_runs.binary_search_by_key(&(needle << 11), |header| (header.0 << 11)) {
Ok(idx) => idx + 1,
Err(idx) => idx,
};
+ // SAFETY: `last_idx` *cannot* be past the end of the array, as the last
+ // element is greater than `std::char::MAX` (the largest possible needle)
+ // as guaranteed by the caller.
+ //
+ // So, we cannot have found it (i.e. `Ok(idx) => idx + 1 != length`) and the
+ // correct location cannot be past it, so `Err(idx) => idx != length` either.
+ //
+ // This means that we can avoid bounds checking for the accesses below, too.
+ //
+ // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+ // in `hint::assert_unchecked` may not be optimized out.
+ unsafe { crate::intrinsics::assume(last_idx < SOR) };
- let mut offset_idx = decode_length(short_offset_runs[last_idx]);
+ let mut offset_idx = short_offset_runs[last_idx].start_index();
let length = if let Some(next) = short_offset_runs.get(last_idx + 1) {
- decode_length(*next) - offset_idx
+ (*next).start_index() - offset_idx
} else {
offsets.len() - offset_idx
};
+
let prev =
- last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0);
+ last_idx.checked_sub(1).map(|prev| short_offset_runs[prev].prefix_sum()).unwrap_or(0);
let total = needle - prev;
let mut prefix_sum = 0;
for _ in 0..(length - 1) {
+ // SAFETY: It is guaranteed that `length <= OFFSETS - offset_idx`,
+ // so it follows that `length - 1 + offset_idx < OFFSETS`, therefore
+ // `offset_idx < OFFSETS` is always true in this loop.
+ //
+ // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+ // in `hint::assert_unchecked` may not be optimized out.
+ unsafe { crate::intrinsics::assume(offset_idx < OFFSETS) };
let offset = offsets[offset_idx];
prefix_sum += offset as u32;
if prefix_sum > total {
@@ -100,15 +133,36 @@ pub const UNICODE_VERSION: (u8, u8, u8) = (16, 0, 0);
#[rustfmt::skip]
pub mod alphabetic {
- static SHORT_OFFSET_RUNS: [u32; 53] = [
- 706, 33559113, 876615277, 956309270, 1166025910, 1314925568, 1319120901, 1398813696,
- 1449151936, 1451271309, 1455465997, 1463867300, 1652619520, 1663105646, 1665203518,
- 1711342208, 1797326647, 1895898848, 2560697242, 2583768976, 2594255920, 2600551419,
- 2608940615, 2613141760, 2615240704, 2619435577, 2621533504, 2652997624, 2688650454,
- 2692853744, 2699145507, 2713826044, 2734799872, 2736903168, 2757875366, 2835472128,
- 2883707536, 2934039760, 2942429152, 2955013632, 2988568880, 3126984704, 3139610336,
- 3141711674, 3145911970, 3154308065, 3158503006, 3162699776, 3164797470, 3166896128,
- 3168998219, 3171099568, 3176407984,
+ use super::ShortOffsetRunHeader;
+
+ static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 53] = [
+ ShortOffsetRunHeader::new(0, 706), ShortOffsetRunHeader::new(16, 4681),
+ ShortOffsetRunHeader::new(418, 5741), ShortOffsetRunHeader::new(456, 7958),
+ ShortOffsetRunHeader::new(556, 9398), ShortOffsetRunHeader::new(627, 11264),
+ ShortOffsetRunHeader::new(629, 12293), ShortOffsetRunHeader::new(667, 13312),
+ ShortOffsetRunHeader::new(691, 19904), ShortOffsetRunHeader::new(692, 42125),
+ ShortOffsetRunHeader::new(694, 42509), ShortOffsetRunHeader::new(698, 55204),
+ ShortOffsetRunHeader::new(788, 63744), ShortOffsetRunHeader::new(793, 64110),
+ ShortOffsetRunHeader::new(794, 64830), ShortOffsetRunHeader::new(816, 66176),
+ ShortOffsetRunHeader::new(857, 67383), ShortOffsetRunHeader::new(904, 73440),
+ ShortOffsetRunHeader::new(1221, 74650), ShortOffsetRunHeader::new(1232, 77712),
+ ShortOffsetRunHeader::new(1237, 78896), ShortOffsetRunHeader::new(1240, 82939),
+ ShortOffsetRunHeader::new(1244, 83527), ShortOffsetRunHeader::new(1246, 90368),
+ ShortOffsetRunHeader::new(1247, 92160), ShortOffsetRunHeader::new(1249, 92729),
+ ShortOffsetRunHeader::new(1250, 93504), ShortOffsetRunHeader::new(1265, 100344),
+ ShortOffsetRunHeader::new(1282, 101590), ShortOffsetRunHeader::new(1284, 110576),
+ ShortOffsetRunHeader::new(1287, 110883), ShortOffsetRunHeader::new(1294, 111356),
+ ShortOffsetRunHeader::new(1304, 113664), ShortOffsetRunHeader::new(1305, 119808),
+ ShortOffsetRunHeader::new(1315, 120486), ShortOffsetRunHeader::new(1352, 122624),
+ ShortOffsetRunHeader::new(1375, 123536), ShortOffsetRunHeader::new(1399, 124112),
+ ShortOffsetRunHeader::new(1403, 124896), ShortOffsetRunHeader::new(1409, 126464),
+ ShortOffsetRunHeader::new(1425, 127280), ShortOffsetRunHeader::new(1491, 131072),
+ ShortOffsetRunHeader::new(1497, 173792), ShortOffsetRunHeader::new(1498, 177978),
+ ShortOffsetRunHeader::new(1500, 183970), ShortOffsetRunHeader::new(1504, 191457),
+ ShortOffsetRunHeader::new(1506, 192094), ShortOffsetRunHeader::new(1508, 194560),
+ ShortOffsetRunHeader::new(1509, 195102), ShortOffsetRunHeader::new(1510, 196608),
+ ShortOffsetRunHeader::new(1511, 201547), ShortOffsetRunHeader::new(1512, 205744),
+ ShortOffsetRunHeader::new(1514, 1319856),
];
static OFFSETS: [u8; 1515] = [
65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 0, 4, 12, 14, 5, 7, 1, 1, 1, 86, 1, 29,
@@ -169,22 +223,44 @@ pub mod alphabetic {
0, 0, 0, 0, 5, 0, 0,
];
pub fn lookup(c: char) -> bool {
- super::skip_search(
- c as u32,
- &SHORT_OFFSET_RUNS,
- &OFFSETS,
- )
+ const {
+ assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+ let mut i = 0;
+ while i < SHORT_OFFSET_RUNS.len() {
+ assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+ i += 1;
+ }
+ }
+ // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+ // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+ unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
}
}
#[rustfmt::skip]
pub mod case_ignorable {
- static SHORT_OFFSET_RUNS: [u32; 37] = [
- 688, 44045149, 572528402, 576724925, 807414908, 878718981, 903913493, 929080568, 933275148,
- 937491230, 1138818560, 1147208189, 1210124160, 1222707713, 1235291428, 1260457643,
- 1277237295, 1537284411, 1545673776, 1604394739, 1667314736, 1692492062, 1700883184,
- 1709272384, 1721855823, 1730260976, 1747041437, 1759629056, 1768018279, 1776409088,
- 1797382144, 1822548654, 1856103659, 1864493264, 1872884731, 1882062849, 1887371760,
+ use super::ShortOffsetRunHeader;
+
+ static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 37] = [
+ ShortOffsetRunHeader::new(0, 688), ShortOffsetRunHeader::new(21, 4957),
+ ShortOffsetRunHeader::new(273, 5906), ShortOffsetRunHeader::new(275, 8125),
+ ShortOffsetRunHeader::new(385, 11388), ShortOffsetRunHeader::new(419, 12293),
+ ShortOffsetRunHeader::new(431, 40981), ShortOffsetRunHeader::new(443, 42232),
+ ShortOffsetRunHeader::new(445, 42508), ShortOffsetRunHeader::new(447, 64286),
+ ShortOffsetRunHeader::new(543, 65024), ShortOffsetRunHeader::new(547, 66045),
+ ShortOffsetRunHeader::new(577, 67456), ShortOffsetRunHeader::new(583, 68097),
+ ShortOffsetRunHeader::new(589, 68900), ShortOffsetRunHeader::new(601, 69291),
+ ShortOffsetRunHeader::new(609, 71727), ShortOffsetRunHeader::new(733, 71995),
+ ShortOffsetRunHeader::new(737, 72752), ShortOffsetRunHeader::new(765, 73459),
+ ShortOffsetRunHeader::new(795, 78896), ShortOffsetRunHeader::new(807, 90398),
+ ShortOffsetRunHeader::new(811, 92912), ShortOffsetRunHeader::new(815, 93504),
+ ShortOffsetRunHeader::new(821, 94031), ShortOffsetRunHeader::new(825, 110576),
+ ShortOffsetRunHeader::new(833, 113821), ShortOffsetRunHeader::new(839, 118528),
+ ShortOffsetRunHeader::new(843, 119143), ShortOffsetRunHeader::new(847, 121344),
+ ShortOffsetRunHeader::new(857, 122880), ShortOffsetRunHeader::new(869, 123566),
+ ShortOffsetRunHeader::new(885, 124139), ShortOffsetRunHeader::new(889, 125136),
+ ShortOffsetRunHeader::new(893, 127995), ShortOffsetRunHeader::new(897, 917505),
+ ShortOffsetRunHeader::new(899, 2032112),
];
static OFFSETS: [u8; 905] = [
39, 1, 6, 1, 11, 1, 35, 1, 1, 1, 71, 1, 4, 1, 1, 1, 4, 1, 2, 2, 0, 192, 4, 2, 4, 1, 9, 2,
@@ -222,20 +298,36 @@ pub mod case_ignorable {
1, 61, 4, 0, 5, 254, 2, 0, 7, 109, 8, 0, 5, 0, 1, 30, 96, 128, 240, 0,
];
pub fn lookup(c: char) -> bool {
- super::skip_search(
- c as u32,
- &SHORT_OFFSET_RUNS,
- &OFFSETS,
- )
+ const {
+ assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+ let mut i = 0;
+ while i < SHORT_OFFSET_RUNS.len() {
+ assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+ i += 1;
+ }
+ }
+ // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+ // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+ unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
}
}
#[rustfmt::skip]
pub mod cased {
- static SHORT_OFFSET_RUNS: [u32; 22] = [
- 4256, 115348384, 136322176, 144711446, 163587254, 320875520, 325101120, 350268208,
- 392231680, 404815649, 413205504, 421595008, 467733632, 484513952, 501313088, 505533440,
- 509728422, 587325184, 635559984, 648145152, 652341552, 657650058,
+ use super::ShortOffsetRunHeader;
+
+ static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 22] = [
+ ShortOffsetRunHeader::new(0, 4256), ShortOffsetRunHeader::new(55, 5024),
+ ShortOffsetRunHeader::new(65, 7296), ShortOffsetRunHeader::new(69, 7958),
+ ShortOffsetRunHeader::new(78, 9398), ShortOffsetRunHeader::new(153, 11264),
+ ShortOffsetRunHeader::new(155, 42560), ShortOffsetRunHeader::new(167, 43824),
+ ShortOffsetRunHeader::new(187, 64256), ShortOffsetRunHeader::new(193, 65313),
+ ShortOffsetRunHeader::new(197, 66560), ShortOffsetRunHeader::new(201, 67456),
+ ShortOffsetRunHeader::new(223, 68736), ShortOffsetRunHeader::new(231, 71840),
+ ShortOffsetRunHeader::new(239, 93760), ShortOffsetRunHeader::new(241, 119808),
+ ShortOffsetRunHeader::new(243, 120486), ShortOffsetRunHeader::new(280, 122624),
+ ShortOffsetRunHeader::new(303, 122928), ShortOffsetRunHeader::new(309, 125184),
+ ShortOffsetRunHeader::new(311, 127280), ShortOffsetRunHeader::new(313, 1241482),
];
static OFFSETS: [u8; 319] = [
65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 195, 1, 4, 4, 208, 1, 36, 7, 2, 30, 5,
@@ -252,39 +344,67 @@ pub mod cased {
8, 0, 10, 1, 20, 6, 6, 0, 62, 0, 68, 0, 26, 6, 26, 6, 26, 0,
];
pub fn lookup(c: char) -> bool {
- super::skip_search(
- c as u32,
- &SHORT_OFFSET_RUNS,
- &OFFSETS,
- )
+ const {
+ assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+ let mut i = 0;
+ while i < SHORT_OFFSET_RUNS.len() {
+ assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+ i += 1;
+ }
+ }
+ // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+ // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+ unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
}
}
#[rustfmt::skip]
pub mod cc {
- static SHORT_OFFSET_RUNS: [u32; 1] = [
- 1114272,
+ use super::ShortOffsetRunHeader;
+
+ static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 1] = [
+ ShortOffsetRunHeader::new(0, 1114272),
];
static OFFSETS: [u8; 5] = [
0, 32, 95, 33, 0,
];
pub fn lookup(c: char) -> bool {
- super::skip_search(
- c as u32,
- &SHORT_OFFSET_RUNS,
- &OFFSETS,
- )
+ const {
+ assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+ let mut i = 0;
+ while i < SHORT_OFFSET_RUNS.len() {
+ assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+ i += 1;
+ }
+ }
+ // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+ // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+ unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
}
}
#[rustfmt::skip]
pub mod grapheme_extend {
- static SHORT_OFFSET_RUNS: [u32; 34] = [
- 768, 2098307, 6292881, 10490717, 522196754, 526393356, 723528943, 731918378, 744531567,
- 752920578, 769719070, 908131840, 912326558, 920715773, 924912129, 937495844, 962662059,
- 971053103, 1256266800, 1323376371, 1386296384, 1407279390, 1415670512, 1424060239,
- 1432468637, 1449250560, 1453445477, 1461836288, 1487003648, 1512170158, 1541530860,
- 1549920464, 1559101472, 1568604656,
+ use super::ShortOffsetRunHeader;
+
+ static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 34] = [
+ ShortOffsetRunHeader::new(0, 768), ShortOffsetRunHeader::new(1, 1155),
+ ShortOffsetRunHeader::new(3, 1425), ShortOffsetRunHeader::new(5, 4957),
+ ShortOffsetRunHeader::new(249, 5906), ShortOffsetRunHeader::new(251, 8204),
+ ShortOffsetRunHeader::new(345, 11503), ShortOffsetRunHeader::new(349, 12330),
+ ShortOffsetRunHeader::new(355, 42607), ShortOffsetRunHeader::new(359, 43010),
+ ShortOffsetRunHeader::new(367, 64286), ShortOffsetRunHeader::new(433, 65024),
+ ShortOffsetRunHeader::new(435, 65438), ShortOffsetRunHeader::new(439, 66045),
+ ShortOffsetRunHeader::new(441, 68097), ShortOffsetRunHeader::new(447, 68900),
+ ShortOffsetRunHeader::new(459, 69291), ShortOffsetRunHeader::new(463, 71727),
+ ShortOffsetRunHeader::new(599, 72752), ShortOffsetRunHeader::new(631, 73459),
+ ShortOffsetRunHeader::new(661, 78912), ShortOffsetRunHeader::new(671, 90398),
+ ShortOffsetRunHeader::new(675, 92912), ShortOffsetRunHeader::new(679, 94031),
+ ShortOffsetRunHeader::new(683, 113821), ShortOffsetRunHeader::new(691, 118528),
+ ShortOffsetRunHeader::new(693, 119141), ShortOffsetRunHeader::new(697, 121344),
+ ShortOffsetRunHeader::new(709, 122880), ShortOffsetRunHeader::new(721, 123566),
+ ShortOffsetRunHeader::new(735, 124140), ShortOffsetRunHeader::new(739, 125136),
+ ShortOffsetRunHeader::new(743, 917536), ShortOffsetRunHeader::new(747, 2032112),
];
static OFFSETS: [u8; 751] = [
0, 112, 0, 7, 0, 45, 1, 1, 1, 2, 1, 2, 1, 1, 72, 11, 48, 21, 16, 1, 101, 7, 2, 6, 2, 2, 1,
@@ -319,12 +439,20 @@ pub mod grapheme_extend {
pub fn lookup(c: char) -> bool {
(c as u32) >= 0x300 && lookup_slow(c)
}
+
+ #[inline(never)]
fn lookup_slow(c: char) -> bool {
- super::skip_search(
- c as u32,
- &SHORT_OFFSET_RUNS,
- &OFFSETS,
- )
+ const {
+ assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+ let mut i = 0;
+ while i < SHORT_OFFSET_RUNS.len() {
+ assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+ i += 1;
+ }
+ }
+ // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+ // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+ unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
}
}
@@ -436,13 +564,30 @@ pub mod lowercase {
#[rustfmt::skip]
pub mod n {
- static SHORT_OFFSET_RUNS: [u32; 42] = [
- 1632, 18876774, 31461440, 102765417, 111154926, 115349830, 132128880, 165684320, 186656630,
- 195046653, 199241735, 203436434, 216049184, 241215536, 249605104, 274792208, 278987015,
- 283181793, 295766104, 320933114, 383848032, 396432464, 438376016, 446765280, 463543280,
- 471932752, 488711168, 497115440, 501312096, 505507184, 522284672, 526503152, 530698944,
- 534894542, 547479872, 551674608, 555869424, 560064711, 568454257, 576844032, 597818352,
- 603126778,
+ use super::ShortOffsetRunHeader;
+
+ static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 42] = [
+ ShortOffsetRunHeader::new(0, 1632), ShortOffsetRunHeader::new(9, 2406),
+ ShortOffsetRunHeader::new(15, 4160), ShortOffsetRunHeader::new(49, 4969),
+ ShortOffsetRunHeader::new(53, 5870), ShortOffsetRunHeader::new(55, 6470),
+ ShortOffsetRunHeader::new(63, 8304), ShortOffsetRunHeader::new(79, 9312),
+ ShortOffsetRunHeader::new(89, 10102), ShortOffsetRunHeader::new(93, 11517),
+ ShortOffsetRunHeader::new(95, 12295), ShortOffsetRunHeader::new(97, 12690),
+ ShortOffsetRunHeader::new(103, 42528), ShortOffsetRunHeader::new(115, 43056),
+ ShortOffsetRunHeader::new(119, 44016), ShortOffsetRunHeader::new(131, 65296),
+ ShortOffsetRunHeader::new(133, 65799), ShortOffsetRunHeader::new(135, 66273),
+ ShortOffsetRunHeader::new(141, 67672), ShortOffsetRunHeader::new(153, 68858),
+ ShortOffsetRunHeader::new(183, 69216), ShortOffsetRunHeader::new(189, 70736),
+ ShortOffsetRunHeader::new(209, 71248), ShortOffsetRunHeader::new(213, 71904),
+ ShortOffsetRunHeader::new(221, 72688), ShortOffsetRunHeader::new(225, 73552),
+ ShortOffsetRunHeader::new(233, 74752), ShortOffsetRunHeader::new(237, 90416),
+ ShortOffsetRunHeader::new(239, 92768), ShortOffsetRunHeader::new(241, 93552),
+ ShortOffsetRunHeader::new(249, 93824), ShortOffsetRunHeader::new(251, 118000),
+ ShortOffsetRunHeader::new(253, 119488), ShortOffsetRunHeader::new(255, 120782),
+ ShortOffsetRunHeader::new(261, 123200), ShortOffsetRunHeader::new(263, 123632),
+ ShortOffsetRunHeader::new(265, 124144), ShortOffsetRunHeader::new(267, 125127),
+ ShortOffsetRunHeader::new(271, 126065), ShortOffsetRunHeader::new(275, 127232),
+ ShortOffsetRunHeader::new(285, 130032), ShortOffsetRunHeader::new(287, 1244154),
];
static OFFSETS: [u8; 289] = [
48, 10, 120, 2, 5, 1, 2, 3, 0, 10, 134, 10, 198, 10, 0, 10, 118, 10, 4, 6, 108, 10, 118,
@@ -459,11 +604,17 @@ pub mod n {
10, 247, 10, 0, 9, 128, 10, 0, 59, 1, 3, 1, 4, 76, 45, 1, 15, 0, 13, 0, 10, 0,
];
pub fn lookup(c: char) -> bool {
- super::skip_search(
- c as u32,
- &SHORT_OFFSET_RUNS,
- &OFFSETS,
- )
+ const {
+ assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+ let mut i = 0;
+ while i < SHORT_OFFSET_RUNS.len() {
+ assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+ i += 1;
+ }
+ }
+ // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+ // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+ unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
}
}
diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs
index 9dcedaa13f661..5c381181218df 100644
--- a/library/std/src/lib.rs
+++ b/library/std/src/lib.rs
@@ -341,6 +341,7 @@
#![feature(exact_size_is_empty)]
#![feature(exclusive_wrapper)]
#![feature(extend_one)]
+#![feature(float_algebraic)]
#![feature(float_gamma)]
#![feature(float_minimum_maximum)]
#![feature(fmt_internals)]
diff --git a/library/std/src/sys/thread_local/mod.rs b/library/std/src/sys/thread_local/mod.rs
index 1ff13154b7b3c..b7ce6fcdc05a7 100644
--- a/library/std/src/sys/thread_local/mod.rs
+++ b/library/std/src/sys/thread_local/mod.rs
@@ -138,6 +138,7 @@ pub(crate) mod key {
not(target_family = "wasm"),
target_family = "unix",
),
+ all(not(target_thread_local), target_vendor = "apple"),
target_os = "teeos",
all(target_os = "wasi", target_env = "p1", target_feature = "atomics"),
))] {
diff --git a/library/std/tests/floats/f128.rs b/library/std/tests/floats/f128.rs
index b4a6c672bf05f..df28e8129ddd9 100644
--- a/library/std/tests/floats/f128.rs
+++ b/library/std/tests/floats/f128.rs
@@ -984,6 +984,25 @@ fn test_total_cmp() {
assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
}
+#[test]
+fn test_algebraic() {
+ let a: f128 = 123.0;
+ let b: f128 = 456.0;
+
+ // Check that individual operations match their primitive counterparts.
+ //
+ // This is a check of current implementations and does NOT imply any form of
+ // guarantee about future behavior. The compiler reserves the right to make
+ // these operations inexact matches in the future.
+ let eps = if cfg!(miri) { 1e-6 } else { 0.0 };
+
+ assert_approx_eq!(a.algebraic_add(b), a + b, eps);
+ assert_approx_eq!(a.algebraic_sub(b), a - b, eps);
+ assert_approx_eq!(a.algebraic_mul(b), a * b, eps);
+ assert_approx_eq!(a.algebraic_div(b), a / b, eps);
+ assert_approx_eq!(a.algebraic_rem(b), a % b, eps);
+}
+
#[test]
fn test_from() {
assert_eq!(f128::from(false), 0.0);
diff --git a/library/std/tests/floats/f16.rs b/library/std/tests/floats/f16.rs
index ca0b8efbe83ba..1a90f00aecceb 100644
--- a/library/std/tests/floats/f16.rs
+++ b/library/std/tests/floats/f16.rs
@@ -954,6 +954,27 @@ fn test_total_cmp() {
assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
}
+#[test]
+fn test_algebraic() {
+ let a: f16 = 123.0;
+ let b: f16 = 456.0;
+
+ // Check that individual operations match their primitive counterparts.
+ //
+ // This is a check of current implementations and does NOT imply any form of
+ // guarantee about future behavior. The compiler reserves the right to make
+ // these operations inexact matches in the future.
+ let eps_add = if cfg!(miri) { 1e1 } else { 0.0 };
+ let eps_mul = if cfg!(miri) { 1e3 } else { 0.0 };
+ let eps_div = if cfg!(miri) { 1e0 } else { 0.0 };
+
+ assert_approx_eq!(a.algebraic_add(b), a + b, eps_add);
+ assert_approx_eq!(a.algebraic_sub(b), a - b, eps_add);
+ assert_approx_eq!(a.algebraic_mul(b), a * b, eps_mul);
+ assert_approx_eq!(a.algebraic_div(b), a / b, eps_div);
+ assert_approx_eq!(a.algebraic_rem(b), a % b, eps_div);
+}
+
#[test]
fn test_from() {
assert_eq!(f16::from(false), 0.0);
diff --git a/library/std/tests/floats/f32.rs b/library/std/tests/floats/f32.rs
index bf7641986ada8..d99b03cb255f7 100644
--- a/library/std/tests/floats/f32.rs
+++ b/library/std/tests/floats/f32.rs
@@ -915,3 +915,24 @@ fn test_total_cmp() {
assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f32::INFINITY));
assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
}
+
+#[test]
+fn test_algebraic() {
+ let a: f32 = 123.0;
+ let b: f32 = 456.0;
+
+ // Check that individual operations match their primitive counterparts.
+ //
+ // This is a check of current implementations and does NOT imply any form of
+ // guarantee about future behavior. The compiler reserves the right to make
+ // these operations inexact matches in the future.
+ let eps_add = if cfg!(miri) { 1e-3 } else { 0.0 };
+ let eps_mul = if cfg!(miri) { 1e-1 } else { 0.0 };
+ let eps_div = if cfg!(miri) { 1e-4 } else { 0.0 };
+
+ assert_approx_eq!(a.algebraic_add(b), a + b, eps_add);
+ assert_approx_eq!(a.algebraic_sub(b), a - b, eps_add);
+ assert_approx_eq!(a.algebraic_mul(b), a * b, eps_mul);
+ assert_approx_eq!(a.algebraic_div(b), a / b, eps_div);
+ assert_approx_eq!(a.algebraic_rem(b), a % b, eps_div);
+}
diff --git a/library/std/tests/floats/f64.rs b/library/std/tests/floats/f64.rs
index cbbfcd15efd26..611670751bb52 100644
--- a/library/std/tests/floats/f64.rs
+++ b/library/std/tests/floats/f64.rs
@@ -894,3 +894,22 @@ fn test_total_cmp() {
assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f64::INFINITY));
assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
}
+
+#[test]
+fn test_algebraic() {
+ let a: f64 = 123.0;
+ let b: f64 = 456.0;
+
+ // Check that individual operations match their primitive counterparts.
+ //
+ // This is a check of current implementations and does NOT imply any form of
+ // guarantee about future behavior. The compiler reserves the right to make
+ // these operations inexact matches in the future.
+ let eps = if cfg!(miri) { 1e-6 } else { 0.0 };
+
+ assert_approx_eq!(a.algebraic_add(b), a + b, eps);
+ assert_approx_eq!(a.algebraic_sub(b), a - b, eps);
+ assert_approx_eq!(a.algebraic_mul(b), a * b, eps);
+ assert_approx_eq!(a.algebraic_div(b), a / b, eps);
+ assert_approx_eq!(a.algebraic_rem(b), a % b, eps);
+}
diff --git a/library/std/tests/floats/lib.rs b/library/std/tests/floats/lib.rs
index ad82f1a44e711..de5a3cdbd0f93 100644
--- a/library/std/tests/floats/lib.rs
+++ b/library/std/tests/floats/lib.rs
@@ -1,4 +1,4 @@
-#![feature(f16, f128, float_gamma, float_minimum_maximum)]
+#![feature(f16, f128, float_algebraic, float_gamma, float_minimum_maximum)]
use std::fmt;
use std::ops::{Add, Div, Mul, Rem, Sub};
@@ -10,7 +10,7 @@ macro_rules! assert_approx_eq {
let (a, b) = (&$a, &$b);
let diff = (*a - *b).abs();
assert!(
- diff < $lim,
+ diff <= $lim,
"{a:?} is not approximately equal to {b:?} (threshold {lim:?}, difference {diff:?})",
lim = $lim
);
diff --git a/library/std/tests/thread_local/lib.rs b/library/std/tests/thread_local/lib.rs
index c52914354253c..26af5f1eb0a9d 100644
--- a/library/std/tests/thread_local/lib.rs
+++ b/library/std/tests/thread_local/lib.rs
@@ -1,3 +1,5 @@
+#![feature(cfg_target_thread_local)]
+
#[cfg(not(any(target_os = "emscripten", target_os = "wasi")))]
mod tests;
diff --git a/library/std/tests/thread_local/tests.rs b/library/std/tests/thread_local/tests.rs
index aa020c2559cc5..e8278361d9337 100644
--- a/library/std/tests/thread_local/tests.rs
+++ b/library/std/tests/thread_local/tests.rs
@@ -1,7 +1,7 @@
use std::cell::{Cell, UnsafeCell};
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::{Arc, Condvar, Mutex};
-use std::thread::{self, Builder, LocalKey};
+use std::thread::{self, LocalKey};
use std::thread_local;
#[derive(Clone, Default)]
@@ -345,8 +345,27 @@ fn join_orders_after_tls_destructors() {
}
// Test that thread::current is still available in TLS destructors.
+//
+// The test won't currently work without target_thread_local, aka with slow tls.
+// The runtime tries very hard to drop last the TLS variable that keeps the information about the
+// current thread, by using several tricks like deffering the drop to a later round of TLS destruction.
+// However, this only seems to work with fast tls.
+//
+// With slow TLS, it seems that multiple libc implementations will just set the value to null the first
+// time they encounter it, regardless of it having a destructor or not. This means that trying to
+// retrieve it later in a drop impl of another TLS variable will not work.
+//
+// ** Apple libc: https://github.com/apple-oss-distributions/libpthread/blob/c032e0b076700a0a47db75528a282b8d3a06531a/src/pthread_tsd.c#L293
+// Sets the variable to null if it has a destructor and the value is not null. However, all variables
+// created with pthread_key_create are marked as having a destructor, even if the fn ptr called with
+// it is null.
+// ** glibc: https://github.com/bminor/glibc/blob/e5893e6349541d871e8a25120bca014551d13ff5/nptl/nptl_deallocate_tsd.c#L59
+// ** musl: https://github.com/kraj/musl/blob/1880359b54ff7dd9f5016002bfdae4b136007dde/src/thread/pthread_key_create.c#L87
+#[cfg(target_thread_local)]
#[test]
fn thread_current_in_dtor() {
+ use std::thread::Builder;
+
// Go through one round of TLS destruction first.
struct Defer;
impl Drop for Defer {
diff --git a/src/librustdoc/html/static/js/main.js b/src/librustdoc/html/static/js/main.js
index 4150c5609a97e..a7ce2bf9048bf 100644
--- a/src/librustdoc/html/static/js/main.js
+++ b/src/librustdoc/html/static/js/main.js
@@ -1101,7 +1101,6 @@ function preLoadCss(cssUrl) {
});
}());
- // @ts-expect-error
window.rustdoc_add_line_numbers_to_examples = () => {
// @ts-expect-error
function generateLine(nb) {
@@ -1123,7 +1122,6 @@ function preLoadCss(cssUrl) {
});
};
- // @ts-expect-error
window.rustdoc_remove_line_numbers_from_examples = () => {
onEachLazy(
document.querySelectorAll(".rustdoc:not(.src) :not(.scraped-example) > .example-wrap"),
@@ -1132,7 +1130,6 @@ function preLoadCss(cssUrl) {
};
if (getSettingValue("line-numbers") === "true") {
- // @ts-expect-error
window.rustdoc_add_line_numbers_to_examples();
}
@@ -1596,7 +1593,7 @@ function preLoadCss(cssUrl) {
/**
* Hide popover menus, clickable tooltips, and the sidebar (if applicable).
*
- * Pass "true" to reset focus for tooltip popovers.
+ * Pass `true` to reset focus for tooltip popovers.
*/
window.hideAllModals = switchFocus => {
hideSidebar();
diff --git a/src/librustdoc/html/static/js/rustdoc.d.ts b/src/librustdoc/html/static/js/rustdoc.d.ts
index 91a58fab86eff..0d2e19e019f34 100644
--- a/src/librustdoc/html/static/js/rustdoc.d.ts
+++ b/src/librustdoc/html/static/js/rustdoc.d.ts
@@ -30,6 +30,8 @@ declare global {
currentCrate: string|null;
/**
* Hide popovers, tooltips, or the mobile sidebar.
+ *
+ * Pass `true` to reset focus for tooltip popovers.
*/
hideAllModals: function(boolean),
/**
@@ -78,6 +80,8 @@ declare global {
pending_implementors?: rustdoc.Implementors,
register_type_impls?: function(rustdoc.TypeImpls): void,
pending_type_impls?: rustdoc.TypeImpls,
+ rustdoc_add_line_numbers_to_examples?: function(),
+ rustdoc_remove_line_numbers_from_examples?: function(),
}
interface HTMLElement {
/** Used by the popover tooltip code. */
@@ -477,4 +481,14 @@ declare namespace rustdoc {
* is a tuple of (filename, subdirs, filenames).
*/
type Dir = [string, rustdoc.Dir[], string[]]
+
+ /**
+ * Indivitual setting object, used in `settings.js`
+ */
+ interface Setting {
+ js_name: string,
+ name: string,
+ options?: string[],
+ default: string | boolean,
+ }
}
diff --git a/src/librustdoc/html/static/js/settings.js b/src/librustdoc/html/static/js/settings.js
index 5f1bbd27328cb..2430b5829b2ba 100644
--- a/src/librustdoc/html/static/js/settings.js
+++ b/src/librustdoc/html/static/js/settings.js
@@ -1,22 +1,39 @@
// Local js definitions:
/* global getSettingValue, updateLocalStorage, updateTheme */
/* global addClass, removeClass, onEach, onEachLazy */
-/* global MAIN_ID, getVar, getSettingsButton, getHelpButton */
-
-// Eventually fix this.
-// @ts-nocheck
+/* global MAIN_ID, getVar, getSettingsButton, getHelpButton, nonnull */
"use strict";
(function() {
const isSettingsPage = window.location.pathname.endsWith("/settings.html");
+ /**
+ * @param {Element} elem
+ * @param {EventTarget|null} target
+ */
+ function elemContainsTarget(elem, target) {
+ if (target instanceof Node) {
+ return elem.contains(target);
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * @overload {"theme"|"preferred-dark-theme"|"preferred-light-theme"}
+ * @param {string} settingName
+ * @param {string} value
+ * @returns
+ * @param {string} settingName
+ * @param {string|boolean} value
+ */
function changeSetting(settingName, value) {
if (settingName === "theme") {
const useSystem = value === "system preference" ? "true" : "false";
updateLocalStorage("use-system-theme", useSystem);
}
- updateLocalStorage(settingName, value);
+ updateLocalStorage(settingName, "" + value);
switch (settingName) {
case "theme":
@@ -27,9 +44,15 @@
break;
case "line-numbers":
if (value === true) {
- window.rustdoc_add_line_numbers_to_examples();
+ const f = window.rustdoc_add_line_numbers_to_examples;
+ if (f !== undefined) {
+ f();
+ }
} else {
- window.rustdoc_remove_line_numbers_from_examples();
+ const f = window.rustdoc_remove_line_numbers_from_examples;
+ if (f !== undefined) {
+ f();
+ }
}
break;
case "hide-sidebar":
@@ -89,6 +112,9 @@
}
}
+ /**
+ * @param {HTMLElement} settingsElement
+ */
function setEvents(settingsElement) {
updateLightAndDark();
onEachLazy(settingsElement.querySelectorAll("input[type=\"checkbox\"]"), toggle => {
@@ -101,23 +127,27 @@
changeSetting(toggle.id, toggle.checked);
};
});
- onEachLazy(settingsElement.querySelectorAll("input[type=\"radio\"]"), elem => {
- const settingId = elem.name;
- let settingValue = getSettingValue(settingId);
- if (settingId === "theme") {
- const useSystem = getSettingValue("use-system-theme");
- if (useSystem === "true" || settingValue === null) {
- // "light" is the default theme
- settingValue = useSystem === "false" ? "light" : "system preference";
+ onEachLazy(
+ settingsElement.querySelectorAll("input[type=\"radio\"]"),
+ /** @param {HTMLInputElement} elem */
+ elem => {
+ const settingId = elem.name;
+ let settingValue = getSettingValue(settingId);
+ if (settingId === "theme") {
+ const useSystem = getSettingValue("use-system-theme");
+ if (useSystem === "true" || settingValue === null) {
+ // "light" is the default theme
+ settingValue = useSystem === "false" ? "light" : "system preference";
+ }
}
- }
- if (settingValue !== null && settingValue !== "null") {
- elem.checked = settingValue === elem.value;
- }
- elem.addEventListener("change", ev => {
- changeSetting(ev.target.name, ev.target.value);
- });
- });
+ if (settingValue !== null && settingValue !== "null") {
+ elem.checked = settingValue === elem.value;
+ }
+ elem.addEventListener("change", () => {
+ changeSetting(elem.name, elem.value);
+ });
+ },
+ );
}
/**
@@ -125,7 +155,7 @@
* as argument which describes each setting and how to render it. It returns a string
* representing the raw HTML.
*
- * @param {Array<Object>} settings
+ * @param {Array<rustdoc.Setting>} settings
*
* @return {string}
*/
@@ -133,11 +163,6 @@
let output = "";
for (const setting of settings) {
- if (setting === "hr") {
- output += "<hr>";
- continue;
- }
-
const js_data_name = setting["js_name"];
const setting_name = setting["name"];
@@ -182,7 +207,9 @@
* @return {HTMLElement}
*/
function buildSettingsPage() {
- const theme_names = getVar("themes").split(",").filter(t => t);
+ const theme_list = getVar("themes");
+ const theme_names = (theme_list === null ? "" : theme_list)
+ .split(",").filter(t => t);
theme_names.push("light", "dark", "ayu");
const settings = [
@@ -272,10 +299,16 @@
el.innerHTML = innerHTML;
if (isSettingsPage) {
- document.getElementById(MAIN_ID).appendChild(el);
+ const mainElem = document.getElementById(MAIN_ID);
+ if (mainElem !== null) {
+ mainElem.appendChild(el);
+ }
} else {
el.setAttribute("tabindex", "-1");
- getSettingsButton().appendChild(el);
+ const settingsBtn = getSettingsButton();
+ if (settingsBtn !== null) {
+ settingsBtn.appendChild(el);
+ }
}
return el;
}
@@ -293,34 +326,44 @@
});
}
+ /**
+ * @param {FocusEvent} event
+ */
function settingsBlurHandler(event) {
- if (!getHelpButton().contains(document.activeElement) &&
- !getHelpButton().contains(event.relatedTarget) &&
- !getSettingsButton().contains(document.activeElement) &&
- !getSettingsButton().contains(event.relatedTarget)
- ) {
+ const helpBtn = getHelpButton();
+ const settingsBtn = getSettingsButton();
+ const helpUnfocused = helpBtn === null ||
+ (!helpBtn.contains(document.activeElement) &&
+ !elemContainsTarget(helpBtn, event.relatedTarget));
+ const settingsUnfocused = settingsBtn === null ||
+ (!settingsBtn.contains(document.activeElement) &&
+ !elemContainsTarget(settingsBtn, event.relatedTarget));
+ if (helpUnfocused && settingsUnfocused) {
window.hidePopoverMenus();
}
}
if (!isSettingsPage) {
// We replace the existing "onclick" callback.
- const settingsButton = getSettingsButton();
- const settingsMenu = document.getElementById("settings");
+ // These elements must exist, as (outside of the settings page)
+ // `settings.js` is only loaded after the settings button is clicked.
+ const settingsButton = nonnull(getSettingsButton());
+ const settingsMenu = nonnull(document.getElementById("settings"));
settingsButton.onclick = event => {
- if (settingsMenu.contains(event.target)) {
+ if (elemContainsTarget(settingsMenu, event.target)) {
return;
}
event.preventDefault();
const shouldDisplaySettings = settingsMenu.style.display === "none";
- window.hideAllModals();
+ window.hideAllModals(false);
if (shouldDisplaySettings) {
displaySettings();
}
};
settingsButton.onblur = settingsBlurHandler;
- settingsButton.querySelector("a").onblur = settingsBlurHandler;
+ // the settings button should always have a link in it
+ nonnull(settingsButton.querySelector("a")).onblur = settingsBlurHandler;
onEachLazy(settingsMenu.querySelectorAll("input"), el => {
el.onblur = settingsBlurHandler;
});
diff --git a/src/tools/miri/src/intrinsics/mod.rs b/src/tools/miri/src/intrinsics/mod.rs
index 85fb280a9a908..a3525dcc77ae6 100644
--- a/src/tools/miri/src/intrinsics/mod.rs
+++ b/src/tools/miri/src/intrinsics/mod.rs
@@ -411,9 +411,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
};
let res = this.binary_op(op, &a, &b)?;
// `binary_op` already called `generate_nan` if needed.
- // Apply a relative error of 16ULP to simulate non-deterministic precision loss
+ // Apply a relative error of 4ULP to simulate non-deterministic precision loss
// due to optimizations.
- let res = apply_random_float_error_to_imm(this, res, 4 /* log2(16) */)?;
+ let res = apply_random_float_error_to_imm(this, res, 2 /* log2(4) */)?;
this.write_immediate(*res, dest)?;
}
@@ -464,9 +464,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
if !float_finite(&res)? {
throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
}
- // Apply a relative error of 16ULP to simulate non-deterministic precision loss
+ // Apply a relative error of 4ULP to simulate non-deterministic precision loss
// due to optimizations.
- let res = apply_random_float_error_to_imm(this, res, 4 /* log2(16) */)?;
+ let res = apply_random_float_error_to_imm(this, res, 2 /* log2(4) */)?;
this.write_immediate(*res, dest)?;
}
diff --git a/src/tools/unicode-table-generator/src/range_search.rs b/src/tools/unicode-table-generator/src/range_search.rs
index 9a51979a2f0d9..02f9cf16d4d37 100644
--- a/src/tools/unicode-table-generator/src/range_search.rs
+++ b/src/tools/unicode-table-generator/src/range_search.rs
@@ -45,45 +45,78 @@ const fn bitset_search<
(word & (1 << (needle % 64) as u64)) != 0
}
-fn decode_prefix_sum(short_offset_run_header: u32) -> u32 {
- short_offset_run_header & ((1 << 21) - 1)
-}
+#[repr(transparent)]
+struct ShortOffsetRunHeader(u32);
+
+impl ShortOffsetRunHeader {
+ const fn new(start_index: usize, prefix_sum: u32) -> Self {
+ assert!(start_index < (1 << 11));
+ assert!(prefix_sum < (1 << 21));
-fn decode_length(short_offset_run_header: u32) -> usize {
- (short_offset_run_header >> 21) as usize
+ Self((start_index as u32) << 21 | prefix_sum)
+ }
+
+ #[inline]
+ const fn start_index(&self) -> usize {
+ (self.0 >> 21) as usize
+ }
+
+ #[inline]
+ const fn prefix_sum(&self) -> u32 {
+ self.0 & ((1 << 21) - 1)
+ }
}
+/// # Safety
+///
+/// - The last element of `short_offset_runs` must be greater than `std::char::MAX`.
+/// - The start indices of all elements in `short_offset_runs` must be less than `OFFSETS`.
#[inline(always)]
-fn skip_search<const SOR: usize, const OFFSETS: usize>(
- needle: u32,
- short_offset_runs: &[u32; SOR],
+unsafe fn skip_search<const SOR: usize, const OFFSETS: usize>(
+ needle: char,
+ short_offset_runs: &[ShortOffsetRunHeader; SOR],
offsets: &[u8; OFFSETS],
) -> bool {
- // Note that this *cannot* be past the end of the array, as the last
- // element is greater than std::char::MAX (the largest possible needle).
- //
- // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct
- // location cannot be past it, so Err(idx) != length either.
- //
- // This means that we can avoid bounds checking for the accesses below, too.
+ let needle = needle as u32;
+
let last_idx =
- match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) {
+ match short_offset_runs.binary_search_by_key(&(needle << 11), |header| (header.0 << 11)) {
Ok(idx) => idx + 1,
Err(idx) => idx,
};
+ // SAFETY: `last_idx` *cannot* be past the end of the array, as the last
+ // element is greater than `std::char::MAX` (the largest possible needle)
+ // as guaranteed by the caller.
+ //
+ // So, we cannot have found it (i.e. `Ok(idx) => idx + 1 != length`) and the
+ // correct location cannot be past it, so `Err(idx) => idx != length` either.
+ //
+ // This means that we can avoid bounds checking for the accesses below, too.
+ //
+ // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+ // in `hint::assert_unchecked` may not be optimized out.
+ unsafe { crate::intrinsics::assume(last_idx < SOR) };
- let mut offset_idx = decode_length(short_offset_runs[last_idx]);
+ let mut offset_idx = short_offset_runs[last_idx].start_index();
let length = if let Some(next) = short_offset_runs.get(last_idx + 1) {
- decode_length(*next) - offset_idx
+ (*next).start_index() - offset_idx
} else {
offsets.len() - offset_idx
};
+
let prev =
- last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0);
+ last_idx.checked_sub(1).map(|prev| short_offset_runs[prev].prefix_sum()).unwrap_or(0);
let total = needle - prev;
let mut prefix_sum = 0;
for _ in 0..(length - 1) {
+ // SAFETY: It is guaranteed that `length <= OFFSETS - offset_idx`,
+ // so it follows that `length - 1 + offset_idx < OFFSETS`, therefore
+ // `offset_idx < OFFSETS` is always true in this loop.
+ //
+ // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+ // in `hint::assert_unchecked` may not be optimized out.
+ unsafe { crate::intrinsics::assume(offset_idx < OFFSETS) };
let offset = offsets[offset_idx];
prefix_sum += offset as u32;
if prefix_sum > total {
diff --git a/src/tools/unicode-table-generator/src/skiplist.rs b/src/tools/unicode-table-generator/src/skiplist.rs
index 8ca18ddc91a80..34c9802e122f6 100644
--- a/src/tools/unicode-table-generator/src/skiplist.rs
+++ b/src/tools/unicode-table-generator/src/skiplist.rs
@@ -1,26 +1,23 @@
-use std::fmt::Write as _;
+use std::fmt::{self, Write as _};
use std::ops::Range;
use crate::fmt_list;
use crate::raw_emitter::RawEmitter;
/// This will get packed into a single u32 before inserting into the data set.
-#[derive(Debug, PartialEq)]
+#[derive(PartialEq)]
struct ShortOffsetRunHeader {
- /// Note, we only allow for 21 bits here.
- prefix_sum: u32,
-
/// Note, we actually only allow for 11 bits here. This should be enough --
/// our largest sets are around ~1400 offsets long.
- start_idx: u16,
-}
+ start_index: u16,
-impl ShortOffsetRunHeader {
- fn pack(&self) -> u32 {
- assert!(self.start_idx < (1 << 11));
- assert!(self.prefix_sum < (1 << 21));
+ /// Note, we only allow for 21 bits here.
+ prefix_sum: u32,
+}
- (self.start_idx as u32) << 21 | self.prefix_sum
+impl fmt::Debug for ShortOffsetRunHeader {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "ShortOffsetRunHeader::new({}, {})", self.start_index, self.prefix_sum)
}
}
@@ -53,7 +50,7 @@ impl RawEmitter {
coded_offsets.push(offset);
} else {
short_offset_runs.push(ShortOffsetRunHeader {
- start_idx: start.try_into().unwrap(),
+ start_index: start.try_into().unwrap(),
prefix_sum,
});
// This is just needed to maintain indices even/odd
@@ -71,11 +68,12 @@ impl RawEmitter {
assert!(inserted);
}
+ writeln!(&mut self.file, "use super::ShortOffsetRunHeader;\n").unwrap();
writeln!(
&mut self.file,
- "static SHORT_OFFSET_RUNS: [u32; {}] = [{}];",
+ "static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; {}] = [{}];",
short_offset_runs.len(),
- fmt_list(short_offset_runs.iter().map(|v| v.pack()))
+ fmt_list(short_offset_runs.iter())
)
.unwrap();
self.bytes_used += 4 * short_offset_runs.len();
@@ -104,15 +102,43 @@ impl RawEmitter {
writeln!(&mut self.file, " (c as u32) >= {first_code_point:#04x} && lookup_slow(c)")
.unwrap();
writeln!(&mut self.file, "}}").unwrap();
+ writeln!(&mut self.file).unwrap();
+ writeln!(&mut self.file, "#[inline(never)]").unwrap();
writeln!(&mut self.file, "fn lookup_slow(c: char) -> bool {{").unwrap();
} else {
writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap();
}
- writeln!(&mut self.file, " super::skip_search(",).unwrap();
- writeln!(&mut self.file, " c as u32,").unwrap();
- writeln!(&mut self.file, " &SHORT_OFFSET_RUNS,").unwrap();
- writeln!(&mut self.file, " &OFFSETS,").unwrap();
- writeln!(&mut self.file, " )").unwrap();
+ writeln!(&mut self.file, " const {{").unwrap();
+ writeln!(
+ &mut self.file,
+ " assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);",
+ )
+ .unwrap();
+ writeln!(&mut self.file, " let mut i = 0;").unwrap();
+ writeln!(&mut self.file, " while i < SHORT_OFFSET_RUNS.len() {{").unwrap();
+ writeln!(
+ &mut self.file,
+ " assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());",
+ )
+ .unwrap();
+ writeln!(&mut self.file, " i += 1;").unwrap();
+ writeln!(&mut self.file, " }}").unwrap();
+ writeln!(&mut self.file, " }}").unwrap();
+ writeln!(
+ &mut self.file,
+ " // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`",
+ )
+ .unwrap();
+ writeln!(
+ &mut self.file,
+ " // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.",
+ )
+ .unwrap();
+ writeln!(
+ &mut self.file,
+ " unsafe {{ super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }}"
+ )
+ .unwrap();
writeln!(&mut self.file, "}}").unwrap();
}
}
diff --git a/tests/codegen/autodiff.rs b/tests/codegen/autodiff.rs
index cace0edb2b544..85358f5fcb693 100644
--- a/tests/codegen/autodiff.rs
+++ b/tests/codegen/autodiff.rs
@@ -11,7 +11,7 @@ fn square(x: &f64) -> f64 {
x * x
}
-// CHECK:define internal fastcc double @diffesquare(double %x.0.val, ptr nocapture align 8 %"x'"
+// CHECK:define internal fastcc double @diffesquare(double %x.0.val, ptr nocapture nonnull align 8 %"x'"
// CHECK-NEXT:invertstart:
// CHECK-NEXT: %_0 = fmul double %x.0.val, %x.0.val
// CHECK-NEXT: %0 = fadd fast double %x.0.val, %x.0.val
@@ -22,7 +22,7 @@ fn square(x: &f64) -> f64 {
// CHECK-NEXT:}
fn main() {
- let x = 3.0;
+ let x = std::hint::black_box(3.0);
let output = square(&x);
assert_eq!(9.0, output);
diff --git a/tests/codegen/autodiffv.rs b/tests/codegen/autodiffv.rs
new file mode 100644
index 0000000000000..e004711640553
--- /dev/null
+++ b/tests/codegen/autodiffv.rs
@@ -0,0 +1,116 @@
+//@ compile-flags: -Zautodiff=Enable -C opt-level=3 -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+//
+// In Enzyme, we test against a large range of LLVM versions (5+) and don't have overly many
+// breakages. One benefit is that we match the IR generated by Enzyme only after running it
+// through LLVM's O3 pipeline, which will remove most of the noise.
+// However, our integration test could also be affected by changes in how rustc lowers MIR into
+// LLVM-IR, which could cause additional noise and thus breakages. If that's the case, we should
+// reduce this test to only match the first lines and the ret instructions.
+
+#![feature(autodiff)]
+
+use std::autodiff::autodiff;
+
+#[autodiff(d_square3, Forward, Dual, DualOnly)]
+#[autodiff(d_square2, Forward, 4, Dual, DualOnly)]
+#[autodiff(d_square1, Forward, 4, Dual, Dual)]
+#[no_mangle]
+fn square(x: &f32) -> f32 {
+ x * x
+}
+
+// d_sqaure2
+// CHECK: define internal fastcc [4 x float] @fwddiffe4square(float %x.0.val, [4 x ptr] %"x'")
+// CHECK-NEXT: start:
+// CHECK-NEXT: %0 = extractvalue [4 x ptr] %"x'", 0
+// CHECK-NEXT: %"_2'ipl" = load float, ptr %0, align 4
+// CHECK-NEXT: %1 = extractvalue [4 x ptr] %"x'", 1
+// CHECK-NEXT: %"_2'ipl1" = load float, ptr %1, align 4
+// CHECK-NEXT: %2 = extractvalue [4 x ptr] %"x'", 2
+// CHECK-NEXT: %"_2'ipl2" = load float, ptr %2, align 4
+// CHECK-NEXT: %3 = extractvalue [4 x ptr] %"x'", 3
+// CHECK-NEXT: %"_2'ipl3" = load float, ptr %3, align 4
+// CHECK-NEXT: %4 = insertelement <4 x float> poison, float %"_2'ipl", i64 0
+// CHECK-NEXT: %5 = insertelement <4 x float> %4, float %"_2'ipl1", i64 1
+// CHECK-NEXT: %6 = insertelement <4 x float> %5, float %"_2'ipl2", i64 2
+// CHECK-NEXT: %7 = insertelement <4 x float> %6, float %"_2'ipl3", i64 3
+// CHECK-NEXT: %8 = fadd fast <4 x float> %7, %7
+// CHECK-NEXT: %9 = insertelement <4 x float> poison, float %x.0.val, i64 0
+// CHECK-NEXT: %10 = shufflevector <4 x float> %9, <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT: %11 = fmul fast <4 x float> %8, %10
+// CHECK-NEXT: %12 = extractelement <4 x float> %11, i64 0
+// CHECK-NEXT: %13 = insertvalue [4 x float] undef, float %12, 0
+// CHECK-NEXT: %14 = extractelement <4 x float> %11, i64 1
+// CHECK-NEXT: %15 = insertvalue [4 x float] %13, float %14, 1
+// CHECK-NEXT: %16 = extractelement <4 x float> %11, i64 2
+// CHECK-NEXT: %17 = insertvalue [4 x float] %15, float %16, 2
+// CHECK-NEXT: %18 = extractelement <4 x float> %11, i64 3
+// CHECK-NEXT: %19 = insertvalue [4 x float] %17, float %18, 3
+// CHECK-NEXT: ret [4 x float] %19
+// CHECK-NEXT: }
+
+// d_square3, the extra float is the original return value (x * x)
+// CHECK: define internal fastcc { float, [4 x float] } @fwddiffe4square.1(float %x.0.val, [4 x ptr] %"x'")
+// CHECK-NEXT: start:
+// CHECK-NEXT: %0 = extractvalue [4 x ptr] %"x'", 0
+// CHECK-NEXT: %"_2'ipl" = load float, ptr %0, align 4
+// CHECK-NEXT: %1 = extractvalue [4 x ptr] %"x'", 1
+// CHECK-NEXT: %"_2'ipl1" = load float, ptr %1, align 4
+// CHECK-NEXT: %2 = extractvalue [4 x ptr] %"x'", 2
+// CHECK-NEXT: %"_2'ipl2" = load float, ptr %2, align 4
+// CHECK-NEXT: %3 = extractvalue [4 x ptr] %"x'", 3
+// CHECK-NEXT: %"_2'ipl3" = load float, ptr %3, align 4
+// CHECK-NEXT: %_0 = fmul float %x.0.val, %x.0.val
+// CHECK-NEXT: %4 = insertelement <4 x float> poison, float %"_2'ipl", i64 0
+// CHECK-NEXT: %5 = insertelement <4 x float> %4, float %"_2'ipl1", i64 1
+// CHECK-NEXT: %6 = insertelement <4 x float> %5, float %"_2'ipl2", i64 2
+// CHECK-NEXT: %7 = insertelement <4 x float> %6, float %"_2'ipl3", i64 3
+// CHECK-NEXT: %8 = fadd fast <4 x float> %7, %7
+// CHECK-NEXT: %9 = insertelement <4 x float> poison, float %x.0.val, i64 0
+// CHECK-NEXT: %10 = shufflevector <4 x float> %9, <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT: %11 = fmul fast <4 x float> %8, %10
+// CHECK-NEXT: %12 = extractelement <4 x float> %11, i64 0
+// CHECK-NEXT: %13 = insertvalue [4 x float] undef, float %12, 0
+// CHECK-NEXT: %14 = extractelement <4 x float> %11, i64 1
+// CHECK-NEXT: %15 = insertvalue [4 x float] %13, float %14, 1
+// CHECK-NEXT: %16 = extractelement <4 x float> %11, i64 2
+// CHECK-NEXT: %17 = insertvalue [4 x float] %15, float %16, 2
+// CHECK-NEXT: %18 = extractelement <4 x float> %11, i64 3
+// CHECK-NEXT: %19 = insertvalue [4 x float] %17, float %18, 3
+// CHECK-NEXT: %20 = insertvalue { float, [4 x float] } undef, float %_0, 0
+// CHECK-NEXT: %21 = insertvalue { float, [4 x float] } %20, [4 x float] %19, 1
+// CHECK-NEXT: ret { float, [4 x float] } %21
+// CHECK-NEXT: }
+
+fn main() {
+ let x = std::hint::black_box(3.0);
+ let output = square(&x);
+ dbg!(&output);
+ assert_eq!(9.0, output);
+ dbg!(square(&x));
+
+ let mut df_dx1 = 1.0;
+ let mut df_dx2 = 2.0;
+ let mut df_dx3 = 3.0;
+ let mut df_dx4 = 0.0;
+ let [o1, o2, o3, o4] = d_square2(&x, &mut df_dx1, &mut df_dx2, &mut df_dx3, &mut df_dx4);
+ dbg!(o1, o2, o3, o4);
+ let [output2, o1, o2, o3, o4] =
+ d_square1(&x, &mut df_dx1, &mut df_dx2, &mut df_dx3, &mut df_dx4);
+ dbg!(o1, o2, o3, o4);
+ assert_eq!(output, output2);
+ assert!((6.0 - o1).abs() < 1e-10);
+ assert!((12.0 - o2).abs() < 1e-10);
+ assert!((18.0 - o3).abs() < 1e-10);
+ assert!((0.0 - o4).abs() < 1e-10);
+ assert_eq!(1.0, df_dx1);
+ assert_eq!(2.0, df_dx2);
+ assert_eq!(3.0, df_dx3);
+ assert_eq!(0.0, df_dx4);
+ assert_eq!(d_square3(&x, &mut df_dx1), 2.0 * o1);
+ assert_eq!(d_square3(&x, &mut df_dx2), 2.0 * o2);
+ assert_eq!(d_square3(&x, &mut df_dx3), 2.0 * o3);
+ assert_eq!(d_square3(&x, &mut df_dx4), 2.0 * o4);
+}
diff --git a/tests/codegen/char-escape-debug-no-bounds-check.rs b/tests/codegen/char-escape-debug-no-bounds-check.rs
new file mode 100644
index 0000000000000..cfde46045e5a5
--- /dev/null
+++ b/tests/codegen/char-escape-debug-no-bounds-check.rs
@@ -0,0 +1,14 @@
+//@ compile-flags: -Copt-level=3
+#![crate_type = "lib"]
+
+use std::char::EscapeDebug;
+
+// Make sure no bounds checks are emitted when escaping a character.
+
+// CHECK-LABEL: @char_escape_debug_no_bounds_check
+#[no_mangle]
+pub fn char_escape_debug_no_bounds_check(c: char) -> EscapeDebug {
+ // CHECK-NOT: panic
+ // CHECK-NOT: panic_bounds_check
+ c.escape_debug()
+}
diff --git a/tests/codegen/float/algebraic.rs b/tests/codegen/float/algebraic.rs
new file mode 100644
index 0000000000000..818a4bcdfe3f1
--- /dev/null
+++ b/tests/codegen/float/algebraic.rs
@@ -0,0 +1,149 @@
+// Verify that algebraic intrinsics generate the correct LLVM calls
+
+// Ensure operations get inlined
+//@ compile-flags: -Copt-level=1
+
+#![crate_type = "lib"]
+#![feature(f16)]
+#![feature(f128)]
+#![feature(float_algebraic)]
+
+// CHECK-LABEL: @f16_algebraic_add
+#[no_mangle]
+pub fn f16_algebraic_add(a: f16, b: f16) -> f16 {
+ // CHECK: fadd reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+ a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_sub
+#[no_mangle]
+pub fn f16_algebraic_sub(a: f16, b: f16) -> f16 {
+ // CHECK: fsub reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+ a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_mul
+#[no_mangle]
+pub fn f16_algebraic_mul(a: f16, b: f16) -> f16 {
+ // CHECK: fmul reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+ a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_div
+#[no_mangle]
+pub fn f16_algebraic_div(a: f16, b: f16) -> f16 {
+ // CHECK: fdiv reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+ a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_rem
+#[no_mangle]
+pub fn f16_algebraic_rem(a: f16, b: f16) -> f16 {
+ // CHECK: frem reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+ a.algebraic_rem(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_add
+#[no_mangle]
+pub fn f32_algebraic_add(a: f32, b: f32) -> f32 {
+ // CHECK: fadd reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+ a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_sub
+#[no_mangle]
+pub fn f32_algebraic_sub(a: f32, b: f32) -> f32 {
+ // CHECK: fsub reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+ a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_mul
+#[no_mangle]
+pub fn f32_algebraic_mul(a: f32, b: f32) -> f32 {
+ // CHECK: fmul reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+ a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_div
+#[no_mangle]
+pub fn f32_algebraic_div(a: f32, b: f32) -> f32 {
+ // CHECK: fdiv reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+ a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_rem
+#[no_mangle]
+pub fn f32_algebraic_rem(a: f32, b: f32) -> f32 {
+ // CHECK: frem reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+ a.algebraic_rem(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_add
+#[no_mangle]
+pub fn f64_algebraic_add(a: f64, b: f64) -> f64 {
+ // CHECK: fadd reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+ a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_sub
+#[no_mangle]
+pub fn f64_algebraic_sub(a: f64, b: f64) -> f64 {
+ // CHECK: fsub reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+ a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_mul
+#[no_mangle]
+pub fn f64_algebraic_mul(a: f64, b: f64) -> f64 {
+ // CHECK: fmul reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+ a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_div
+#[no_mangle]
+pub fn f64_algebraic_div(a: f64, b: f64) -> f64 {
+ // CHECK: fdiv reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+ a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_rem
+#[no_mangle]
+pub fn f64_algebraic_rem(a: f64, b: f64) -> f64 {
+ // CHECK: frem reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+ a.algebraic_rem(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_add
+#[no_mangle]
+pub fn f128_algebraic_add(a: f128, b: f128) -> f128 {
+ // CHECK: fadd reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+ a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_sub
+#[no_mangle]
+pub fn f128_algebraic_sub(a: f128, b: f128) -> f128 {
+ // CHECK: fsub reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+ a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_mul
+#[no_mangle]
+pub fn f128_algebraic_mul(a: f128, b: f128) -> f128 {
+ // CHECK: fmul reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+ a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_div
+#[no_mangle]
+pub fn f128_algebraic_div(a: f128, b: f128) -> f128 {
+ // CHECK: fdiv reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+ a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_rem
+#[no_mangle]
+pub fn f128_algebraic_rem(a: f128, b: f128) -> f128 {
+ // CHECK: frem reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+ a.algebraic_rem(b)
+}
diff --git a/tests/codegen/float_math.rs b/tests/codegen/float_math.rs
index 31387ec82b920..9a1e0b4d2d062 100644
--- a/tests/codegen/float_math.rs
+++ b/tests/codegen/float_math.rs
@@ -3,7 +3,10 @@
#![crate_type = "lib"]
#![feature(core_intrinsics)]
-use std::intrinsics::{fadd_fast, fdiv_fast, fmul_fast, frem_fast, fsub_fast};
+use std::intrinsics::{
+ fadd_algebraic, fadd_fast, fdiv_algebraic, fdiv_fast, fmul_algebraic, fmul_fast,
+ frem_algebraic, frem_fast, fsub_algebraic, fsub_fast,
+};
// CHECK-LABEL: @add
#[no_mangle]
@@ -13,30 +16,72 @@ pub fn add(x: f32, y: f32) -> f32 {
x + y
}
-// CHECK-LABEL: @addition
+// CHECK-LABEL: @test_fadd_algebraic
#[no_mangle]
-pub fn addition(x: f32, y: f32) -> f32 {
- // CHECK: fadd fast float
+pub fn test_fadd_algebraic(x: f32, y: f32) -> f32 {
+ // CHECK: fadd reassoc nsz arcp contract float %x, %y
+ fadd_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fsub_algebraic
+#[no_mangle]
+pub fn test_fsub_algebraic(x: f32, y: f32) -> f32 {
+ // CHECK: fsub reassoc nsz arcp contract float %x, %y
+ fsub_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fmul_algebraic
+#[no_mangle]
+pub fn test_fmul_algebraic(x: f32, y: f32) -> f32 {
+ // CHECK: fmul reassoc nsz arcp contract float %x, %y
+ fmul_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fdiv_algebraic
+#[no_mangle]
+pub fn test_fdiv_algebraic(x: f32, y: f32) -> f32 {
+ // CHECK: fdiv reassoc nsz arcp contract float %x, %y
+ fdiv_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_frem_algebraic
+#[no_mangle]
+pub fn test_frem_algebraic(x: f32, y: f32) -> f32 {
+ // CHECK: frem reassoc nsz arcp contract float %x, %y
+ frem_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fadd_fast
+#[no_mangle]
+pub fn test_fadd_fast(x: f32, y: f32) -> f32 {
+ // CHECK: fadd fast float %x, %y
unsafe { fadd_fast(x, y) }
}
-// CHECK-LABEL: @subtraction
+// CHECK-LABEL: @test_fsub_fast
#[no_mangle]
-pub fn subtraction(x: f32, y: f32) -> f32 {
- // CHECK: fsub fast float
+pub fn test_fsub_fast(x: f32, y: f32) -> f32 {
+ // CHECK: fsub fast float %x, %y
unsafe { fsub_fast(x, y) }
}
-// CHECK-LABEL: @multiplication
+// CHECK-LABEL: @test_fmul_fast
#[no_mangle]
-pub fn multiplication(x: f32, y: f32) -> f32 {
- // CHECK: fmul fast float
+pub fn test_fmul_fast(x: f32, y: f32) -> f32 {
+ // CHECK: fmul fast float %x, %y
unsafe { fmul_fast(x, y) }
}
-// CHECK-LABEL: @division
+// CHECK-LABEL: @test_fdiv_fast
#[no_mangle]
-pub fn division(x: f32, y: f32) -> f32 {
- // CHECK: fdiv fast float
+pub fn test_fdiv_fast(x: f32, y: f32) -> f32 {
+ // CHECK: fdiv fast float %x, %y
unsafe { fdiv_fast(x, y) }
}
+
+// CHECK-LABEL: @test_frem_fast
+#[no_mangle]
+pub fn test_frem_fast(x: f32, y: f32) -> f32 {
+ // CHECK: frem fast float %x, %y
+ unsafe { frem_fast(x, y) }
+}
diff --git a/tests/crashes/137874.rs b/tests/crashes/137874.rs
deleted file mode 100644
index 4471880902454..0000000000000
--- a/tests/crashes/137874.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-//@ known-bug: #137874
-fn a() {
- match b { deref !(0c) };
-}
diff --git a/tests/pretty/autodiff_forward.pp b/tests/pretty/autodiff_forward.pp
index dc7a2712f4231..4b2fb6166ff7e 100644
--- a/tests/pretty/autodiff_forward.pp
+++ b/tests/pretty/autodiff_forward.pp
@@ -25,27 +25,31 @@
// We want to be sure that the same function can be differentiated in different ways
+
+ // Make sure, that we add the None for the default return.
+
+
::core::panicking::panic("not implemented")
}
-#[rustc_autodiff(Forward, Dual, Const, Dual,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Dual)]
#[inline(never)]
-pub fn df1(x: &[f64], bx: &[f64], y: f64) -> (f64, f64) {
+pub fn df1(x: &[f64], bx_0: &[f64], y: f64) -> (f64, f64) {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f1(x, y));
- ::core::hint::black_box((bx,));
- ::core::hint::black_box((f1(x, y), f64::default()))
+ ::core::hint::black_box((bx_0,));
+ ::core::hint::black_box(<(f64, f64)>::default())
}
#[rustc_autodiff]
#[inline(never)]
pub fn f2(x: &[f64], y: f64) -> f64 {
::core::panicking::panic("not implemented")
}
-#[rustc_autodiff(Forward, Dual, Const, Const,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Const)]
#[inline(never)]
-pub fn df2(x: &[f64], bx: &[f64], y: f64) -> f64 {
+pub fn df2(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f2(x, y));
- ::core::hint::black_box((bx,));
+ ::core::hint::black_box((bx_0,));
::core::hint::black_box(f2(x, y))
}
#[rustc_autodiff]
@@ -53,20 +57,20 @@
pub fn f3(x: &[f64], y: f64) -> f64 {
::core::panicking::panic("not implemented")
}
-#[rustc_autodiff(Forward, Dual, Const, Const,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Const)]
#[inline(never)]
-pub fn df3(x: &[f64], bx: &[f64], y: f64) -> f64 {
+pub fn df3(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f3(x, y));
- ::core::hint::black_box((bx,));
+ ::core::hint::black_box((bx_0,));
::core::hint::black_box(f3(x, y))
}
#[rustc_autodiff]
#[inline(never)]
pub fn f4() {}
-#[rustc_autodiff(Forward, None)]
+#[rustc_autodiff(Forward, 1, None)]
#[inline(never)]
-pub fn df4() {
+pub fn df4() -> () {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f4());
::core::hint::black_box(());
@@ -76,28 +80,82 @@
pub fn f5(x: &[f64], y: f64) -> f64 {
::core::panicking::panic("not implemented")
}
-#[rustc_autodiff(Forward, Const, Dual, Const,)]
+#[rustc_autodiff(Forward, 1, Const, Dual, Const)]
#[inline(never)]
-pub fn df5_y(x: &[f64], y: f64, by: f64) -> f64 {
+pub fn df5_y(x: &[f64], y: f64, by_0: f64) -> f64 {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f5(x, y));
- ::core::hint::black_box((by,));
+ ::core::hint::black_box((by_0,));
::core::hint::black_box(f5(x, y))
}
-#[rustc_autodiff(Forward, Dual, Const, Const,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Const)]
#[inline(never)]
-pub fn df5_x(x: &[f64], bx: &[f64], y: f64) -> f64 {
+pub fn df5_x(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f5(x, y));
- ::core::hint::black_box((bx,));
+ ::core::hint::black_box((bx_0,));
::core::hint::black_box(f5(x, y))
}
-#[rustc_autodiff(Reverse, Duplicated, Const, Active,)]
+#[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
#[inline(never)]
-pub fn df5_rev(x: &[f64], dx: &mut [f64], y: f64, dret: f64) -> f64 {
+pub fn df5_rev(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f5(x, y));
- ::core::hint::black_box((dx, dret));
+ ::core::hint::black_box((dx_0, dret));
::core::hint::black_box(f5(x, y))
}
+struct DoesNotImplDefault;
+#[rustc_autodiff]
+#[inline(never)]
+pub fn f6() -> DoesNotImplDefault {
+ ::core::panicking::panic("not implemented")
+}
+#[rustc_autodiff(Forward, 1, Const)]
+#[inline(never)]
+pub fn df6() -> DoesNotImplDefault {
+ unsafe { asm!("NOP", options(pure, nomem)); };
+ ::core::hint::black_box(f6());
+ ::core::hint::black_box(());
+ ::core::hint::black_box(f6())
+}
+#[rustc_autodiff]
+#[inline(never)]
+pub fn f7(x: f32) -> () {}
+#[rustc_autodiff(Forward, 1, Const, None)]
+#[inline(never)]
+pub fn df7(x: f32) -> () {
+ unsafe { asm!("NOP", options(pure, nomem)); };
+ ::core::hint::black_box(f7(x));
+ ::core::hint::black_box(());
+}
+#[no_mangle]
+#[rustc_autodiff]
+#[inline(never)]
+fn f8(x: &f32) -> f32 { ::core::panicking::panic("not implemented") }
+#[rustc_autodiff(Forward, 4, Dual, Dual)]
+#[inline(never)]
+fn f8_3(x: &f32, bx_0: &f32, bx_1: &f32, bx_2: &f32, bx_3: &f32)
+ -> [f32; 5usize] {
+ unsafe { asm!("NOP", options(pure, nomem)); };
+ ::core::hint::black_box(f8(x));
+ ::core::hint::black_box((bx_0, bx_1, bx_2, bx_3));
+ ::core::hint::black_box(<[f32; 5usize]>::default())
+}
+#[rustc_autodiff(Forward, 4, Dual, DualOnly)]
+#[inline(never)]
+fn f8_2(x: &f32, bx_0: &f32, bx_1: &f32, bx_2: &f32, bx_3: &f32)
+ -> [f32; 4usize] {
+ unsafe { asm!("NOP", options(pure, nomem)); };
+ ::core::hint::black_box(f8(x));
+ ::core::hint::black_box((bx_0, bx_1, bx_2, bx_3));
+ ::core::hint::black_box(<[f32; 4usize]>::default())
+}
+#[rustc_autodiff(Forward, 1, Dual, DualOnly)]
+#[inline(never)]
+fn f8_1(x: &f32, bx_0: &f32) -> f32 {
+ unsafe { asm!("NOP", options(pure, nomem)); };
+ ::core::hint::black_box(f8(x));
+ ::core::hint::black_box((bx_0,));
+ ::core::hint::black_box(<f32>::default())
+}
fn main() {}
diff --git a/tests/pretty/autodiff_forward.rs b/tests/pretty/autodiff_forward.rs
index bc5582116322b..a765738c2a815 100644
--- a/tests/pretty/autodiff_forward.rs
+++ b/tests/pretty/autodiff_forward.rs
@@ -36,4 +36,22 @@ pub fn f5(x: &[f64], y: f64) -> f64 {
unimplemented!()
}
+struct DoesNotImplDefault;
+#[autodiff(df6, Forward, Const)]
+pub fn f6() -> DoesNotImplDefault {
+ unimplemented!()
+}
+
+// Make sure, that we add the None for the default return.
+#[autodiff(df7, Forward, Const)]
+pub fn f7(x: f32) -> () {}
+
+#[autodiff(f8_1, Forward, Dual, DualOnly)]
+#[autodiff(f8_2, Forward, 4, Dual, DualOnly)]
+#[autodiff(f8_3, Forward, 4, Dual, Dual)]
+#[no_mangle]
+fn f8(x: &f32) -> f32 {
+ unimplemented!()
+}
+
fn main() {}
diff --git a/tests/pretty/autodiff_reverse.pp b/tests/pretty/autodiff_reverse.pp
index b2cf0244af4c6..31920694a3ad5 100644
--- a/tests/pretty/autodiff_reverse.pp
+++ b/tests/pretty/autodiff_reverse.pp
@@ -28,18 +28,18 @@
::core::panicking::panic("not implemented")
}
-#[rustc_autodiff(Reverse, Duplicated, Const, Active,)]
+#[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
#[inline(never)]
-pub fn df1(x: &[f64], dx: &mut [f64], y: f64, dret: f64) -> f64 {
+pub fn df1(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f1(x, y));
- ::core::hint::black_box((dx, dret));
+ ::core::hint::black_box((dx_0, dret));
::core::hint::black_box(f1(x, y))
}
#[rustc_autodiff]
#[inline(never)]
pub fn f2() {}
-#[rustc_autodiff(Reverse, None)]
+#[rustc_autodiff(Reverse, 1, None)]
#[inline(never)]
pub fn df2() {
unsafe { asm!("NOP", options(pure, nomem)); };
@@ -51,12 +51,12 @@
pub fn f3(x: &[f64], y: f64) -> f64 {
::core::panicking::panic("not implemented")
}
-#[rustc_autodiff(Reverse, Duplicated, Const, Active,)]
+#[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
#[inline(never)]
-pub fn df3(x: &[f64], dx: &mut [f64], y: f64, dret: f64) -> f64 {
+pub fn df3(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f3(x, y));
- ::core::hint::black_box((dx, dret));
+ ::core::hint::black_box((dx_0, dret));
::core::hint::black_box(f3(x, y))
}
enum Foo { Reverse, }
@@ -64,7 +64,7 @@
#[rustc_autodiff]
#[inline(never)]
pub fn f4(x: f32) { ::core::panicking::panic("not implemented") }
-#[rustc_autodiff(Reverse, Const, None)]
+#[rustc_autodiff(Reverse, 1, Const, None)]
#[inline(never)]
pub fn df4(x: f32) {
unsafe { asm!("NOP", options(pure, nomem)); };
@@ -76,11 +76,11 @@
pub fn f5(x: *const f32, y: &f32) {
::core::panicking::panic("not implemented")
}
-#[rustc_autodiff(Reverse, DuplicatedOnly, Duplicated, None)]
+#[rustc_autodiff(Reverse, 1, DuplicatedOnly, Duplicated, None)]
#[inline(never)]
-pub unsafe fn df5(x: *const f32, dx: *mut f32, y: &f32, dy: &mut f32) {
+pub unsafe fn df5(x: *const f32, dx_0: *mut f32, y: &f32, dy_0: &mut f32) {
unsafe { asm!("NOP", options(pure, nomem)); };
::core::hint::black_box(f5(x, y));
- ::core::hint::black_box((dx, dy));
+ ::core::hint::black_box((dx_0, dy_0));
}
fn main() {}
diff --git a/tests/run-make/apple-slow-tls/rmake.rs b/tests/run-make/apple-slow-tls/rmake.rs
new file mode 100644
index 0000000000000..231e0b1668e99
--- /dev/null
+++ b/tests/run-make/apple-slow-tls/rmake.rs
@@ -0,0 +1,37 @@
+//! Test if compilation with has-thread-local=false works, and if the output
+//! has indeed no fast TLS variables.
+
+//@ only-apple
+
+use run_make_support::serde_json::{self, Value};
+use run_make_support::{cargo, llvm_nm, rfs, rustc};
+
+fn main() {
+ let output =
+ rustc().print("target-spec-json").args(["-Z", "unstable-options"]).run().stdout_utf8();
+
+ let mut target_json: Value = serde_json::from_str(&output).unwrap();
+ let has_thread_local = &mut target_json["has-thread-local"];
+ assert!(matches!(has_thread_local, Value::Bool(true)), "{:?}", has_thread_local);
+ *has_thread_local = Value::Bool(false);
+
+ let out_path = "t.json";
+ rfs::write(out_path, serde_json::to_string(&target_json).unwrap());
+
+ cargo()
+ .args([
+ "b",
+ "--manifest-path",
+ "tls_test/Cargo.toml",
+ "--target",
+ "t.json",
+ "-Zbuild-std=std,core,panic_abort",
+ ])
+ .run();
+
+ // If a binary has any fast TLS variables, it should also contain the symbols
+ // __tlv_bootstrap and __tlv_atexit. We don't want them.
+ let output = llvm_nm().arg("tls_test/target/t/debug/tls_test").run().stdout_utf8();
+ assert!(!output.contains("_tlv_bootstrap"));
+ assert!(!output.contains("_tlv_atexit"));
+}
diff --git a/tests/run-make/apple-slow-tls/tls_test/Cargo.toml b/tests/run-make/apple-slow-tls/tls_test/Cargo.toml
new file mode 100644
index 0000000000000..c8e5a228eef79
--- /dev/null
+++ b/tests/run-make/apple-slow-tls/tls_test/Cargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "tls_test"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
diff --git a/tests/run-make/apple-slow-tls/tls_test/src/main.rs b/tests/run-make/apple-slow-tls/tls_test/src/main.rs
new file mode 100644
index 0000000000000..d48d22bb3d17b
--- /dev/null
+++ b/tests/run-make/apple-slow-tls/tls_test/src/main.rs
@@ -0,0 +1,10 @@
+use std::cell::RefCell;
+
+fn main() {
+ thread_local! {
+ static S: RefCell<String> = RefCell::default();
+ }
+
+ S.with(|x| *x.borrow_mut() = "pika pika".to_string());
+ S.with(|x| println!("{}", x.borrow()));
+}
diff --git a/tests/run-make/target-specs/rmake.rs b/tests/run-make/target-specs/rmake.rs
index f36a5784c8930..9184e5f772f78 100644
--- a/tests/run-make/target-specs/rmake.rs
+++ b/tests/run-make/target-specs/rmake.rs
@@ -14,7 +14,7 @@ fn main() {
.input("foo.rs")
.target("my-invalid-platform.json")
.run_fail()
- .assert_stderr_contains("Error loading target specification");
+ .assert_stderr_contains("error loading target specification");
rustc()
.input("foo.rs")
.target("my-incomplete-platform.json")
diff --git a/tests/ui-fulldeps/stable-mir/check_assoc_items.rs b/tests/ui-fulldeps/stable-mir/check_assoc_items.rs
new file mode 100644
index 0000000000000..f6f895588f27b
--- /dev/null
+++ b/tests/ui-fulldeps/stable-mir/check_assoc_items.rs
@@ -0,0 +1,145 @@
+//@ run-pass
+//! Test that users are able to retrieve all associated items from a definition.
+//! definition.
+
+//@ ignore-stage1
+//@ ignore-cross-compile
+//@ ignore-remote
+//@ edition: 2021
+
+#![feature(rustc_private)]
+#![feature(assert_matches)]
+
+extern crate rustc_middle;
+#[macro_use]
+extern crate rustc_smir;
+extern crate rustc_driver;
+extern crate rustc_interface;
+extern crate stable_mir;
+
+use rustc_smir::rustc_internal;
+use std::io::Write;
+use std::collections::HashSet;
+use stable_mir::CrateDef;
+use stable_mir::*;
+use stable_mir::ty::*;
+use std::ops::ControlFlow;
+
+const CRATE_NAME: &str = "crate_assoc_items";
+
+/// This function uses the Stable MIR APIs to get information about the test crate.
+fn test_assoc_items() -> ControlFlow<()> {
+ let local_crate = stable_mir::local_crate();
+ check_items(
+ &local_crate.fn_defs(),
+ &[
+ "AStruct::new",
+ "<AStruct as ATrait>::assoc_fn_no_self",
+ "<AStruct as ATrait>::assoc_fn_has_self",
+ "ATrait::rpitit",
+ "ATrait::assoc_fn_has_self",
+ "ATrait::assoc_fn_no_self",
+ "<AStruct as ATrait>::rpitit",
+ ],
+ );
+
+ let local_impls = local_crate.trait_impls();
+ let local_traits = local_crate.trait_decls();
+
+ let trait_assoc_item_defs: Vec<AssocDef> = local_traits[0].associated_items()
+ .iter().map(|assoc_item| assoc_item.def_id).collect();
+ check_items(
+ &trait_assoc_item_defs,
+ &[
+ "ATrait::{synthetic#0}",
+ "ATrait::rpitit",
+ "ATrait::Assoc",
+ "ATrait::assoc_fn_no_self",
+ "ATrait::assoc_fn_has_self",
+ ]
+ );
+
+ let impl_assoc_item_defs: Vec<AssocDef> = local_impls[0].associated_items()
+ .iter().map(|assoc_item| assoc_item.def_id).collect();
+ check_items(
+ &impl_assoc_item_defs,
+ &[
+ "<AStruct as ATrait>::{synthetic#0}",
+ "<AStruct as ATrait>::rpitit",
+ "<AStruct as ATrait>::Assoc",
+ "<AStruct as ATrait>::assoc_fn_no_self",
+ "<AStruct as ATrait>::assoc_fn_has_self",
+ ]
+ );
+
+ ControlFlow::Continue(())
+}
+
+/// Check if the list of definitions matches the expected list.
+/// Note that order doesn't matter.
+fn check_items<T: CrateDef>(items: &[T], expected: &[&str]) {
+ let expected: HashSet<_> = expected.iter().map(|s| s.to_string()).collect();
+ let item_names: HashSet<_> = items.iter().map(|item| item.name()).collect();
+ assert_eq!(item_names, expected);
+}
+
+fn main() {
+ let path = "assoc_items.rs";
+ generate_input(&path).unwrap();
+ let args = vec![
+ "rustc".to_string(),
+ "--crate-type=lib".to_string(),
+ "--crate-name".to_string(),
+ CRATE_NAME.to_string(),
+ path.to_string(),
+ ];
+ run!(args, test_assoc_items).unwrap();
+}
+
+fn generate_input(path: &str) -> std::io::Result<()> {
+ let mut file = std::fs::File::create(path)?;
+ write!(
+ file,
+ r#"
+ #![allow(dead_code, unused_variables)]
+ struct AStruct;
+
+ impl AStruct {{
+ const ASSOC_CONST: &str = "Nina";
+
+ fn new() -> Self {{
+ AStruct{{}}
+ }}
+ }}
+
+ trait ATrait {{
+ type Assoc;
+
+ fn assoc_fn_no_self() {{
+ }}
+
+ fn assoc_fn_has_self(&self) {{
+ }}
+
+ fn rpitit(&self) -> impl std::fmt::Debug {{
+ "ciallo"
+ }}
+ }}
+
+ impl ATrait for AStruct {{
+ type Assoc = u32;
+
+ fn assoc_fn_no_self() {{
+ }}
+
+ fn assoc_fn_has_self(&self) {{
+ }}
+
+ fn rpitit(&self) -> impl std::fmt::Debug {{
+ "ciallo~"
+ }}
+ }}
+ "#
+ )?;
+ Ok(())
+}
diff --git a/tests/ui/associated-consts/issue-93835.rs b/tests/ui/associated-consts/issue-93835.rs
index 048681f047789..d6c2acaa9edce 100644
--- a/tests/ui/associated-consts/issue-93835.rs
+++ b/tests/ui/associated-consts/issue-93835.rs
@@ -3,11 +3,10 @@
fn e() {
type_ascribe!(p, a<p:p<e=6>>);
//~^ ERROR cannot find type `a` in this scope
- //~| ERROR path separator must be a double colon
//~| ERROR cannot find value
//~| ERROR associated const equality
+ //~| ERROR cannot find trait `p` in this scope
//~| ERROR associated const equality
- //~| ERROR failed to resolve: use of unresolved module or unlinked crate `p`
}
fn main() {}
diff --git a/tests/ui/associated-consts/issue-93835.stderr b/tests/ui/associated-consts/issue-93835.stderr
index e154ae25de26c..551b50d0eb6b5 100644
--- a/tests/ui/associated-consts/issue-93835.stderr
+++ b/tests/ui/associated-consts/issue-93835.stderr
@@ -1,15 +1,3 @@
-error: path separator must be a double colon
- --> $DIR/issue-93835.rs:4:25
- |
-LL | type_ascribe!(p, a<p:p<e=6>>);
- | ^
- |
- = note: if you meant to annotate an expression with a type, the type ascription syntax has been removed, see issue #101728 <https://github.com/rust-lang/rust/issues/101728>
-help: use a double colon instead
- |
-LL | type_ascribe!(p, a<p::p<e=6>>);
- | +
-
error[E0425]: cannot find value `p` in this scope
--> $DIR/issue-93835.rs:4:19
|
@@ -22,6 +10,12 @@ error[E0412]: cannot find type `a` in this scope
LL | type_ascribe!(p, a<p:p<e=6>>);
| ^ not found in this scope
+error[E0405]: cannot find trait `p` in this scope
+ --> $DIR/issue-93835.rs:4:26
+ |
+LL | type_ascribe!(p, a<p:p<e=6>>);
+ | ^ not found in this scope
+
error[E0658]: associated const equality is incomplete
--> $DIR/issue-93835.rs:4:28
|
@@ -43,15 +37,7 @@ LL | type_ascribe!(p, a<p:p<e=6>>);
= note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
= note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
-error[E0433]: failed to resolve: use of unresolved module or unlinked crate `p`
- --> $DIR/issue-93835.rs:4:24
- |
-LL | type_ascribe!(p, a<p:p<e=6>>);
- | ^ use of unresolved module or unlinked crate `p`
- |
- = help: you might be missing a crate named `p`
-
-error: aborting due to 6 previous errors
+error: aborting due to 5 previous errors
-Some errors have detailed explanations: E0412, E0425, E0433, E0658.
-For more information about an error, try `rustc --explain E0412`.
+Some errors have detailed explanations: E0405, E0412, E0425, E0658.
+For more information about an error, try `rustc --explain E0405`.
diff --git a/tests/ui/autodiff/autodiff_illegal.rs b/tests/ui/autodiff/autodiff_illegal.rs
index e810b9ba565ba..2f2cd8d93532f 100644
--- a/tests/ui/autodiff/autodiff_illegal.rs
+++ b/tests/ui/autodiff/autodiff_illegal.rs
@@ -177,4 +177,11 @@ fn f21(x: f32) -> f32 {
unimplemented!()
}
+struct DoesNotImplDefault;
+#[autodiff(df22, Forward, Dual)]
+pub fn f22() -> DoesNotImplDefault {
+ //~^^ ERROR the function or associated item `default` exists for tuple `(DoesNotImplDefault, DoesNotImplDefault)`, but its trait bounds were not satisfied
+ unimplemented!()
+}
+
fn main() {}
diff --git a/tests/ui/autodiff/autodiff_illegal.stderr b/tests/ui/autodiff/autodiff_illegal.stderr
index 47d53492700ba..3752b27e7dd1b 100644
--- a/tests/ui/autodiff/autodiff_illegal.stderr
+++ b/tests/ui/autodiff/autodiff_illegal.stderr
@@ -19,32 +19,24 @@ error: expected 1 activities, but found 2
|
LL | #[autodiff(df3, Reverse, Duplicated, Const)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error: expected 1 activities, but found 0
--> $DIR/autodiff_illegal.rs:27:1
|
LL | #[autodiff(df4, Reverse)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error: Dual can not be used in Reverse Mode
--> $DIR/autodiff_illegal.rs:34:1
|
LL | #[autodiff(df5, Reverse, Dual)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error: Duplicated can not be used in Forward Mode
--> $DIR/autodiff_illegal.rs:41:1
|
LL | #[autodiff(df6, Forward, Duplicated)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error: Duplicated can not be used for this type
--> $DIR/autodiff_illegal.rs:42:14
@@ -107,7 +99,6 @@ LL | #[autodiff(fn_exists, Reverse, Active)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `fn_exists` redefined here
|
= note: `fn_exists` must be defined only once in the value namespace of this module
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error: autodiff requires at least a name and mode
--> $DIR/autodiff_illegal.rs:95:1
@@ -135,42 +126,49 @@ error: invalid return activity Active in Forward Mode
|
LL | #[autodiff(df19, Forward, Dual, Active)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error: invalid return activity Dual in Reverse Mode
--> $DIR/autodiff_illegal.rs:167:1
|
LL | #[autodiff(df20, Reverse, Active, Dual)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error: invalid return activity Duplicated in Reverse Mode
--> $DIR/autodiff_illegal.rs:174:1
|
LL | #[autodiff(df21, Reverse, Active, Duplicated)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error[E0433]: failed to resolve: use of undeclared type `MyFloat`
--> $DIR/autodiff_illegal.rs:130:1
|
LL | #[autodiff(df15, Reverse, Active, Active)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ use of undeclared type `MyFloat`
- |
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
error[E0433]: failed to resolve: use of undeclared type `F64Trans`
--> $DIR/autodiff_illegal.rs:154:1
|
LL | #[autodiff(df18, Reverse, Active, Active)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ use of undeclared type `F64Trans`
+
+error[E0599]: the function or associated item `default` exists for tuple `(DoesNotImplDefault, DoesNotImplDefault)`, but its trait bounds were not satisfied
+ --> $DIR/autodiff_illegal.rs:181:1
+ |
+LL | struct DoesNotImplDefault;
+ | ------------------------- doesn't satisfy `DoesNotImplDefault: Default`
+LL | #[autodiff(df22, Forward, Dual)]
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function or associated item cannot be called on `(DoesNotImplDefault, DoesNotImplDefault)` due to unsatisfied trait bounds
+ |
+ = note: the following trait bounds were not satisfied:
+ `DoesNotImplDefault: Default`
+ which is required by `(DoesNotImplDefault, DoesNotImplDefault): Default`
+help: consider annotating `DoesNotImplDefault` with `#[derive(Default)]`
+ |
+LL + #[derive(Default)]
+LL | struct DoesNotImplDefault;
|
- = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
-error: aborting due to 22 previous errors
+error: aborting due to 23 previous errors
-Some errors have detailed explanations: E0428, E0433, E0658.
+Some errors have detailed explanations: E0428, E0433, E0599, E0658.
For more information about an error, try `rustc --explain E0428`.
diff --git a/tests/ui/errors/wrong-target-spec.rs b/tests/ui/errors/wrong-target-spec.rs
index 9b31c943e3e98..a3a0e05d82662 100644
--- a/tests/ui/errors/wrong-target-spec.rs
+++ b/tests/ui/errors/wrong-target-spec.rs
@@ -7,4 +7,4 @@
fn main() {}
-//~? ERROR Error loading target specification: Could not find specification for target "x86_64_unknown-linux-musl"
+//~? ERROR error loading target specification: could not find specification for target "x86_64_unknown-linux-musl"
diff --git a/tests/ui/errors/wrong-target-spec.stderr b/tests/ui/errors/wrong-target-spec.stderr
index 8b06f404078d0..98b03ae00cb3a 100644
--- a/tests/ui/errors/wrong-target-spec.stderr
+++ b/tests/ui/errors/wrong-target-spec.stderr
@@ -1,2 +1,4 @@
-error: Error loading target specification: Could not find specification for target "x86_64_unknown-linux-musl". Run `rustc --print target-list` for a list of built-in targets
+error: error loading target specification: could not find specification for target "x86_64_unknown-linux-musl"
+ |
+ = help: run `rustc --print target-list` for a list of built-in targets
diff --git a/tests/ui/macros/failed-to-reparse-issue-137874.rs b/tests/ui/macros/failed-to-reparse-issue-137874.rs
new file mode 100644
index 0000000000000..3e55ac376fea8
--- /dev/null
+++ b/tests/ui/macros/failed-to-reparse-issue-137874.rs
@@ -0,0 +1,12 @@
+// This originally crashed because `Recovery::Forbidden` wasn't being applied
+// when fragments pasted by declarative macros were reparsed.
+
+macro_rules! m {
+ ($p:pat) => {
+ if let $p = 0 {}
+ }
+}
+
+fn main() {
+ m!(0X0); //~ ERROR invalid base prefix for number literal
+}
diff --git a/tests/ui/macros/failed-to-reparse-issue-137874.stderr b/tests/ui/macros/failed-to-reparse-issue-137874.stderr
new file mode 100644
index 0000000000000..5bbb8b7f9e2f4
--- /dev/null
+++ b/tests/ui/macros/failed-to-reparse-issue-137874.stderr
@@ -0,0 +1,10 @@
+error: invalid base prefix for number literal
+ --> $DIR/failed-to-reparse-issue-137874.rs:11:8
+ |
+LL | m!(0X0);
+ | ^^^ help: try making the prefix lowercase (notice the capitalization): `0x0`
+ |
+ = note: base prefixes (`0xff`, `0b1010`, `0o755`) are lowercase
+
+error: aborting due to 1 previous error
+