Reject unsupported extern "{abi}"s consistently in all positions

compiler/rustc_ast_lowering/src/item.rs

@@ -2,7 +2,7 @@ use rustc_abi::ExternAbi;
 use rustc_ast::ptr::P;
 use rustc_ast::visit::AssocCtxt;
 use rustc_ast::*;
-use rustc_errors::ErrorGuaranteed;
+use rustc_errors::{E0570, ErrorGuaranteed, struct_span_code_err};
 use rustc_hir::def::{DefKind, PerNS, Res};
 use rustc_hir::def_id::{CRATE_DEF_ID, LocalDefId};
 use rustc_hir::{self as hir, HirId, LifetimeSource, PredicateOrigin};
@@ -1617,9 +1617,29 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.error_on_invalid_abi(abi_str);
             ExternAbi::Rust
         });
-        let sess = self.tcx.sess;
-        let features = self.tcx.features();
-        gate_unstable_abi(sess, features, span, extern_abi);
+        let tcx = self.tcx;
+
+        // we can't do codegen for unsupported ABIs, so error now so we won't get farther
+        if !tcx.sess.target.is_abi_supported(extern_abi) {
+            let mut err = struct_span_code_err!(
+                tcx.dcx(),
+                span,
+                E0570,
+                "`{extern_abi}` is not a supported ABI for the current target",
+            );
+
+            if let ExternAbi::Stdcall { unwind } = extern_abi {
+                let c_abi = ExternAbi::C { unwind };
+                let system_abi = ExternAbi::System { unwind };
+                err.help(format!("if you need `extern {extern_abi}` on win32 and `extern {c_abi}` everywhere else, \
+                use `extern {system_abi}`"
+                ));
+            }
+            err.emit();
+        }
+        // Show required feature gate even if we already errored, as the user is likely to build the code
+        // for the actually intended target next and then they will need the feature gate.
+        gate_unstable_abi(tcx.sess, tcx.features(), span, extern_abi);
         extern_abi
     }
 

compiler/rustc_hir_analysis/src/check/check.rs

@@ -1,7 +1,7 @@
 use std::cell::LazyCell;
 use std::ops::ControlFlow;
 
-use rustc_abi::FieldIdx;
+use rustc_abi::{ExternAbi, FieldIdx};
 use rustc_attr_data_structures::ReprAttr::ReprPacked;
 use rustc_data_structures::unord::{UnordMap, UnordSet};
 use rustc_errors::codes::*;
@@ -12,7 +12,6 @@ use rustc_infer::infer::{RegionVariableOrigin, TyCtxtInferExt};
 use rustc_infer::traits::{Obligation, ObligationCauseCode};
 use rustc_lint_defs::builtin::{
     REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS, UNSUPPORTED_CALLING_CONVENTIONS,
-    UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
 };
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::resolve_bound_vars::ResolvedArg;
@@ -52,49 +51,22 @@ fn add_abi_diag_help<T: EmissionGuarantee>(abi: ExternAbi, diag: &mut Diag<'_, T
 }
 
 pub fn check_abi(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
-    // FIXME: this should be checked earlier, e.g. in `rustc_ast_lowering`, to fix
-    // things like #86232.
+    // FIXME: This should be checked earlier, e.g. in `rustc_ast_lowering`, as this
+    // currently only guards function imports, function definitions, and function pointer types.
+    // Functions in trait declarations can still use "deprecated" ABIs without any warning.
 
     match AbiMap::from_target(&tcx.sess.target).canonize_abi(abi, false) {
         AbiMapping::Direct(..) => (),
+        // already erred in rustc_ast_lowering
         AbiMapping::Invalid => {
-            let mut err = struct_span_code_err!(
-                tcx.dcx(),
-                span,
-                E0570,
-                "`{abi}` is not a supported ABI for the current target",
-            );
-            add_abi_diag_help(abi, &mut err);
-            err.emit();
+            tcx.dcx().span_delayed_bug(span, format!("{abi} should be rejected in ast_lowering"));
         }
         AbiMapping::Deprecated(..) => {
             tcx.node_span_lint(UNSUPPORTED_CALLING_CONVENTIONS, hir_id, span, |lint| {
-                lint.primary_message("use of calling convention not supported on this target");
-                add_abi_diag_help(abi, lint);
-            });
-        }
-    }
-}
-
-pub fn check_abi_fn_ptr(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
-    // This is always an FCW, even for `AbiMapping::Invalid`, since we started linting later than
-    // in `check_abi` above.
-    match AbiMap::from_target(&tcx.sess.target).canonize_abi(abi, false) {
-        AbiMapping::Direct(..) => (),
-        // This is not a redundant match arm: these ABIs started linting after introducing
-        // UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS already existed and we want to
-        // avoid expanding the scope of that lint so it can move to a hard error sooner.
-        AbiMapping::Deprecated(..) => {
-            tcx.node_span_lint(UNSUPPORTED_CALLING_CONVENTIONS, hir_id, span, |lint| {
-                lint.primary_message("use of calling convention not supported on this target");
-                add_abi_diag_help(abi, lint);
-            });
-        }
-        AbiMapping::Invalid => {
-            tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
                 lint.primary_message(format!(
-                    "the calling convention {abi} is not supported on this target"
+                    "{abi} is not a supported ABI for the current target"
                 ));
+                add_abi_diag_help(abi, lint);
             });
         }
     }
@@ -855,6 +827,7 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
             let hir::ItemKind::ForeignMod { abi, items } = it.kind else {
                 return;
             };
+
             check_abi(tcx, it.hir_id(), it.span, abi);
 
             for item in items {

compiler/rustc_hir_analysis/src/check/mod.rs

@@ -72,8 +72,8 @@ pub mod wfcheck;
 
 use std::num::NonZero;
 
-pub use check::{check_abi, check_abi_fn_ptr};
-use rustc_abi::{ExternAbi, VariantIdx};
+pub use check::check_abi;
+use rustc_abi::VariantIdx;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_errors::{Diag, ErrorGuaranteed, pluralize, struct_span_code_err};
 use rustc_hir::def_id::{DefId, LocalDefId};

compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs

@@ -50,7 +50,7 @@ use rustc_trait_selection::traits::wf::object_region_bounds;
 use rustc_trait_selection::traits::{self, ObligationCtxt};
 use tracing::{debug, instrument};
 
-use crate::check::check_abi_fn_ptr;
+use crate::check::check_abi;
 use crate::errors::{AmbiguousLifetimeBound, BadReturnTypeNotation};
 use crate::hir_ty_lowering::errors::{GenericsArgsErrExtend, prohibit_assoc_item_constraint};
 use crate::hir_ty_lowering::generics::{check_generic_arg_count, lower_generic_args};
@@ -2743,7 +2743,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         if let hir::Node::Ty(hir::Ty { kind: hir::TyKind::BareFn(bare_fn_ty), span, .. }) =
             tcx.hir_node(hir_id)
         {
-            check_abi_fn_ptr(tcx, hir_id, *span, bare_fn_ty.abi);
+            check_abi(tcx, hir_id, *span, bare_fn_ty.abi);
         }
 
         // reject function types that violate cmse ABI requirements

compiler/rustc_lint/src/lib.rs

@@ -609,6 +609,7 @@ fn register_builtins(store: &mut LintStore) {
         "converted into hard error, see issue #127323 \
          <https://github.com/rust-lang/rust/issues/127323> for more information",
     );
+    store.register_removed("unsupported_fn_ptr_calling_conventions", "converted into hard error");
     store.register_removed(
         "undefined_naked_function_abi",
         "converted into hard error, see PR #139001 \

compiler/rustc_lint_defs/src/builtin.rs

@@ -123,7 +123,6 @@ declare_lint_pass! {
         UNSAFE_OP_IN_UNSAFE_FN,
         UNSTABLE_NAME_COLLISIONS,
         UNSTABLE_SYNTAX_PRE_EXPANSION,
-        UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
         UNUSED_ASSIGNMENTS,
         UNUSED_ASSOCIATED_TYPE_BOUNDS,
         UNUSED_ATTRIBUTES,

tests/auxiliary/minicore.rs

@@ -16,6 +16,7 @@
 
 #![feature(
     no_core,
+    intrinsics,
     lang_items,
     auto_traits,
     freeze_impls,
@@ -190,3 +191,9 @@ impl<'a, 'b: 'a, T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<&'a U> for &'b
 trait Drop {
     fn drop(&mut self);
 }
+
+pub mod mem {
+    #[rustc_nounwind]
+    #[rustc_intrinsic]
+    pub unsafe fn transmute<Src, Dst>(src: Src) -> Dst;
+}

tests/ui/abi/unsupported-abi-transmute.rs

@@ -0,0 +1,15 @@
+//@ add-core-stubs
+//@ compile-flags: --crate-type=lib --target x86_64-unknown-none
+//@ needs-llvm-components: x86
+//@ edition: 2018
+#![no_core]
+#![feature(no_core, lang_items)]
+extern crate minicore;
+use minicore::*;
+
+// Check we error before unsupported ABIs reach codegen stages.
+
+fn anything() {
+    let a = unsafe { mem::transmute::<usize, extern "thiscall" fn(i32)>(4) }(2);
+    //~^ ERROR: is not a supported ABI for the current target [E0570]
+}

tests/ui/abi/unsupported-abi-transmute.stderr

@@ -0,0 +1,9 @@
+error[E0570]: `"thiscall"` is not a supported ABI for the current target
+  --> $DIR/unsupported-abi-transmute.rs:13:53
+   |
+LL |     let a = unsafe { mem::transmute::<usize, extern "thiscall" fn(i32)>(4) }(2);
+   |                                                     ^^^^^^^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0570`.

tests/ui/abi/unsupported-varargs-fnptr.rs

@@ -0,0 +1,18 @@
+// FIXME(workingjubilee): add revisions and generalize to other platform-specific varargs ABIs,
+// preferably after the only-arch directive is enhanced with an "or pattern" syntax
+//@ only-x86_64
+
+// We have to use this flag to force ABI computation of an invalid ABI
+//@ compile-flags: -Clink-dead-code
+
+#![feature(extended_varargs_abi_support)]
+
+// sometimes fn ptrs with varargs make layout and ABI computation ICE
+// as found in https://github.com/rust-lang/rust/issues/142107
+
+fn aapcs(f: extern "aapcs" fn(usize, ...)) {
+//~^ ERROR [E0570]
+// Note we DO NOT have to actually make a call to trigger the ICE!
+}
+
+fn main() {}

tests/ui/abi/unsupported-varargs-fnptr.stderr

@@ -0,0 +1,9 @@
+error[E0570]: `"aapcs"` is not a supported ABI for the current target
+  --> $DIR/unsupported-varargs-fnptr.rs:13:20
+   |
+LL | fn aapcs(f: extern "aapcs" fn(usize, ...)) {
+   |                    ^^^^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0570`.