diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl index a637ae8184b4f..735a64f426c95 100644 --- a/compiler/rustc_codegen_llvm/messages.ftl +++ b/compiler/rustc_codegen_llvm/messages.ftl @@ -11,6 +11,12 @@ codegen_llvm_from_llvm_diag = {$message} codegen_llvm_from_llvm_optimization_diag = {$filename}:{$line}:{$column} {$pass_name} ({$kind}): {$message} +codegen_llvm_intrinsic_signature_mismatch = + intrinsic signature mismatch for `{$name}`: expected signature `{$llvm_fn_ty}`, found `{$rust_fn_ty}` + +codegen_llvm_intrinsic_wrong_arch = + intrinsic `{$name}` cannot be used with target arch `{$target_arch}` + codegen_llvm_load_bitcode = failed to load bitcode of module "{$name}" codegen_llvm_load_bitcode_with_llvm_err = failed to load bitcode of module "{$name}": {$llvm_err} diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index c3c1caf086f09..8a7d19c5164a6 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -308,6 +308,8 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { + fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>; fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv; @@ -325,26 +327,30 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { } impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { - fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + match &self.ret.mode { + PassMode::Ignore => cx.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), + PassMode::Cast { cast, .. } => cast.llvm_type(cx), + PassMode::Indirect { .. } => cx.type_void(), + } + } + + fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type> { + let indirect_return = matches!(self.ret.mode, PassMode::Indirect { .. }); + // Ignore "extra" args from the call site for C variadic functions. // Only the "fixed" args are part of the LLVM function signature. let args = if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args }; // This capacity calculation is approximate. - let mut llargument_tys = Vec::with_capacity( - self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 }, - ); + let mut llargument_tys = + Vec::with_capacity(self.args.len() + if indirect_return { 1 } else { 0 }); - let llreturn_ty = match &self.ret.mode { - PassMode::Ignore => cx.type_void(), - PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx), - PassMode::Indirect { .. } => { - llargument_tys.push(cx.type_ptr()); - cx.type_void() - } - }; + if indirect_return { + llargument_tys.push(cx.type_ptr()); + } for arg in args { // Note that the exact number of arguments pushed here is carefully synchronized with @@ -391,10 +397,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(llarg_ty); } + llargument_tys + } + + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + let return_ty = self.llvm_return_type(cx); + let argument_tys = self.llvm_argument_types(cx); + if self.c_variadic { - cx.type_variadic_func(&llargument_tys, llreturn_ty) + cx.type_variadic_func(&argument_tys, return_ty) } else { - cx.type_func(&llargument_tys, llreturn_ty) + cx.type_func(&argument_tys, return_ty) } } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index c73140e041b60..368c3b2ec37f5 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -183,3 +183,22 @@ pub(crate) struct FixedX18InvalidArch<'a> { #[derive(Diagnostic)] #[diag(codegen_llvm_sanitizer_kcfi_arity_requires_llvm_21_0_0)] pub(crate) struct SanitizerKcfiArityRequiresLLVM2100; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_intrinsic_signature_mismatch)] +pub(crate) struct IntrinsicSignatureMismatch<'a> { + pub name: &'a str, + pub llvm_fn_ty: &'a str, + pub rust_fn_ty: &'a str, + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_intrinsic_wrong_arch)] +pub(crate) struct IntrinsicWrongArch<'a> { + pub name: &'a str, + pub target_arch: &'a str, + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index b4057eea735ea..395fcaf4ac28b 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,7 +1,7 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; use std::ffi::c_uint; -use std::ptr; +use std::{iter, ptr}; use rustc_abi::{ Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, @@ -21,10 +21,11 @@ use rustc_middle::ty::offload_meta::OffloadMetadata; use rustc_middle::ty::{self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt, TypingEnv}; use rustc_middle::{bug, span_bug}; use rustc_session::config::CrateType; +use rustc_session::lint::builtin::{DEPRECATED_LLVM_INTRINSIC, UNKNOWN_LLVM_INTRINSIC}; use rustc_span::{Span, Symbol, sym}; use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; -use rustc_target::callconv::PassMode; -use rustc_target::spec::Os; +use rustc_target::callconv::{FnAbi, PassMode}; +use rustc_target::spec::{Arch, Os}; use tracing::debug; use crate::abi::FnAbiLlvmExt; @@ -34,7 +35,8 @@ use crate::builder::gpu_offload::{gen_call_handling, gen_define_handling}; use crate::context::CodegenCx; use crate::declare::declare_raw_fn; use crate::errors::{ - AutoDiffWithoutEnable, AutoDiffWithoutLto, OffloadWithoutEnable, OffloadWithoutFatLTO, + AutoDiffWithoutEnable, AutoDiffWithoutLto, IntrinsicSignatureMismatch, IntrinsicWrongArch, + OffloadWithoutEnable, OffloadWithoutFatLTO, }; use crate::llvm::{self, Metadata, Type, Value}; use crate::type_of::LayoutLlvmExt; @@ -642,50 +644,33 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &mut self, instance: ty::Instance<'tcx>, args: &[OperandRef<'tcx, Self::Value>], - is_cleanup: bool, + _is_cleanup: bool, ) -> Self::Value { - let tcx = self.tcx(); - - // FIXME remove usage of fn_abi let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); assert!(!fn_abi.ret.is_indirect()); - let fn_ty = fn_abi.llvm_type(self); let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) { llfn } else { - let sym = tcx.symbol_name(instance).name; + let sym = self.tcx.symbol_name(instance).name; - // FIXME use get_intrinsic let llfn = if let Some(llfn) = self.get_declared_value(sym) { llfn } else { - // Function addresses in Rust are never significant, allowing functions to - // be merged. - let llfn = declare_raw_fn( - self, - sym, - fn_abi.llvm_cconv(self), - llvm::UnnamedAddr::Global, - llvm::Visibility::Default, - fn_ty, - ); - fn_abi.apply_attrs_llfn(self, llfn, Some(instance)); - - llfn + intrinsic_fn(self, sym, fn_abi, instance) }; self.intrinsic_instances.borrow_mut().insert(instance, llfn); - llfn }; + let fn_ty = self.get_type_of_global(fn_ptr); let mut llargs = vec![]; for arg in args { match arg.val { OperandValue::ZeroSized => {} - OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Immediate(a) => llargs.push(a), OperandValue::Pair(a, b) => { llargs.push(a); llargs.push(b); @@ -711,24 +696,38 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } debug!("call intrinsic {:?} with args ({:?})", instance, llargs); - let args = self.check_call("call", fn_ty, fn_ptr, &llargs); + + for (dest_ty, arg) in iter::zip(self.func_params_types(fn_ty), &mut llargs) { + let src_ty = self.val_ty(arg); + assert!( + equate_ty(self, src_ty, dest_ty), + "Cannot match `{dest_ty:?}` (expected) with {src_ty:?} (found) in `{fn_ptr:?}" + ); + + *arg = autocast(self, fn_ptr, arg, src_ty, dest_ty, true); + } + let llret = unsafe { llvm::LLVMBuildCallWithOperandBundles( self.llbuilder, fn_ty, fn_ptr, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, + llargs.as_ptr(), + llargs.len() as c_uint, ptr::dangling(), 0, c"".as_ptr(), ) }; - if is_cleanup { - self.apply_attrs_to_cleanup_callsite(llret); - } - llret + let src_ty = self.val_ty(llret); + let dest_ty = fn_abi.llvm_return_type(self); + assert!( + equate_ty(self, dest_ty, src_ty), + "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`" + ); + + autocast(self, fn_ptr, llret, src_ty, dest_ty, false) } fn abort(&mut self) { @@ -778,6 +777,234 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } +fn llvm_arch_for(rust_arch: &Arch) -> Option<&'static str> { + Some(match rust_arch { + Arch::AArch64 | Arch::Arm64EC => "aarch64", + Arch::AmdGpu => "amdgcn", + Arch::Arm => "arm", + Arch::Bpf => "bpf", + Arch::Hexagon => "hexagon", + Arch::LoongArch32 | Arch::LoongArch64 => "loongarch", + Arch::Mips | Arch::Mips32r6 | Arch::Mips64 | Arch::Mips64r6 => "mips", + Arch::Nvptx64 => "nvvm", + Arch::PowerPC | Arch::PowerPC64 | Arch::PowerPC64LE => "ppc", + Arch::RiscV32 | Arch::RiscV64 => "riscv", + Arch::S390x => "s390", + Arch::SpirV => "spv", + Arch::Wasm32 | Arch::Wasm64 => "wasm", + Arch::X86 | Arch::X86_64 => "x86", + _ => return None, // fallback for unknown archs + }) +} + +fn equate_ty<'ll>(cx: &CodegenCx<'ll, '_>, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool { + if rust_ty == llvm_ty { + return true; + } + + match cx.type_kind(llvm_ty) { + // Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust + // due to auto field-alignment in non-packed structs (packed structs are represented in LLVM + // as, well, packed structs, so they won't match with those either) + TypeKind::Struct if cx.type_kind(rust_ty) == TypeKind::Struct => { + let rust_element_tys = cx.struct_element_types(rust_ty); + let llvm_element_tys = cx.struct_element_types(llvm_ty); + + if rust_element_tys.len() != llvm_element_tys.len() { + return false; + } + + iter::zip(rust_element_tys, llvm_element_tys).all( + |(rust_element_ty, llvm_element_ty)| { + equate_ty(cx, rust_element_ty, llvm_element_ty) + }, + ) + } + TypeKind::Vector => { + let llvm_element_ty = cx.element_type(llvm_ty); + let element_count = cx.vector_length(llvm_ty) as u64; + + if llvm_element_ty == cx.type_bf16() { + rust_ty == cx.type_vector(cx.type_i16(), element_count) + } else if llvm_element_ty == cx.type_i1() { + let int_width = element_count.next_power_of_two().max(8); + rust_ty == cx.type_ix(int_width) + } else { + false + } + } + TypeKind::BFloat => rust_ty == cx.type_i16(), + _ => false, + } +} + +fn autocast<'ll>( + bx: &mut Builder<'_, 'll, '_>, + llfn: &'ll Value, + val: &'ll Value, + src_ty: &'ll Type, + dest_ty: &'ll Type, + is_argument: bool, +) -> &'ll Value { + let (rust_ty, llvm_ty) = if is_argument { (src_ty, dest_ty) } else { (dest_ty, src_ty) }; + + if rust_ty == llvm_ty { + return val; + } + + match bx.type_kind(llvm_ty) { + TypeKind::Struct => { + let mut ret = bx.const_poison(dest_ty); + for (idx, (src_element_ty, dest_element_ty)) in + iter::zip(bx.struct_element_types(src_ty), bx.struct_element_types(dest_ty)) + .enumerate() + { + let elt = bx.extract_value(val, idx as u64); + let casted_elt = + autocast(bx, llfn, elt, src_element_ty, dest_element_ty, is_argument); + ret = bx.insert_value(ret, casted_elt, idx as u64); + } + ret + } + TypeKind::Vector if bx.element_type(llvm_ty) == bx.type_i1() => { + let vector_length = bx.vector_length(llvm_ty) as u64; + let int_width = vector_length.next_power_of_two().max(8); + + if is_argument { + let bitcasted = bx.bitcast(val, bx.type_vector(bx.type_i1(), int_width)); + if vector_length == int_width { + bitcasted + } else { + let shuffle_mask: Vec<_> = + (0..vector_length).map(|i| bx.const_i32(i as i32)).collect(); + bx.shuffle_vector(bitcasted, bitcasted, bx.const_vector(&shuffle_mask)) + } + } else { + let val = if vector_length != int_width { + let shuffle_indices = match vector_length { + 0 => unreachable!("zero length vectors are not allowed"), + 1 => vec![0, 1, 1, 1, 1, 1, 1, 1], + 2 => vec![0, 1, 2, 3, 2, 3, 2, 3], + 3 => vec![0, 1, 2, 3, 4, 5, 3, 4], + 4.. => (0..int_width as i32).collect(), + }; + let shuffle_mask = + shuffle_indices.into_iter().map(|i| bx.const_i32(i)).collect::>(); + bx.shuffle_vector(val, bx.const_null(src_ty), bx.const_vector(&shuffle_mask)) + } else { + val + }; + + bx.bitcast(val, dest_ty) + } + } + _ => bx.bitcast(val, dest_ty), // for `bf16(xN)` <-> `u16(xN)` + } +} + +fn intrinsic_fn<'ll, 'tcx>( + bx: &Builder<'_, 'll, 'tcx>, + name: &str, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + instance: ty::Instance<'tcx>, +) -> &'ll Value { + let tcx = bx.tcx; + + let intrinsic = llvm::Intrinsic::lookup(name.as_bytes()); + + if let Some(intrinsic) = intrinsic + && intrinsic.is_target_specific() + { + let (llvm_arch, _) = name[5..].split_once('.').unwrap(); + let rust_arch = &tcx.sess.target.arch; + + if let Some(correct_llvm_arch) = llvm_arch_for(rust_arch) + && llvm_arch != correct_llvm_arch + { + tcx.dcx().emit_fatal(IntrinsicWrongArch { + name, + target_arch: rust_arch.desc(), + span: tcx.def_span(instance.def_id()), + }); + } + } + + if let Some(intrinsic) = intrinsic + && !intrinsic.is_overloaded() + { + // FIXME: also do this for overloaded intrinsics + let llfn = intrinsic.get_declaration(bx.llmod, &[]); + let fn_ty = bx.get_type_of_global(llfn); + + let rust_return_ty = fn_abi.llvm_return_type(bx); + let rust_argument_tys = fn_abi.llvm_argument_types(bx); + + let llvm_return_ty = bx.get_return_type(fn_ty); + let llvm_argument_tys = bx.func_params_types(fn_ty); + let llvm_is_variadic = bx.func_is_variadic(fn_ty); + + let is_correct_signature = fn_abi.c_variadic == llvm_is_variadic + && rust_argument_tys.len() == llvm_argument_tys.len() + && iter::once((rust_return_ty, llvm_return_ty)) + .chain(iter::zip(rust_argument_tys, llvm_argument_tys)) + .all(|(rust_ty, llvm_ty)| equate_ty(bx, rust_ty, llvm_ty)); + + if !is_correct_signature { + tcx.dcx().emit_fatal(IntrinsicSignatureMismatch { + name, + llvm_fn_ty: &format!("{fn_ty:?}"), + rust_fn_ty: &format!("{:?}", fn_abi.llvm_type(bx)), + span: tcx.def_span(instance.def_id()), + }); + } + + return llfn; + } + + // Function addresses in Rust are never significant, allowing functions to be merged. + let llfn = declare_raw_fn( + bx, + name, + fn_abi.llvm_cconv(bx), + llvm::UnnamedAddr::Global, + llvm::Visibility::Default, + fn_abi.llvm_type(bx), + ); + + // we can emit diagnostics for local crates only + if intrinsic.is_none() + && let Some(def_id) = instance.def_id().as_local() + { + let hir_id = tcx.local_def_id_to_hir_id(def_id); + let span = tcx.def_span(def_id); + + let mut new_llfn = None; + let can_upgrade = unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn) }; + + if can_upgrade { + // not all intrinsics are upgraded to some other intrinsics, most are upgraded to instruction sequences + let msg = if let Some(new_llfn) = new_llfn { + format!( + "using deprecated intrinsic `{name}`, `{}` can be used instead", + str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap() + ) + } else { + format!("using deprecated intrinsic `{name}`") + }; + tcx.node_lint(DEPRECATED_LLVM_INTRINSIC, hir_id, |d| { + d.primary_message(msg).span(span); + }); + } else { + // This is either plain wrong, or this can be caused by incompatible LLVM versions, we let the user decide + tcx.node_lint(UNKNOWN_LLVM_INTRINSIC, hir_id, |d| { + d.primary_message(format!("invalid LLVM Intrinsic `{name}`")).span(span); + }); + } + } + + llfn +} + fn catch_unwind_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, try_func: &'ll Value, diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index b11310b970d03..4d73b2ba1002a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -73,7 +73,6 @@ unsafe extern "C" { pub(crate) fn LLVMDumpModule(M: &Module); pub(crate) fn LLVMDumpValue(V: &Value); pub(crate) fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; - pub(crate) fn LLVMGetReturnType(T: &Type) -> &Type; pub(crate) fn LLVMGetParams(Fnc: &Value, params: *mut &Value); pub(crate) fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> Option<&Value>; } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 75b3e5955b78f..2e701bd862598 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -920,6 +920,9 @@ unsafe extern "C" { pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; + // Operations on non-IEEE real types + pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type; + // Operations on function types pub(crate) fn LLVMFunctionType<'a>( ReturnType: &'a Type, @@ -929,6 +932,8 @@ unsafe extern "C" { ) -> &'a Type; pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); + pub(crate) fn LLVMGetReturnType(FunctionTy: &Type) -> &Type; + pub(crate) fn LLVMIsFunctionVarArg(FunctionTy: &Type) -> Bool; // Operations on struct types pub(crate) fn LLVMStructTypeInContext<'a>( @@ -1083,12 +1088,18 @@ unsafe extern "C" { // Operations about llvm intrinsics pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint; + pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero) -> Bool; pub(crate) fn LLVMGetIntrinsicDeclaration<'a>( Mod: &'a Module, ID: NonZero, ParamTypes: *const &'a Type, ParamCount: size_t, ) -> &'a Value; + pub(crate) fn LLVMRustUpgradeIntrinsicFunction<'a>( + Fn: &'a Value, + NewFn: &mut Option<&'a Value>, + ) -> bool; + pub(crate) fn LLVMRustIsTargetIntrinsic(ID: NonZero) -> bool; // Operations on parameters pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; @@ -1604,6 +1615,9 @@ unsafe extern "C" { Packed: Bool, ); + pub(crate) fn LLVMCountStructElementTypes(StructTy: &Type) -> c_uint; + pub(crate) fn LLVMGetStructElementTypes<'a>(StructTy: &'a Type, Dest: *mut &'a Type); + pub(crate) safe fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; pub(crate) safe fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 621004e756ec1..0138b13ae1391 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -319,6 +319,14 @@ impl Intrinsic { NonZero::new(id).map(|id| Self { id }) } + pub(crate) fn is_overloaded(self) -> bool { + unsafe { LLVMIntrinsicIsOverloaded(self.id).is_true() } + } + + pub(crate) fn is_target_specific(self) -> bool { + unsafe { LLVMRustIsTargetIntrinsic(self.id) } + } + pub(crate) fn get_declaration<'ll>( self, llmod: &'ll Module, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index d8b77369a34ff..9c1186cc0ddf8 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -77,6 +77,10 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { unsafe { llvm::LLVMAddFunction(self.llmod(), name.as_ptr(), ty) } } + pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type { + unsafe { llvm::LLVMGetReturnType(ty) } + } + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { unsafe { let n_args = llvm::LLVMCountParamTypes(ty) as usize; @@ -86,6 +90,20 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { args } } + + pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool { + unsafe { llvm::LLVMIsFunctionVarArg(ty).is_true() } + } + + pub(crate) fn struct_element_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + unsafe { + let n_args = llvm::LLVMCountStructElementTypes(ty) as usize; + let mut args = Vec::with_capacity(n_args); + llvm::LLVMGetStructElementTypes(ty, args.as_mut_ptr()); + args.set_len(n_args); + args + } + } } impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn type_bool(&self) -> &'ll Type { @@ -165,6 +183,10 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { ) } } + + pub(crate) fn type_bf16(&self) -> &'ll Type { + unsafe { llvm::LLVMBFloatTypeInContext(self.llcx()) } + } } impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index 8c69cc089cafb..18cc905fecdfe 100644 --- a/compiler/rustc_lint_defs/src/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -33,6 +33,7 @@ declare_lint_pass! { DEPENDENCY_ON_UNIT_NEVER_TYPE_FALLBACK, DEPRECATED, DEPRECATED_IN_FUTURE, + DEPRECATED_LLVM_INTRINSIC, DEPRECATED_SAFE_2024, DEPRECATED_WHERE_CLAUSE_LOCATION, DUPLICATE_MACRO_ATTRIBUTES, @@ -118,6 +119,7 @@ declare_lint_pass! { UNKNOWN_CRATE_TYPES, UNKNOWN_DIAGNOSTIC_ATTRIBUTES, UNKNOWN_LINTS, + UNKNOWN_LLVM_INTRINSIC, UNNAMEABLE_TEST_ITEMS, UNNAMEABLE_TYPES, UNREACHABLE_CODE, @@ -5315,3 +5317,79 @@ declare_lint! { report_in_deps: false, }; } + +declare_lint! { + /// The `unknown_llvm_intrinsic` lint detects usage of unknown LLVM intrinsics. + /// + /// ### Example + /// + /// ```rust,compile_fail + /// #![feature(link_llvm_intrinsics, abi_unadjusted)] + /// + /// unsafe extern "unadjusted" { + /// #[link_name = "llvm.abcde"] + /// fn foo(); + /// } + /// + /// #[inline(never)] + /// pub fn main() { + /// unsafe { foo() } + /// } + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// Linking to an unknown LLVM intrinsic may cause linker errors (in general it's UB), + /// so this lint captures those undesirable scenarios. + pub UNKNOWN_LLVM_INTRINSIC, + Deny, + "detects uses of unknown LLVM intrinsics", + @feature_gate = link_llvm_intrinsics; +} + +declare_lint! { + /// The `deprecated_llvm_intrinsic` lint detects usage of deprecated LLVM intrinsics. + /// + /// ### Example + /// + /// ```rust,ignore (requires x86) + /// #![cfg(any(target_arch = "x86", target_arch = "x86_64"))] + /// #![feature(link_llvm_intrinsics, abi_unadjusted)] + /// #![deny(deprecated_llvm_intrinsic)] + /// + /// unsafe extern "unadjusted" { + /// #[link_name = "llvm.x86.addcarryx.u32"] + /// fn foo(a: u8, b: u32, c: u32, d: &mut u32) -> u8; + /// } + /// + /// #[inline(never)] + /// #[target_feature(enable = "adx")] + /// pub fn bar(a: u8, b: u32, c: u32, d: &mut u32) -> u8 { + /// unsafe { foo(a, b, c, d) } + /// } + /// ``` + /// + /// This will produce: + /// + /// ```text + /// error: Using deprecated intrinsic `llvm.x86.addcarryx.u32` + /// --> example.rs:7:5 + /// | + /// 7 | fn foo(a: u8, b: u32, c: u32, d: &mut u32) -> u8; + /// | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + /// | + /// ``` + /// + /// ### Explanation + /// + /// LLVM periodically updates its list of intrinsics. Removed intrinsics are unlikely + /// to be removed, but they may optimize less well than their new versions, so it's + /// best to use the new version. Also, some deprecated intrinsics might have buggy + /// behavior + pub DEPRECATED_LLVM_INTRINSIC, + Allow, + "detects uses of deprecated LLVM intrinsics", + @feature_gate = link_llvm_intrinsics; +} diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 02e6abf24627f..85fb9a936251b 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -9,6 +9,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DiagnosticHandler.h" @@ -1813,6 +1814,19 @@ extern "C" void LLVMRustSetNoSanitizeHWAddress(LLVMValueRef Global) { GV.setSanitizerMetadata(MD); } +extern "C" bool LLVMRustUpgradeIntrinsicFunction(LLVMValueRef Fn, + LLVMValueRef *NewFn) { + Function *F = unwrap(Fn); + Function *NewF = nullptr; + bool CanUpgrade = UpgradeIntrinsicFunction(F, NewF, false); + *NewFn = wrap(NewF); + return CanUpgrade; +} + +extern "C" bool LLVMRustIsTargetIntrinsic(unsigned ID) { + return Intrinsic::isTargetIntrinsic(ID); +} + // Statically assert that the fixed metadata kind IDs declared in // `metadata_kind.rs` match the ones actually used by LLVM. #define FIXED_MD_KIND(VARIANT, VALUE) \ diff --git a/tests/codegen-llvm/inject-autocast.rs b/tests/codegen-llvm/inject-autocast.rs new file mode 100644 index 0000000000000..fbabc48362a78 --- /dev/null +++ b/tests/codegen-llvm/inject-autocast.rs @@ -0,0 +1,94 @@ +//@ compile-flags: -C opt-level=0 +//@ only-x86_64 + +#![feature(link_llvm_intrinsics, abi_unadjusted, repr_simd, simd_ffi, portable_simd, f16)] +#![crate_type = "lib"] + +use std::simd::{f32x4, i16x8, i64x2}; + +#[repr(C, packed)] +pub struct Bar(u32, i64x2, i64x2, i64x2, i64x2, i64x2, i64x2); +// CHECK: %Bar = type <{ i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> }> + +#[repr(simd)] +pub struct f16x8([f16; 8]); + +// CHECK-LABEL: @struct_autocast +#[no_mangle] +pub unsafe fn struct_autocast(key_metadata: u32, key: i64x2) -> Bar { + extern "unadjusted" { + #[link_name = "llvm.x86.encodekey128"] + fn foo(key_metadata: u32, key: i64x2) -> Bar; + } + + // CHECK: [[A:%[0-9]+]] = call { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32 {{.*}}, <2 x i64> {{.*}}) + // CHECK: [[B:%[0-9]+]] = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[A]], 0 + // CHECK: [[C:%[0-9]+]] = insertvalue %Bar poison, i32 [[B]], 0 + // CHECK: [[D:%[0-9]+]] = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[A]], 1 + // CHECK: [[E:%[0-9]+]] = insertvalue %Bar [[C]], <2 x i64> [[D]], 1 + // CHECK: [[F:%[0-9]+]] = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[A]], 2 + // CHECK: [[G:%[0-9]+]] = insertvalue %Bar [[E]], <2 x i64> [[F]], 2 + // CHECK: [[H:%[0-9]+]] = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[A]], 3 + // CHECK: [[I:%[0-9]+]] = insertvalue %Bar [[G]], <2 x i64> [[H]], 3 + // CHECK: [[J:%[0-9]+]] = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[A]], 4 + // CHECK: [[K:%[0-9]+]] = insertvalue %Bar [[I]], <2 x i64> [[J]], 4 + // CHECK: [[L:%[0-9]+]] = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[A]], 5 + // CHECK: [[M:%[0-9]+]] = insertvalue %Bar [[K]], <2 x i64> [[L]], 5 + // CHECK: [[N:%[0-9]+]] = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[A]], 6 + // CHECK: insertvalue %Bar [[M]], <2 x i64> [[N]], 6 + foo(key_metadata, key) +} + +// CHECK-LABEL: @struct_with_i1_vector_autocast +#[no_mangle] +pub unsafe fn struct_with_i1_vector_autocast(a: i64x2, b: i64x2) -> (u8, u8) { + extern "unadjusted" { + #[link_name = "llvm.x86.avx512.vp2intersect.q.128"] + fn foo(a: i64x2, b: i64x2) -> (u8, u8); + } + + // CHECK: [[A:%[0-9]+]] = call { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64> {{.*}}, <2 x i64> {{.*}}) + // CHECK: [[B:%[0-9]+]] = extractvalue { <2 x i1>, <2 x i1> } [[A]], 0 + // CHECK: [[C:%[0-9]+]] = shufflevector <2 x i1> [[B]], <2 x i1> zeroinitializer, <8 x i32> + // CHECK: [[D:%[0-9]+]] = bitcast <8 x i1> [[C]] to i8 + // CHECK: [[E:%[0-9]+]] = insertvalue { i8, i8 } poison, i8 [[D]], 0 + // CHECK: [[F:%[0-9]+]] = extractvalue { <2 x i1>, <2 x i1> } [[A]], 1 + // CHECK: [[G:%[0-9]+]] = shufflevector <2 x i1> [[F]], <2 x i1> zeroinitializer, <8 x i32> + // CHECK: [[H:%[0-9]+]] = bitcast <8 x i1> [[G]] to i8 + // CHECK: insertvalue { i8, i8 } [[E]], i8 [[H]], 1 + foo(a, b) +} + +// CHECK-LABEL: @i1_vector_autocast +#[no_mangle] +pub unsafe fn i1_vector_autocast(a: f16x8) -> u8 { + extern "unadjusted" { + #[link_name = "llvm.x86.avx512fp16.fpclass.ph.128"] + fn foo(a: f16x8, b: i32) -> u8; + } + + // CHECK: [[A:%[0-9]+]] = call <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half> {{.*}}, i32 1) + // CHECK: bitcast <8 x i1> [[A]] to i8 + foo(a, 1) +} + +// CHECK-LABEL: @bf16_vector_autocast +#[no_mangle] +pub unsafe fn bf16_vector_autocast(a: f32x4) -> i16x8 { + extern "unadjusted" { + #[link_name = "llvm.x86.vcvtneps2bf16128"] + fn foo(a: f32x4) -> i16x8; + } + + // CHECK: [[A:%[0-9]+]] = call <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float> {{.*}}) + // CHECK: bitcast <8 x bfloat> [[A]] to <8 x i16> + foo(a) +} + +// CHECK: declare { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32, <2 x i64>) + +// CHECK: declare { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64>, <2 x i64>) + +// CHECK: declare <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half>, i32 immarg) + +// CHECK: declare <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float>) diff --git a/tests/run-make/simd-ffi/simd.rs b/tests/run-make/simd-ffi/simd.rs index 1cd961ff87e7b..3f12dabdb65ea 100644 --- a/tests/run-make/simd-ffi/simd.rs +++ b/tests/run-make/simd-ffi/simd.rs @@ -35,7 +35,7 @@ extern "C" { fn integer(a: i32x4, b: i32x4) -> i32x4; // vmaxq_s32 #[cfg(target_arch = "aarch64")] - #[link_name = "llvm.aarch64.neon.maxs.v4i32"] + #[link_name = "llvm.aarch64.neon.smax.v4i32"] fn integer(a: i32x4, b: i32x4) -> i32x4; // Use a generic LLVM intrinsic to do type checking on other platforms diff --git a/tests/ui/codegen/deprecated-llvm-intrinsic.rs b/tests/ui/codegen/deprecated-llvm-intrinsic.rs new file mode 100644 index 0000000000000..33bc5f419151a --- /dev/null +++ b/tests/ui/codegen/deprecated-llvm-intrinsic.rs @@ -0,0 +1,28 @@ +//@ add-minicore +//@ build-fail +//@ compile-flags: --target aarch64-unknown-linux-gnu +//@ needs-llvm-components: aarch64 +//@ ignore-backends: gcc +#![feature(no_core, lang_items, link_llvm_intrinsics, abi_unadjusted, repr_simd, simd_ffi)] +#![no_std] +#![no_core] +#![allow(internal_features, non_camel_case_types, improper_ctypes)] +#![crate_type = "lib"] + +extern crate minicore; +use minicore::*; + +#[repr(simd)] +pub struct i8x8([i8; 8]); + +extern "unadjusted" { + #[deny(deprecated_llvm_intrinsic)] + #[link_name = "llvm.aarch64.neon.rbit.v8i8"] + fn foo(a: i8x8) -> i8x8; + //~^ ERROR: using deprecated intrinsic `llvm.aarch64.neon.rbit.v8i8`, `llvm.bitreverse.v8i8` can be used instead +} + +#[target_feature(enable = "neon")] +pub unsafe fn bar(a: i8x8) -> i8x8 { + foo(a) +} diff --git a/tests/ui/codegen/deprecated-llvm-intrinsic.stderr b/tests/ui/codegen/deprecated-llvm-intrinsic.stderr new file mode 100644 index 0000000000000..40e4684a8ea4f --- /dev/null +++ b/tests/ui/codegen/deprecated-llvm-intrinsic.stderr @@ -0,0 +1,14 @@ +error: using deprecated intrinsic `llvm.aarch64.neon.rbit.v8i8`, `llvm.bitreverse.v8i8` can be used instead + --> $DIR/deprecated-llvm-intrinsic.rs:21:5 + | +LL | fn foo(a: i8x8) -> i8x8; + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +note: the lint level is defined here + --> $DIR/deprecated-llvm-intrinsic.rs:19:12 + | +LL | #[deny(deprecated_llvm_intrinsic)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/codegen/incorrect-arch-intrinsic.rs b/tests/ui/codegen/incorrect-arch-intrinsic.rs new file mode 100644 index 0000000000000..9576cb8f81318 --- /dev/null +++ b/tests/ui/codegen/incorrect-arch-intrinsic.rs @@ -0,0 +1,18 @@ +//@ build-fail +//@ ignore-s390x +//@ normalize-stderr: "target arch `(.*)`" -> "target arch `TARGET_ARCH`" +//@ ignore-backends: gcc + +#![feature(link_llvm_intrinsics, abi_unadjusted)] + +extern "unadjusted" { + #[link_name = "llvm.s390.sfpc"] + fn foo(a: i32); + //~^ ERROR: intrinsic `llvm.s390.sfpc` cannot be used with target arch +} + +pub fn main() { + unsafe { + foo(0); + } +} diff --git a/tests/ui/codegen/incorrect-arch-intrinsic.stderr b/tests/ui/codegen/incorrect-arch-intrinsic.stderr new file mode 100644 index 0000000000000..5b44419aa741f --- /dev/null +++ b/tests/ui/codegen/incorrect-arch-intrinsic.stderr @@ -0,0 +1,8 @@ +error: intrinsic `llvm.s390.sfpc` cannot be used with target arch `TARGET_ARCH` + --> $DIR/incorrect-arch-intrinsic.rs:10:5 + | +LL | fn foo(a: i32); + | ^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/codegen/incorrect-llvm-intrinsic-signature.rs b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.rs new file mode 100644 index 0000000000000..4b86f37f922a6 --- /dev/null +++ b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.rs @@ -0,0 +1,14 @@ +//@ build-fail +//@ ignore-backends: gcc + +#![feature(link_llvm_intrinsics, abi_unadjusted)] + +extern "unadjusted" { + #[link_name = "llvm.assume"] + fn foo(); + //~^ ERROR: intrinsic signature mismatch for `llvm.assume`: expected signature `void (i1)`, found `void ()` +} + +pub fn main() { + unsafe { foo() } +} diff --git a/tests/ui/codegen/incorrect-llvm-intrinsic-signature.stderr b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.stderr new file mode 100644 index 0000000000000..4e58e5ebdc731 --- /dev/null +++ b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.stderr @@ -0,0 +1,8 @@ +error: intrinsic signature mismatch for `llvm.assume`: expected signature `void (i1)`, found `void ()` + --> $DIR/incorrect-llvm-intrinsic-signature.rs:8:5 + | +LL | fn foo(); + | ^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/codegen/invalid-llvm-intrinsic.rs b/tests/ui/codegen/invalid-llvm-intrinsic.rs new file mode 100644 index 0000000000000..1f4c6f93660f8 --- /dev/null +++ b/tests/ui/codegen/invalid-llvm-intrinsic.rs @@ -0,0 +1,14 @@ +//@ build-fail +//@ ignore-backends: gcc + +#![feature(link_llvm_intrinsics, abi_unadjusted)] + +extern "unadjusted" { + #[link_name = "llvm.abcde"] + fn foo(); + //~^ ERROR: invalid LLVM Intrinsic `llvm.abcde` +} + +pub fn main() { + unsafe { foo() } +} diff --git a/tests/ui/codegen/invalid-llvm-intrinsic.stderr b/tests/ui/codegen/invalid-llvm-intrinsic.stderr new file mode 100644 index 0000000000000..510550e631e47 --- /dev/null +++ b/tests/ui/codegen/invalid-llvm-intrinsic.stderr @@ -0,0 +1,10 @@ +error: invalid LLVM Intrinsic `llvm.abcde` + --> $DIR/invalid-llvm-intrinsic.rs:8:5 + | +LL | fn foo(); + | ^^^^^^^^^ + | + = note: `#[deny(unknown_llvm_intrinsic)]` on by default + +error: aborting due to 1 previous error +