diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs
index 75a33394e3d2f..44ca468d4f10e 100644
--- a/src/libcore/intrinsics.rs
+++ b/src/libcore/intrinsics.rs
@@ -962,6 +962,33 @@ extern "rust-intrinsic" {
/// value is not necessarily valid to be used to actually access memory.
pub fn arith_offset<T>(dst: *const T, offset: isize) -> *const T;
+ /// Equivalent to the appropriate `llvm.memcpy.element.unordered.atomic.p0i8.p0i8.*` intrinsic, with
+ /// a size of `count` * `size_of::<T>()`, an alignment of
+ /// `min_align_of::<T>()`, and an element size of `size_of::<T>()`.
+ ///
+ /// `size_of::<T>` must be an integer power of two no larger than the
+ /// target-specific atomic access size limit.
+ #[cfg(not(stage0))]
+ pub fn atomic_element_copy_nonoverlapping_memory_unordered<T>(dst: *mut T, src: *const T, count: usize);
+
+ /// Equivalent to the appropriate `llvm.memmove.unordered.atomic.p0i8.p0i8.*` intrinsic, with
+ /// a size of `count` * `size_of::<T>()`, an alignment of
+ /// `min_align_of::<T>()`, and an element size of `size_of::<T>()`.
+ ///
+ /// `size_of::<T>` must be an integer power of two no larger than the
+ /// target-specific atomic access size limit.
+ #[cfg(not(stage0))]
+ pub fn atomic_element_copy_memory_unordered<T>(dst: *mut T, src: *const T, count: usize);
+
+ /// Equivalent to the appropriate `llvm.memset.unordered.atomic.p0i8.p0i8.*` intrinsic, with
+ /// a size of `count` * `size_of::<T>()`, an alignment of
+ /// `min_align_of::<T>()`, and an element size of `size_of::<T>()`.
+ ///
+ /// `size_of::<T>` must be an integer power of two no larger than the
+ /// target-specific atomic access size limit.
+ #[cfg(not(stage0))]
+ pub fn atomic_element_set_memory_unordered<T>(dst: *mut T, val: u8, count: usize);
+
/// Equivalent to the appropriate `llvm.memcpy.p0i8.0i8.*` intrinsic, with
/// a size of `count` * `size_of::<T>()` and an alignment of
/// `min_align_of::<T>()`
diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs
index 123fda1e215ff..6e7b213f59470 100644
--- a/src/librustc_codegen_llvm/builder.rs
+++ b/src/librustc_codegen_llvm/builder.rs
@@ -966,6 +966,48 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
self.call(llintrinsicfn, &[ptr, fill_byte, size, align, volatile], None);
}
+ fn atomic_element_unordered_memcpy(&mut self, dst: &'ll Value, dst_align: Align,
+ src: &'ll Value, src_align: Align,
+ size: &'ll Value, element_size: u32) {
+ let size = self.intcast(size, self.type_isize(), false);
+ let dst = self.pointercast(dst, self.type_i8p());
+ let src = self.pointercast(src, self.type_i8p());
+ unsafe {
+ llvm::LLVMRustBuildElementUnorderedAtomicMemCpy(self.llbuilder, dst, dst_align.bytes() as c_uint,
+ src, src_align.bytes() as c_uint, size, element_size);
+ }
+ }
+
+ fn atomic_element_unordered_memmove(&mut self, dst: &'ll Value, dst_align: Align,
+ src: &'ll Value, src_align: Align,
+ size: &'ll Value, element_size: u32) {
+ let size = self.intcast(size, self.type_isize(), false);
+ let dst = self.pointercast(dst, self.type_i8p());
+ let src = self.pointercast(src, self.type_i8p());
+ let ret_ref = unsafe {
+ llvm::LLVMRustBuildElementUnorderedAtomicMemMove(self.llbuilder,
+ dst, dst_align.bytes() as c_uint,
+ src, src_align.bytes() as c_uint,
+ size, element_size)
+ };
+ if ret_ref.is_none() {
+ bug!("llvm.memmove.element.unordered.atomic.* is not supported with LLVM prior to 7.0");
+ }
+ }
+
+ fn atomic_element_unordered_memset(&mut self, ptr: &'ll Value, fill_byte: &'ll Value,
+ size: &'ll Value, align: Align, element_size: u32) {
+ let size = self.intcast(size, self.type_isize(), false);
+ let ptr = self.pointercast(ptr, self.type_i8p());
+ let ret_ref = unsafe {
+ llvm::LLVMRustBuildElementUnorderedAtomicMemSet(self.llbuilder, ptr, fill_byte,
+ size, align.bytes() as c_uint, element_size)
+ };
+ if ret_ref.is_none() {
+ bug!("llvm.memset.element.unordered.atomic.* is not supported with LLVM prior to 7.0");
+ }
+ }
+
fn select(
&mut self, cond: &'ll Value,
then_val: &'ll Value,
diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs
index ceb08f943678b..a1dc096e49292 100644
--- a/src/librustc_codegen_llvm/intrinsic.rs
+++ b/src/librustc_codegen_llvm/intrinsic.rs
@@ -102,6 +102,13 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
let llret_ty = self.layout_of(ret_ty).llvm_type(self);
let result = PlaceRef::new_sized(llresult, fn_ty.ret.layout, fn_ty.ret.layout.align.abi);
+ let invalid_integer_monomorphization = |ty| {
+ span_invalid_monomorphization_error(tcx.sess, span,
+ &format!("invalid monomorphization of `{}` intrinsic: \
+ expected basic integer type, found `{}`", name, ty));
+ };
+
+
let simple = get_simple_intrinsic(self, name);
let llval = match name {
_ if simple.is_some() => {
@@ -503,10 +510,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
_ => bug!(),
},
None => {
- span_invalid_monomorphization_error(
- tcx.sess, span,
- &format!("invalid monomorphization of `{}` intrinsic: \
- expected basic integer type, found `{}`", name, ty));
+ invalid_integer_monomorphization(ty);
return;
}
}
@@ -548,6 +552,17 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
Err(()) => return
}
}
+ name if name.starts_with("atomic_element_") => {
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, self).is_some() {
+ atomic_element_intrinsic(self, name,
+ substs.type_at(0),
+ args);
+ return;
+ } else {
+ return invalid_integer_monomorphization(ty);
+ }
+ }
// This requires that atomic intrinsics follow a specific naming pattern:
// "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
name if name.starts_with("atomic_") => {
@@ -582,12 +597,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
_ => self.sess().fatal("Atomic intrinsic not in correct format"),
};
- let invalid_monomorphization = |ty| {
- span_invalid_monomorphization_error(tcx.sess, span,
- &format!("invalid monomorphization of `{}` intrinsic: \
- expected basic integer type, found `{}`", name, ty));
- };
-
match split[1] {
"cxchg" | "cxchgweak" => {
let ty = substs.type_at(0);
@@ -610,7 +619,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
self.store(success, dest.llval, dest.align);
return;
} else {
- return invalid_monomorphization(ty);
+ return invalid_integer_monomorphization(ty);
}
}
@@ -620,7 +629,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
let size = self.size_of(ty);
self.atomic_load(args[0].immediate(), order, size)
} else {
- return invalid_monomorphization(ty);
+ return invalid_integer_monomorphization(ty);
}
}
@@ -636,7 +645,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
);
return;
} else {
- return invalid_monomorphization(ty);
+ return invalid_integer_monomorphization(ty);
}
}
@@ -676,7 +685,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
order
)
} else {
- return invalid_monomorphization(ty);
+ return invalid_integer_monomorphization(ty);
}
}
}
@@ -754,6 +763,54 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
}
}
+fn atomic_element_intrinsic(
+ bx: &mut Builder<'a, 'll, 'tcx>,
+ name: &str,
+ ty: Ty<'tcx>,
+ args: &[OperandRef<'tcx, &'ll Value>],
+) {
+ let (element_size, align) = bx.size_and_align_of(ty);
+ let element_size = element_size.bytes();
+ assert!(element_size <= u32::max_value() as u64);
+
+ let size = bx.mul(bx.const_usize(element_size), args[2].immediate());
+
+ match name {
+ "atomic_element_copy_nonoverlapping_memory_unordered" => {
+ bx.atomic_element_unordered_memcpy(
+ args[0].immediate(),
+ align,
+ args[1].immediate(),
+ align,
+ size,
+ element_size as u32
+ );
+ }
+ "atomic_element_copy_memory_unordered" => {
+ bx.atomic_element_unordered_memmove(
+ args[0].immediate(),
+ align,
+ args[1].immediate(),
+ align,
+ size,
+ element_size as u32
+ );
+ }
+ "atomic_element_set_memory_unordered" => {
+ bx.atomic_element_unordered_memset(
+ args[0].immediate(),
+ args[1].immediate(),
+ size,
+ align,
+ element_size as u32
+ );
+ }
+ _ => {
+ bug!("unknown intrinsic '{}'", name);
+ }
+ }
+}
+
fn copy_intrinsic(
bx: &mut Builder<'a, 'll, 'tcx>,
allow_overlap: bool,
@@ -777,6 +834,7 @@ fn copy_intrinsic(
}
}
+
fn memset_intrinsic(
bx: &mut Builder<'a, 'll, 'tcx>,
volatile: bool,
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
index 2ad6d9c053a20..924358d77ee78 100644
--- a/src/librustc_codegen_llvm/llvm/ffi.rs
+++ b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -1154,6 +1154,28 @@ extern "C" {
Size: &'a Value,
IsVolatile: bool)
-> &'a Value;
+ pub fn LLVMRustBuildElementUnorderedAtomicMemCpy(B: &Builder<'a>,
+ Dst: &'a Value,
+ DstAlign: c_uint,
+ Src: &'a Value,
+ SrcAlign: c_uint,
+ Size: &'a Value,
+ ElementSize: u32)
+ -> &'a Value;
+ pub fn LLVMRustBuildElementUnorderedAtomicMemMove(B: &Builder<'a>,
+ Dst: &'a Value,
+ DstAlign: c_uint,
+ Src: &'a Value,
+ SrcAlign: c_uint,
+ Size: &'a Value,
+ ElementSize: u32) -> Option<&'a Value>;
+ pub fn LLVMRustBuildElementUnorderedAtomicMemSet(B: &Builder<'a>,
+ Ptr: &'a Value,
+ Val: &'a Value,
+ Size: &'a Value,
+ Align: c_uint,
+ ElementSize: u32) -> Option<&'a Value>;
+
pub fn LLVMRustBuildMemMove(B: &Builder<'a>,
Dst: &'a Value,
DstAlign: c_uint,
diff --git a/src/librustc_codegen_ssa/traits/builder.rs b/src/librustc_codegen_ssa/traits/builder.rs
index 48142fc9fa9f4..12775492d8237 100644
--- a/src/librustc_codegen_ssa/traits/builder.rs
+++ b/src/librustc_codegen_ssa/traits/builder.rs
@@ -190,6 +190,33 @@ pub trait BuilderMethods<'a, 'tcx: 'a>:
flags: MemFlags,
);
+ fn atomic_element_unordered_memcpy(
+ &mut self,
+ dst: Self::Value,
+ dst_align: Align,
+ src: Self::Value,
+ src_align: Align,
+ size: Self::Value,
+ element_size: u32,
+ );
+ fn atomic_element_unordered_memmove(
+ &mut self,
+ dst: Self::Value,
+ dst_align: Align,
+ src: Self::Value,
+ src_align: Align,
+ size: Self::Value,
+ element_size: u32,
+ );
+ fn atomic_element_unordered_memset(
+ &mut self,
+ ptr: Self::Value,
+ fill_byte: Self::Value,
+ size: Self::Value,
+ align: Align,
+ element_size: u32,
+ );
+
fn select(
&mut self,
cond: Self::Value,
diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs
index 40c60caffa42d..2a26098677ec5 100644
--- a/src/librustc_typeck/check/intrinsic.rs
+++ b/src/librustc_typeck/check/intrinsic.rs
@@ -92,7 +92,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
})
};
- let (n_tps, inputs, output, unsafety) = if name.starts_with("atomic_") {
+ let (n_tps, inputs, output, unsafety) = if name.starts_with("atomic_") && ! name.starts_with("atomic_element_") {
let split : Vec<&str> = name.split('_').collect();
assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format");
@@ -197,7 +197,8 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
],
tcx.mk_unit())
}
- "volatile_copy_memory" | "volatile_copy_nonoverlapping_memory" => {
+ "volatile_copy_memory" | "volatile_copy_nonoverlapping_memory" |
+ "atomic_element_copy_memory_unordered" | "atomic_element_copy_nonoverlapping_memory_unordered" => {
(1,
vec![
tcx.mk_ptr(ty::TypeAndMut {
@@ -212,7 +213,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
],
tcx.mk_unit())
}
- "write_bytes" | "volatile_set_memory" => {
+ "write_bytes" | "volatile_set_memory" | "atomic_element_set_memory_unordered" => {
(1,
vec![
tcx.mk_ptr(ty::TypeAndMut {
diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp
index a00417a362927..5fe8b68ee9547 100644
--- a/src/rustllvm/RustWrapper.cpp
+++ b/src/rustllvm/RustWrapper.cpp
@@ -1268,6 +1268,42 @@ extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B,
#endif
}
+extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemCpy(LLVMBuilderRef B,
+ LLVMValueRef Dst, unsigned DstAlign,
+ LLVMValueRef Src, unsigned SrcAlign,
+ LLVMValueRef Size, uint32_t ElementSize) {
+ return wrap(unwrap(B)->CreateElementUnorderedAtomicMemCpy(
+ unwrap(Dst), DstAlign,
+ unwrap(Src), SrcAlign,
+ unwrap(Size), ElementSize));
+}
+
+extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemMove(LLVMBuilderRef B,
+ LLVMValueRef Dst, unsigned DstAlign,
+ LLVMValueRef Src, unsigned SrcAlign,
+ LLVMValueRef Size, uint32_t ElementSize) {
+#if LLVM_VERSION_GE(7, 0)
+ return wrap(unwrap(B)->CreateElementUnorderedAtomicMemMove(
+ unwrap(Dst), DstAlign,
+ unwrap(Src), SrcAlign,
+ unwrap(Size), ElementSize));
+#else
+ return nullptr;
+#endif
+}
+
+extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemSet(LLVMBuilderRef B,
+ LLVMValueRef Ptr, LLVMValueRef Val,
+ LLVMValueRef Size, unsigned Align, uint32_t ElementSize) {
+#if LLVM_VERSION_GE(7, 0)
+ return wrap(unwrap(B)->CreateElementUnorderedAtomicMemSet(
+ unwrap(Ptr), unwrap(Val),
+ unwrap(Size), Align, ElementSize));
+#else
+ return nullptr;
+#endif
+}
+
extern "C" LLVMValueRef
LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args,
unsigned NumArgs, LLVMBasicBlockRef Then,
diff --git a/src/test/run-pass/intrinsics/intrinsic-atomics.rs b/src/test/run-pass/intrinsics/intrinsic-atomics.rs
index 608cf3dee5230..f2179395f1212 100644
--- a/src/test/run-pass/intrinsics/intrinsic-atomics.rs
+++ b/src/test/run-pass/intrinsics/intrinsic-atomics.rs
@@ -29,6 +29,10 @@ mod rusti {
pub fn atomic_xsub<T>(dst: *mut T, src: T) -> T;
pub fn atomic_xsub_acq<T>(dst: *mut T, src: T) -> T;
pub fn atomic_xsub_rel<T>(dst: *mut T, src: T) -> T;
+
+ pub fn atomic_element_copy_nonoverlapping_memory_unordered<T>(dst: *mut T, src: *const T, count: usize);
+ pub fn atomic_element_copy_memory_unordered<T>(dst: *mut T, src: *const T, count: usize);
+ pub fn atomic_element_set_memory_unordered<T>(dst: *mut T, val: u8, count: usize);
}
}
@@ -99,5 +103,16 @@ pub fn main() {
}
}
assert_eq!(*x, 3);
+
+ let mut mem = [0; 4];
+ let src = [1, 2, 3, 4];
+ let dst = &mut mem[0] as *mut _;
+
+ rusti::atomic_element_copy_nonoverlapping_memory_unordered(dst, &src[0], 4);
+ assert_eq!(mem, src);
+ rusti::atomic_element_copy_memory_unordered(dst, &mem[1], 2);
+ assert_eq!(mem, [2, 3, 3, 4]);
+ rusti::atomic_element_set_memory_unordered(dst, 1, 3);
+ assert_eq!(mem, [1, 1, 1, 4]);
}
}