diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 8001725c438b8..a5d6c1a9f28a8 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -2,6 +2,7 @@ //! looking at their MIR. Intrinsics/functions supported here are shared by CTFE //! and miri. +mod atomic; mod simd; use std::assert_matches; @@ -20,9 +21,9 @@ use tracing::trace; use super::memory::MemoryKind; use super::util::ensure_monomorphic_enough; use super::{ - AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer, - PointerArithmetic, Projectable, Provenance, Scalar, err_ub_format, err_unsup_format, interp_ok, - throw_inval, throw_ub, throw_ub_format, + AllocId, AtomicRmwOp, CheckInAllocMsg, ImmTy, Immediate, InterpCx, InterpResult, Machine, OpTy, + PlaceTy, Pointer, PointerArithmetic, Projectable, Provenance, Scalar, err_ub_format, + err_unsup_format, interp_ok, throw_inval, throw_ub, throw_ub_format, }; use crate::interpret::{MPlaceTy, Writeable}; @@ -177,6 +178,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let instance_args = instance.args; let intrinsic_name = self.tcx.item_name(instance.def_id()); + if intrinsic_name.as_str().starts_with("atomic_") { + return self.eval_atomic_intrinsic(intrinsic_name, instance_args, args, dest, ret); + } if intrinsic_name.as_str().starts_with("simd_") { return self.eval_simd_intrinsic(intrinsic_name, instance_args, args, dest, ret); } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/atomic.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/atomic.rs new file mode 100644 index 0000000000000..32967addafe12 --- /dev/null +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/atomic.rs @@ -0,0 +1,131 @@ +use rustc_middle::mir::BinOp; +use rustc_middle::{mir, span_bug, ty}; +use rustc_span::{Symbol, sym}; +use tracing::trace; + +use super::{ + AtomicRmwOp, Immediate, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Scalar, interp_ok, +}; + +impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { + /// Returns `true` if emulation happened. + /// Here we implement the intrinsics that are common to all CTFE instances; individual machines can add their own + /// intrinsic handling. + pub fn eval_atomic_intrinsic( + &mut self, + intrinsic_name: Symbol, + generic_args: ty::GenericArgsRef<'tcx>, + args: &[OpTy<'tcx, M::Provenance>], + dest: &PlaceTy<'tcx, M::Provenance>, + ret: Option, + ) -> InterpResult<'tcx, bool> { + let get_ord_at = |i: usize| { + let ordering = generic_args.const_at(i).to_value(); + ordering.to_branch()[0].to_value().to_leaf().to_atomic_ordering() + }; + + match intrinsic_name { + sym::atomic_load => { + let ord = get_ord_at(1); + let [ptr] = args else { span_bug!(self.cur_span(), "invalid `atomic_load` call") }; + + let place = self.deref_pointer(ptr)?; + let val = M::atomic_load(self, &place, ord)?; + self.write_scalar(val, dest)?; + } + sym::atomic_store => { + let ord = get_ord_at(1); + let [ptr, val] = args else { + span_bug!(self.cur_span(), "invalid `atomic_store` call") + }; + + let place = self.deref_pointer(ptr)?; + let val = self.read_immediate(val)?; + M::atomic_store(self, &place, &val, ord)?; + } + sym::atomic_or + | sym::atomic_xor + | sym::atomic_and + | sym::atomic_nand + | sym::atomic_xadd + | sym::atomic_xsub + | sym::atomic_min + | sym::atomic_umin + | sym::atomic_max + | sym::atomic_umax + | sym::atomic_xchg => { + let num_ty_generics = match intrinsic_name { + sym::atomic_min + | sym::atomic_umin + | sym::atomic_max + | sym::atomic_umax + | sym::atomic_xchg => 1, + _ => 2, + }; + let ord = get_ord_at(num_ty_generics); + let [ptr, operand] = args else { + span_bug!(self.cur_span(), "invalid `{intrinsic_name}` call") + }; + + let place = self.deref_pointer(ptr)?; + let operand = self.read_immediate(operand)?; + + let op = match intrinsic_name { + sym::atomic_or => AtomicRmwOp::MirOp { op: BinOp::BitOr, neg: false }, + sym::atomic_xor => AtomicRmwOp::MirOp { op: BinOp::BitXor, neg: false }, + sym::atomic_and => AtomicRmwOp::MirOp { op: BinOp::BitAnd, neg: false }, + sym::atomic_nand => AtomicRmwOp::MirOp { op: BinOp::BitAnd, neg: true }, + sym::atomic_xadd => AtomicRmwOp::MirOp { op: BinOp::Add, neg: false }, + sym::atomic_xsub => AtomicRmwOp::MirOp { op: BinOp::Sub, neg: false }, + sym::atomic_min => AtomicRmwOp::Min, + sym::atomic_umin => AtomicRmwOp::Min, + sym::atomic_max => AtomicRmwOp::Max, + sym::atomic_umax => AtomicRmwOp::Max, + sym::atomic_xchg => AtomicRmwOp::Swap, + _ => unreachable!(), + }; + + let res = M::atomic_rmw(self, &place, op, &operand, ord)?; + self.write_scalar(res, dest)?; + } + sym::atomic_cxchg | sym::atomic_cxchgweak => { + let success_ord = get_ord_at(1); + let failure_ord = get_ord_at(2); + let [ptr, expected_old, new] = args else { + span_bug!(self.cur_span(), "invalid `{intrinsic_name}` call") + }; + + let place = self.deref_pointer(ptr)?; + let expected_old = self.read_immediate(expected_old)?; + let new = self.read_immediate(new)?; + + let (actual_old, success) = M::atomic_compare_exchange( + self, + &place, + &expected_old, + &new, + /* can_fail_spuriously */ intrinsic_name == sym::atomic_cxchgweak, + success_ord, + failure_ord, + )?; + let res = Immediate::ScalarPair(actual_old, Scalar::from_bool(success)); + self.write_immediate(res, dest)?; + } + sym::atomic_fence | sym::atomic_singlethreadfence => { + let ord = get_ord_at(0); + let [] = args else { + span_bug!(self.cur_span(), "invalid `{intrinsic_name}` call") + }; + + M::atomic_fence(self, ord, intrinsic_name == sym::atomic_singlethreadfence)?; + } + + // Unsupported intrinsic: skip the return_to_block below. + _ => return interp_ok(false), + } + + trace!("{:?}", self.dump_place(&dest.clone().into())); + self.return_to_block(ret)?; + interp_ok(true) + } +} diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 40251cbc8155b..0528fee35031c 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -9,16 +9,17 @@ use std::hash::Hash; use rustc_abi::{Align, Size}; use rustc_apfloat::{Float, FloatConvert}; use rustc_middle::query::TyCtxtAt; -use rustc_middle::ty::Ty; use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::{AtomicOrdering, Ty}; use rustc_middle::{mir, ty}; use rustc_span::def_id::DefId; use rustc_target::callconv::FnAbi; use super::{ - AllocBytes, AllocId, AllocKind, AllocRange, Allocation, CTFE_ALLOC_SALT, ConstAllocation, - CtfeProvenance, EnteredTraceSpan, FnArg, Frame, ImmTy, InterpCx, InterpResult, MPlaceTy, - MemoryKind, Misalignment, OpTy, PlaceTy, Pointer, Provenance, RangeSet, interp_ok, throw_unsup, + AllocBytes, AllocId, AllocKind, AllocRange, Allocation, AtomicRmwOp, CTFE_ALLOC_SALT, + ConstAllocation, CtfeProvenance, EnteredTraceSpan, FnArg, Frame, ImmTy, InterpCx, InterpResult, + MPlaceTy, MemoryKind, Misalignment, OpTy, PlaceTy, Pointer, Provenance, RangeSet, Scalar, + interp_ok, throw_unsup, }; /// Data returned by [`Machine::after_stack_pop`], and consumed by @@ -281,8 +282,6 @@ pub trait Machine<'tcx>: Sized { ) -> InterpResult<'tcx>; /// Called for all binary operations where the LHS has pointer type. - /// - /// Returns a (value, overflowed) pair if the operation succeeded fn binary_ptr_op( ecx: &InterpCx<'tcx, Self>, bin_op: mir::BinOp, @@ -315,7 +314,46 @@ pub trait Machine<'tcx>: Sized { } /// Determines whether the `fmuladd` intrinsics fuse the multiply-add or use separate operations. - fn float_fuse_mul_add(_ecx: &InterpCx<'tcx, Self>) -> bool; + fn float_fuse_mul_add(ecx: &InterpCx<'tcx, Self>) -> bool; + + fn atomic_load( + ecx: &InterpCx<'tcx, Self>, + place: &MPlaceTy<'tcx, Self::Provenance>, + ordering: AtomicOrdering, + ) -> InterpResult<'tcx, Scalar>; + + fn atomic_store( + ecx: &mut InterpCx<'tcx, Self>, + place: &MPlaceTy<'tcx, Self::Provenance>, + val: &ImmTy<'tcx, Self::Provenance>, + ordering: AtomicOrdering, + ) -> InterpResult<'tcx>; + + /// Returns the old value. + fn atomic_rmw( + ecx: &mut InterpCx<'tcx, Self>, + place: &MPlaceTy<'tcx, Self::Provenance>, + op: AtomicRmwOp, + operand: &ImmTy<'tcx, Self::Provenance>, + ordering: AtomicOrdering, + ) -> InterpResult<'tcx, Scalar>; + + /// Returns a pair of the old value and a boolean indicating whether the update happened. + fn atomic_compare_exchange( + ecx: &mut InterpCx<'tcx, Self>, + place: &MPlaceTy<'tcx, Self::Provenance>, + expected_old: &ImmTy<'tcx, Self::Provenance>, + new: &ImmTy<'tcx, Self::Provenance>, + can_fail_spuriously: bool, + success_ordering: AtomicOrdering, + failure_ordering: AtomicOrdering, + ) -> InterpResult<'tcx, (Scalar, bool)>; + + fn atomic_fence( + ecx: &InterpCx<'tcx, Self>, + ordering: AtomicOrdering, + singlethread: bool, + ) -> InterpResult<'tcx>; /// Called before a basic block terminator is executed. #[inline] @@ -708,6 +746,70 @@ pub macro compile_time_machine(<$tcx: lifetime>) { true } + #[inline(always)] + fn atomic_load( + ecx: &InterpCx<$tcx, Self>, + place: &MPlaceTy<$tcx, Self::Provenance>, + _ordering: AtomicOrdering, + ) -> InterpResult<$tcx, Scalar> { + // Compile-time machines are single-threaded so this is like a regular load. + ecx.read_scalar(place) + } + + #[inline(always)] + fn atomic_store( + ecx: &mut InterpCx<$tcx, Self>, + place: &MPlaceTy<$tcx, Self::Provenance>, + val: &ImmTy<$tcx, Self::Provenance>, + _ordering: AtomicOrdering, + ) -> InterpResult<$tcx> { + // Compile-time machines are single-threaded so this is like a regular store. + ecx.write_scalar(val.to_scalar(), place) + } + + fn atomic_rmw( + ecx: &mut InterpCx<$tcx, Self>, + place: &MPlaceTy<$tcx, Self::Provenance>, + op: AtomicRmwOp, + operand: &ImmTy<$tcx, Self::Provenance>, + _ordering: AtomicOrdering, + ) -> InterpResult<$tcx, Scalar> { + // Compile-time machines are single-threaded so we ignore the ordering. + let old_val = ecx.read_immediate(place)?; + let new_val = ecx.atomic_rmw_op(op, &old_val, operand)?; + ecx.write_immediate(*new_val, place)?; + interp_ok(old_val.to_scalar()) + } + + fn atomic_compare_exchange( + ecx: &mut InterpCx<$tcx, Self>, + place: &MPlaceTy<$tcx, Self::Provenance>, + expected_old: &ImmTy<$tcx, Self::Provenance>, + new: &ImmTy<$tcx, Self::Provenance>, + _can_fail_spuriously: bool, + _success_ordering: AtomicOrdering, + _failure_ordering: AtomicOrdering, + ) -> InterpResult<$tcx, (Scalar, bool)> { + // Compile-time machines are single-threaded so we ignore the ordering. + // They are also deterministic so we do not fail spuriously. + let actual_old = ecx.read_immediate(place)?; + let eq = ecx.binary_op(mir::BinOp::Eq, &actual_old, expected_old)?.to_scalar().to_bool()?; + if eq { + ecx.write_immediate(**new, place)?; + } + interp_ok((actual_old.to_scalar(), eq)) + } + + #[inline(always)] + fn atomic_fence( + _ecx: &InterpCx<$tcx, Self>, + _ordering: AtomicOrdering, + _singlethread: bool, + ) -> InterpResult<$tcx> { + // Compile-time machines are single-threaded so this is a NOP. + interp_ok(()) + } + #[inline(always)] fn adjust_global_allocation<'b>( _ecx: &InterpCx<$tcx, Self>, diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs index d52c1f9306444..6e66599fc15c8 100644 --- a/compiler/rustc_const_eval/src/interpret/mod.rs +++ b/compiler/rustc_const_eval/src/interpret/mod.rs @@ -35,6 +35,7 @@ pub use self::machine::{ pub use self::memory::{AllocInfo, AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind}; use self::operand::Operand; pub use self::operand::{ImmTy, Immediate, OpTy}; +pub use self::operator::AtomicRmwOp; pub use self::place::{MPlaceTy, MemPlaceMeta, PlaceTy, Writeable}; use self::place::{MemPlace, Place}; pub use self::projection::{OffsetMode, Projectable}; diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index ca8c096d3ab41..95e513cc97b4e 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -10,6 +10,19 @@ use tracing::trace; use super::{ImmTy, InterpCx, Machine, MemPlaceMeta, interp_ok, throw_ub}; +/// Describes an atomic RMW operation. +pub enum AtomicRmwOp { + MirOp { + op: mir::BinOp, + /// Indicates whether the result of the operation should be negated (`UnOp::Not`, must be a + /// boolean/integer-typed operation). + neg: bool, + }, + Max, + Min, + Swap, +} + impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn three_way_compare(&self, lhs: T, rhs: T) -> ImmTy<'tcx, M::Provenance> { let res = Ord::cmp(&lhs, &rhs); @@ -504,4 +517,27 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } } + + pub fn atomic_rmw_op( + &self, + op: AtomicRmwOp, + left: &ImmTy<'tcx, M::Provenance>, + right: &ImmTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { + interp_ok(match op { + AtomicRmwOp::MirOp { op, neg } => { + let val = self.binary_op(op, &left, right)?; + if neg { self.unary_op(mir::UnOp::Not, &val)? } else { val } + } + AtomicRmwOp::Max => { + let lt = self.binary_op(mir::BinOp::Lt, &left, right)?.to_scalar().to_bool()?; + if lt { right } else { &left }.clone() + } + AtomicRmwOp::Min => { + let lt = self.binary_op(mir::BinOp::Lt, &left, right)?.to_scalar().to_bool()?; + if lt { &left } else { right }.clone() + } + AtomicRmwOp::Swap => right.clone(), + }) + } } diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 7210d850c864a..87963bcce8ebb 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -95,7 +95,7 @@ pub enum AtomicOrdering { /// For example, [`AtomicBool::compare_exchange`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_cxchg< +pub const unsafe fn atomic_cxchg< T: Copy, const ORD_SUCC: AtomicOrdering, const ORD_FAIL: AtomicOrdering, @@ -113,7 +113,7 @@ pub unsafe fn atomic_cxchg< /// For example, [`AtomicBool::compare_exchange_weak`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_cxchgweak< +pub const unsafe fn atomic_cxchgweak< T: Copy, const ORD_SUCC: AtomicOrdering, const ORD_FAIL: AtomicOrdering, @@ -130,7 +130,7 @@ pub unsafe fn atomic_cxchgweak< /// [`atomic`] types via the `load` method. For example, [`AtomicBool::load`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_load(src: *const T) -> T; +pub const unsafe fn atomic_load(src: *const T) -> T; /// Stores the value at the specified memory location. /// `T` must be an integer or pointer type. @@ -139,7 +139,7 @@ pub unsafe fn atomic_load(src: *const T) -> /// [`atomic`] types via the `store` method. For example, [`AtomicBool::store`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_store(dst: *mut T, val: T); +pub const unsafe fn atomic_store(dst: *mut T, val: T); /// Stores the value at the specified memory location, returning the old value. /// `T` must be an integer or pointer type. @@ -148,7 +148,7 @@ pub unsafe fn atomic_store(dst: *mut T, val: /// [`atomic`] types via the `swap` method. For example, [`AtomicBool::swap`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_xchg(dst: *mut T, src: T) -> T; +pub const unsafe fn atomic_xchg(dst: *mut T, src: T) -> T; /// Adds to the current value, returning the previous value. /// `T` must be an integer or pointer type. @@ -158,7 +158,10 @@ pub unsafe fn atomic_xchg(dst: *mut T, src: /// [`atomic`] types via the `fetch_add` method. For example, [`AtomicIsize::fetch_add`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_xadd(dst: *mut T, src: U) -> T; +pub const unsafe fn atomic_xadd( + dst: *mut T, + src: U, +) -> T; /// Subtract from the current value, returning the previous value. /// `T` must be an integer or pointer type. @@ -168,7 +171,10 @@ pub unsafe fn atomic_xadd(dst: *mut /// [`atomic`] types via the `fetch_sub` method. For example, [`AtomicIsize::fetch_sub`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_xsub(dst: *mut T, src: U) -> T; +pub const unsafe fn atomic_xsub( + dst: *mut T, + src: U, +) -> T; /// Bitwise and with the current value, returning the previous value. /// `T` must be an integer or pointer type. @@ -178,7 +184,10 @@ pub unsafe fn atomic_xsub(dst: *mut /// [`atomic`] types via the `fetch_and` method. For example, [`AtomicBool::fetch_and`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_and(dst: *mut T, src: U) -> T; +pub const unsafe fn atomic_and( + dst: *mut T, + src: U, +) -> T; /// Bitwise nand with the current value, returning the previous value. /// `T` must be an integer or pointer type. @@ -188,7 +197,10 @@ pub unsafe fn atomic_and(dst: *mut /// [`AtomicBool`] type via the `fetch_nand` method. For example, [`AtomicBool::fetch_nand`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_nand(dst: *mut T, src: U) -> T; +pub const unsafe fn atomic_nand( + dst: *mut T, + src: U, +) -> T; /// Bitwise or with the current value, returning the previous value. /// `T` must be an integer or pointer type. @@ -198,7 +210,10 @@ pub unsafe fn atomic_nand(dst: *mut /// [`atomic`] types via the `fetch_or` method. For example, [`AtomicBool::fetch_or`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_or(dst: *mut T, src: U) -> T; +pub const unsafe fn atomic_or( + dst: *mut T, + src: U, +) -> T; /// Bitwise xor with the current value, returning the previous value. /// `T` must be an integer or pointer type. @@ -208,7 +223,10 @@ pub unsafe fn atomic_or(dst: *mut T /// [`atomic`] types via the `fetch_xor` method. For example, [`AtomicBool::fetch_xor`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_xor(dst: *mut T, src: U) -> T; +pub const unsafe fn atomic_xor( + dst: *mut T, + src: U, +) -> T; /// Maximum with the current value using a signed comparison. /// `T` must be a signed integer type. @@ -217,7 +235,7 @@ pub unsafe fn atomic_xor(dst: *mut /// [`atomic`] signed integer types via the `fetch_max` method. For example, [`AtomicI32::fetch_max`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_max(dst: *mut T, src: T) -> T; +pub const unsafe fn atomic_max(dst: *mut T, src: T) -> T; /// Minimum with the current value using a signed comparison. /// `T` must be a signed integer type. @@ -226,7 +244,7 @@ pub unsafe fn atomic_max(dst: *mut T, src: T /// [`atomic`] signed integer types via the `fetch_min` method. For example, [`AtomicI32::fetch_min`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_min(dst: *mut T, src: T) -> T; +pub const unsafe fn atomic_min(dst: *mut T, src: T) -> T; /// Minimum with the current value using an unsigned comparison. /// `T` must be an unsigned integer type. @@ -235,7 +253,7 @@ pub unsafe fn atomic_min(dst: *mut T, src: T /// [`atomic`] unsigned integer types via the `fetch_min` method. For example, [`AtomicU32::fetch_min`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_umin(dst: *mut T, src: T) -> T; +pub const unsafe fn atomic_umin(dst: *mut T, src: T) -> T; /// Maximum with the current value using an unsigned comparison. /// `T` must be an unsigned integer type. @@ -244,7 +262,7 @@ pub unsafe fn atomic_umin(dst: *mut T, src: /// [`atomic`] unsigned integer types via the `fetch_max` method. For example, [`AtomicU32::fetch_max`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_umax(dst: *mut T, src: T) -> T; +pub const unsafe fn atomic_umax(dst: *mut T, src: T) -> T; /// An atomic fence. /// @@ -252,7 +270,7 @@ pub unsafe fn atomic_umax(dst: *mut T, src: /// [`atomic::fence`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_fence(); +pub const unsafe fn atomic_fence(); /// An atomic fence for synchronization within a single thread. /// @@ -260,7 +278,7 @@ pub unsafe fn atomic_fence(); /// [`atomic::compiler_fence`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_singlethreadfence(); +pub const unsafe fn atomic_singlethreadfence(); /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction /// for the given address if supported; otherwise, it is a no-op. diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 1a767d8092bda..71502f8d918e7 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -622,7 +622,8 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "atomic_access", since = "1.15.0")] - pub fn get_mut(&mut self) -> &mut bool { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut(&mut self) -> &mut bool { // SAFETY: the mutable reference guarantees unique ownership. unsafe { &mut *self.as_ptr() } } @@ -642,7 +643,8 @@ impl AtomicBool { #[inline] #[cfg(target_has_atomic_primitive_alignment = "8")] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn from_mut(v: &mut bool) -> &mut Self { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut(v: &mut bool) -> &mut Self { // SAFETY: the mutable reference guarantees unique ownership, and // alignment of both `bool` and `Self` is 1. unsafe { &mut *(v as *mut bool as *mut Self) } @@ -676,7 +678,8 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn get_mut_slice(this: &mut [Self]) -> &mut [bool] { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut_slice(this: &mut [Self]) -> &mut [bool] { // SAFETY: the mutable reference guarantees unique ownership. unsafe { &mut *(this as *mut [Self] as *mut [bool]) } } @@ -700,7 +703,8 @@ impl AtomicBool { #[inline] #[cfg(target_has_atomic_primitive_alignment = "8")] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn from_mut_slice(v: &mut [bool]) -> &mut [Self] { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut_slice(v: &mut [bool]) -> &mut [Self] { // SAFETY: the mutable reference guarantees unique ownership, and // alignment of both `bool` and `Self` is 1. unsafe { &mut *(v as *mut [bool] as *mut [Self]) } @@ -750,8 +754,9 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub fn load(&self, order: Ordering) -> bool { + pub const fn load(&self, order: Ordering) -> bool { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. unsafe { atomic_load(self.v.get().cast::(), order) != 0 } @@ -778,9 +783,10 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn store(&self, val: bool, order: Ordering) { + pub const fn store(&self, val: bool, order: Ordering) { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. unsafe { @@ -810,10 +816,11 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn swap(&self, val: bool, order: Ordering) -> bool { + pub const fn swap(&self, val: bool, order: Ordering) -> bool { if EMULATE_ATOMIC_BOOL { if val { self.fetch_or(true, order) } else { self.fetch_and(false, order) } } else { @@ -874,6 +881,7 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[deprecated( since = "1.50.0", note = "Use `compare_exchange` or `compare_exchange_weak` instead" @@ -881,7 +889,7 @@ impl AtomicBool { #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { + pub const fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { Ok(x) => x, Err(x) => x, @@ -939,11 +947,12 @@ impl AtomicBool { /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[doc(alias = "compare_and_swap")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange( + pub const fn compare_exchange( &self, current: bool, new: bool, @@ -1041,11 +1050,12 @@ impl AtomicBool { /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[doc(alias = "compare_and_swap")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange_weak( + pub const fn compare_exchange_weak( &self, current: bool, new: bool, @@ -1105,10 +1115,11 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_and(&self, val: bool, order: Ordering) -> bool { + pub const fn fetch_and(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_and(self.v.get().cast::(), val as u8, order) != 0 } } @@ -1148,10 +1159,11 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_nand(&self, val: bool, order: Ordering) -> bool { + pub const fn fetch_nand(&self, val: bool, order: Ordering) -> bool { // We can't use atomic_nand here because it can result in a bool with // an invalid value. This happens because the atomic operation is done // with an 8-bit integer internally, which would set the upper 7 bits. @@ -1201,10 +1213,11 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_or(&self, val: bool, order: Ordering) -> bool { + pub const fn fetch_or(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_or(self.v.get().cast::(), val as u8, order) != 0 } } @@ -1243,10 +1256,11 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_xor(&self, val: bool, order: Ordering) -> bool { + pub const fn fetch_xor(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_xor(self.v.get().cast::(), val as u8, order) != 0 } } @@ -1281,10 +1295,11 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "atomic_bool_fetch_not", since = "1.81.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_not(&self, order: Ordering) -> bool { + pub const fn fetch_not(&self, order: Ordering) -> bool { self.fetch_xor(true, order) } @@ -1585,7 +1600,8 @@ impl AtomicPtr { /// ``` #[inline] #[stable(feature = "atomic_access", since = "1.15.0")] - pub fn get_mut(&mut self) -> &mut *mut T { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut(&mut self) -> &mut *mut T { // SAFETY: // `Atomic` is essentially a transparent wrapper around `T`. unsafe { &mut *self.as_ptr() } @@ -1610,7 +1626,8 @@ impl AtomicPtr { #[inline] #[cfg(target_has_atomic_primitive_alignment = "ptr")] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn from_mut(v: &mut *mut T) -> &mut Self { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut(v: &mut *mut T) -> &mut Self { let [] = [(); align_of::>() - align_of::<*mut ()>()]; // SAFETY: // - the mutable reference guarantees unique ownership. @@ -1653,7 +1670,8 @@ impl AtomicPtr { /// ``` #[inline] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn get_mut_slice(this: &mut [Self]) -> &mut [*mut T] { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut_slice(this: &mut [Self]) -> &mut [*mut T] { // SAFETY: the mutable reference guarantees unique ownership. unsafe { &mut *(this as *mut [Self] as *mut [*mut T]) } } @@ -1687,7 +1705,8 @@ impl AtomicPtr { #[inline] #[cfg(target_has_atomic_primitive_alignment = "ptr")] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn from_mut_slice(v: &mut [*mut T]) -> &mut [Self] { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut_slice(v: &mut [*mut T]) -> &mut [Self] { // SAFETY: // - the mutable reference guarantees unique ownership. // - the alignment of `*mut T` and `Self` is the same on all platforms @@ -1739,8 +1758,9 @@ impl AtomicPtr { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub fn load(&self, order: Ordering) -> *mut T { + pub const fn load(&self, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_load(self.as_ptr(), order) } } @@ -1768,9 +1788,10 @@ impl AtomicPtr { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn store(&self, ptr: *mut T, order: Ordering) { + pub const fn store(&self, ptr: *mut T, order: Ordering) { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_store(self.as_ptr(), ptr, order); @@ -1801,10 +1822,11 @@ impl AtomicPtr { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(target_has_atomic = "ptr")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { + pub const fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_swap(self.as_ptr(), ptr, order) } } @@ -2724,7 +2746,8 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable_access] - pub fn get_mut(&mut self) -> &mut $int_type { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut(&mut self) -> &mut $int_type { // SAFETY: // `Atomic` is essentially a transparent wrapper around `T`. unsafe { &mut *self.as_ptr() } @@ -2755,7 +2778,8 @@ macro_rules! atomic_int { #[inline] #[cfg(any($cfg_align, doc))] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn from_mut(v: &mut $int_type) -> &mut Self { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut(v: &mut $int_type) -> &mut Self { let [] = [(); align_of::() - align_of::<$int_type>()]; // SAFETY: // - the mutable reference guarantees unique ownership. @@ -2795,7 +2819,8 @@ macro_rules! atomic_int { /// ``` #[inline] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn get_mut_slice(this: &mut [Self]) -> &mut [$int_type] { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut_slice(this: &mut [Self]) -> &mut [$int_type] { // SAFETY: the mutable reference guarantees unique ownership. unsafe { &mut *(this as *mut [Self] as *mut [$int_type]) } } @@ -2830,7 +2855,8 @@ macro_rules! atomic_int { #[inline] #[cfg(any($cfg_align, doc))] #[stable(feature = "atomic_from_mut", since = "1.98.0")] - pub fn from_mut_slice(v: &mut [$int_type]) -> &mut [Self] { + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut_slice(v: &mut [$int_type]) -> &mut [Self] { let [] = [(); align_of::() - align_of::<$int_type>()]; // SAFETY: // - the mutable reference guarantees unique ownership. @@ -2883,8 +2909,9 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub fn load(&self, order: Ordering) -> $int_type { + pub const fn load(&self, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_load(self.as_ptr(), order) } } @@ -2911,9 +2938,10 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn store(&self, val: $int_type, order: Ordering) { + pub const fn store(&self, val: $int_type, order: Ordering) { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_store(self.as_ptr(), val, order); } } @@ -2940,10 +2968,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn swap(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn swap(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_swap(self.as_ptr(), val, order) } } @@ -3002,6 +3031,7 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[deprecated( since = "1.50.0", note = "Use `compare_exchange` or `compare_exchange_weak` instead") @@ -3009,7 +3039,7 @@ macro_rules! atomic_int { #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_and_swap(&self, + pub const fn compare_and_swap(&self, current: $int_type, new: $int_type, order: Ordering) -> $int_type { @@ -3076,10 +3106,11 @@ macro_rules! atomic_int { /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] #[$stable_cxchg] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange(&self, + pub const fn compare_exchange(&self, current: $int_type, new: $int_type, success: Ordering, @@ -3141,10 +3172,11 @@ macro_rules! atomic_int { /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] #[$stable_cxchg] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange_weak(&self, + pub const fn compare_exchange_weak(&self, current: $int_type, new: $int_type, success: Ordering, @@ -3179,10 +3211,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_add(self.as_ptr(), val, order) } } @@ -3211,10 +3244,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_sub(self.as_ptr(), val, order) } } @@ -3246,10 +3280,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_and(self.as_ptr(), val, order) } } @@ -3281,10 +3316,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable_nand] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_nand(self.as_ptr(), val, order) } } @@ -3316,10 +3352,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_or(self.as_ptr(), val, order) } } @@ -3351,10 +3388,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_xor(self.as_ptr(), val, order) } } @@ -3554,10 +3592,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { $max_fn(self.as_ptr(), val, order) } } @@ -3603,10 +3642,11 @@ macro_rules! atomic_int { /// ``` #[inline] #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg(any($cfg_cas, doc))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { + pub const fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { // SAFETY: data races are prevented by atomic intrinsics. unsafe { $min_fn(self.as_ptr(), val, order) } } @@ -3918,7 +3958,7 @@ atomic_int_ptr_sized! { #[inline] #[cfg(target_has_atomic)] -fn strongest_failure_ordering(order: Ordering) -> Ordering { +const fn strongest_failure_ordering(order: Ordering) -> Ordering { match order { Release => Relaxed, Relaxed => Relaxed, @@ -3930,7 +3970,8 @@ fn strongest_failure_ordering(order: Ordering) -> Ordering { #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_store(dst: *mut T, val: T, order: Ordering) { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_store(dst: *mut T, val: T, order: Ordering) { // SAFETY: the caller must uphold the safety contract for `atomic_store`. unsafe { match order { @@ -3945,7 +3986,8 @@ unsafe fn atomic_store(dst: *mut T, val: T, order: Ordering) { #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_load(dst: *const T, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_load(dst: *const T, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_load`. unsafe { match order { @@ -3961,7 +4003,8 @@ unsafe fn atomic_load(dst: *const T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_swap(dst: *mut T, val: T, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_swap(dst: *mut T, val: T, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_swap`. unsafe { match order { @@ -3978,7 +4021,8 @@ unsafe fn atomic_swap(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_add(dst: *mut T, val: U, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_add(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_add`. unsafe { match order { @@ -3995,7 +4039,8 @@ unsafe fn atomic_add(dst: *mut T, val: U, order: Ordering) -> #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_sub(dst: *mut T, val: U, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_sub(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_sub`. unsafe { match order { @@ -4014,7 +4059,8 @@ unsafe fn atomic_sub(dst: *mut T, val: U, order: Ordering) -> #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[unstable(feature = "core_intrinsics", issue = "none")] #[doc(hidden)] -pub unsafe fn atomic_compare_exchange( +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +pub const unsafe fn atomic_compare_exchange( dst: *mut T, old: T, new: T, @@ -4079,7 +4125,8 @@ pub unsafe fn atomic_compare_exchange( #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_compare_exchange_weak( +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_compare_exchange_weak( dst: *mut T, old: T, new: T, @@ -4144,7 +4191,8 @@ unsafe fn atomic_compare_exchange_weak( #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_and(dst: *mut T, val: U, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_and(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_and` unsafe { match order { @@ -4160,7 +4208,8 @@ unsafe fn atomic_and(dst: *mut T, val: U, order: Ordering) -> #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_nand(dst: *mut T, val: U, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_nand(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_nand` unsafe { match order { @@ -4176,7 +4225,8 @@ unsafe fn atomic_nand(dst: *mut T, val: U, order: Ordering) -> #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_or(dst: *mut T, val: U, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_or(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_or` unsafe { match order { @@ -4192,7 +4242,8 @@ unsafe fn atomic_or(dst: *mut T, val: U, order: Ordering) -> T #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_xor(dst: *mut T, val: U, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_xor(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_xor` unsafe { match order { @@ -4209,7 +4260,8 @@ unsafe fn atomic_xor(dst: *mut T, val: U, order: Ordering) -> #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_max(dst: *mut T, val: T, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_max(dst: *mut T, val: T, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_max` unsafe { match order { @@ -4226,7 +4278,8 @@ unsafe fn atomic_max(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_min(dst: *mut T, val: T, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_min(dst: *mut T, val: T, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_min` unsafe { match order { @@ -4243,7 +4296,8 @@ unsafe fn atomic_min(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_umax(dst: *mut T, val: T, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_umax(dst: *mut T, val: T, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_umax` unsafe { match order { @@ -4260,7 +4314,8 @@ unsafe fn atomic_umax(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] +const unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_umin` unsafe { match order { @@ -4427,10 +4482,11 @@ unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[rustc_diagnostic_item = "fence"] #[doc(alias = "atomic_thread_fence")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -pub fn fence(order: Ordering) { +pub const fn fence(order: Ordering) { // SAFETY: using an atomic fence is safe. unsafe { match order { @@ -4511,10 +4567,11 @@ pub fn fence(order: Ordering) { /// ``` #[inline] #[stable(feature = "compiler_fences", since = "1.21.0")] +#[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[rustc_diagnostic_item = "compiler_fence"] #[doc(alias = "atomic_signal_fence")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -pub fn compiler_fence(order: Ordering) { +pub const fn compiler_fence(order: Ordering) { // SAFETY: using an atomic fence is safe. unsafe { match order { diff --git a/src/tools/miri/src/concurrency/data_race.rs b/src/tools/miri/src/concurrency/data_race.rs index 1cdfb1d21b8cb..b3192174a07fb 100644 --- a/src/tools/miri/src/concurrency/data_race.rs +++ b/src/tools/miri/src/concurrency/data_race.rs @@ -51,14 +51,13 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_index::{Idx, IndexVec}; use rustc_log::tracing; use rustc_middle::mir; -use rustc_middle::ty::Ty; +use rustc_middle::ty::{AtomicOrdering, Ty}; use rustc_span::Span; use super::vector_clock::{VClock, VTimestamp, VectorIdx}; use super::weak_memory::EvalContextExt as _; use crate::concurrency::GlobalDataRaceHandler; use crate::diagnostics::RacingOp; -use crate::intrinsics::AtomicRmwOp; use crate::*; pub type AllocState = VClockAlloc; @@ -73,6 +72,19 @@ pub enum AtomicRwOrd { SeqCst, } +impl AtomicRwOrd { + pub fn from(ordering: AtomicOrdering) -> Self { + use AtomicRwOrd::*; + match ordering { + AtomicOrdering::Relaxed => Relaxed, + AtomicOrdering::Release => Release, + AtomicOrdering::Acquire => Acquire, + AtomicOrdering::AcqRel => AcqRel, + AtomicOrdering::SeqCst => SeqCst, + } + } +} + /// Valid atomic read orderings, subset of atomic::Ordering. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicReadOrd { @@ -81,6 +93,18 @@ pub enum AtomicReadOrd { SeqCst, } +impl AtomicReadOrd { + pub fn from(ordering: AtomicOrdering) -> Self { + use AtomicReadOrd::*; + match ordering { + AtomicOrdering::Relaxed => Relaxed, + AtomicOrdering::Acquire => Acquire, + AtomicOrdering::SeqCst => SeqCst, + _ => panic!("invalid atomic read ordering: {ordering:?}"), + } + } +} + /// Valid atomic write orderings, subset of atomic::Ordering. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicWriteOrd { @@ -89,6 +113,18 @@ pub enum AtomicWriteOrd { SeqCst, } +impl AtomicWriteOrd { + pub fn from(ordering: AtomicOrdering) -> Self { + use AtomicWriteOrd::*; + match ordering { + AtomicOrdering::Relaxed => Relaxed, + AtomicOrdering::Release => Release, + AtomicOrdering::SeqCst => SeqCst, + _ => panic!("invalid atomic write ordering: {ordering:?}"), + } + } +} + /// Valid atomic fence orderings, subset of atomic::Ordering. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicFenceOrd { @@ -98,6 +134,19 @@ pub enum AtomicFenceOrd { SeqCst, } +impl AtomicFenceOrd { + pub fn from(ordering: AtomicOrdering) -> Self { + use AtomicFenceOrd::*; + match ordering { + AtomicOrdering::Acquire => Acquire, + AtomicOrdering::Release => Release, + AtomicOrdering::SeqCst => SeqCst, + AtomicOrdering::AcqRel => AcqRel, + _ => panic!("invalid atomic fence ordering: {ordering:?}"), + } + } +} + /// The current set of vector clocks describing the state /// of a thread, contains the happens-before clock and /// additional metadata to model atomic fence operations. @@ -789,13 +838,13 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { } /// Perform an atomic RMW operation on a memory location. - fn atomic_rmw_op_immediate( + fn atomic_rmw( &mut self, place: &MPlaceTy<'tcx>, rhs: &ImmTy<'tcx>, atomic_op: AtomicRmwOp, ord: AtomicRwOrd, - ) -> InterpResult<'tcx, ImmTy<'tcx>> { + ) -> InterpResult<'tcx, Scalar> { let this = self.eval_context_mut(); this.atomic_access_check(place, AtomicAccessType::Rmw)?; @@ -803,7 +852,7 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { // Inform GenMC about the atomic rmw operation. if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() { - let (old_val, new_val) = genmc_ctx.atomic_rmw_op( + let (old_val, new_val) = genmc_ctx.atomic_rmw( this, place.ptr().addr(), place.layout.size, @@ -816,69 +865,17 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { if let Some(new_val) = new_val { this.allow_data_races_mut(|this| this.write_scalar(new_val, place))?; } - return interp_ok(ImmTy::from_scalar(old_val, old.layout)); + return interp_ok(old_val); } trace!("atomic_rmw({:?}, {} bytes)", place.ptr(), place.layout.size.bytes()); - let val = match atomic_op { - AtomicRmwOp::MirOp { op, neg } => { - let val = this.binary_op(op, &old, rhs)?; - if neg { this.unary_op(mir::UnOp::Not, &val)? } else { val } - } - AtomicRmwOp::Max => { - let lt = this.binary_op(mir::BinOp::Lt, &old, rhs)?.to_scalar().to_bool()?; - if lt { rhs } else { &old }.clone() - } - AtomicRmwOp::Min => { - let lt = this.binary_op(mir::BinOp::Lt, &old, rhs)?.to_scalar().to_bool()?; - if lt { &old } else { rhs }.clone() - } - }; + let val = this.atomic_rmw_op(atomic_op, &old, rhs)?; this.allow_data_races_mut(|this| this.write_immediate(*val, place))?; this.validate_atomic_rmw(place, ord)?; this.buffered_atomic_rmw(val.to_scalar(), place, ord, old.to_scalar())?; - interp_ok(old) - } - - /// Perform an atomic exchange with a memory place and a new - /// scalar value, the old value is returned. - fn atomic_exchange_scalar( - &mut self, - place: &MPlaceTy<'tcx>, - new: Scalar, - atomic: AtomicRwOrd, - ) -> InterpResult<'tcx, Scalar> { - let this = self.eval_context_mut(); - this.atomic_access_check(place, AtomicAccessType::Rmw)?; - - let old = this.allow_data_races_mut(|this| this.read_scalar(place))?; - - // Inform GenMC about the atomic atomic exchange. - if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() { - let (old_val, new_val) = genmc_ctx.atomic_exchange( - this, - place.ptr().addr(), - place.layout.size, - new, - atomic, - old, - )?; - // The store might be the latest store in coherence order (determined by GenMC). - // If it is, we need to update the value in Miri's memory: - if let Some(new_val) = new_val { - this.allow_data_races_mut(|this| this.write_scalar(new_val, place))?; - } - return interp_ok(old_val); - } - - trace!("atomic_exchange_scalar({:?}, {} bytes)", place.ptr(), place.layout.size.bytes()); - - this.allow_data_races_mut(|this| this.write_scalar(new, place))?; - this.validate_atomic_rmw(place, atomic)?; - this.buffered_atomic_rmw(new, place, atomic, old)?; - interp_ok(old) + interp_ok(old.to_scalar()) } /// Perform an atomic compare and exchange at a given memory location. @@ -887,7 +884,7 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { /// then we treat it as a "compare_exchange_weak" operation, and /// some portion of the time fail even when the values are actually /// identical. - fn atomic_compare_exchange_scalar( + fn atomic_compare_exchange( &mut self, place: &MPlaceTy<'tcx>, expect_old: &ImmTy<'tcx>, @@ -895,7 +892,7 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { success: AtomicRwOrd, fail: AtomicReadOrd, can_fail_spuriously: bool, - ) -> InterpResult<'tcx, Immediate> { + ) -> InterpResult<'tcx, (Scalar, bool)> { let this = self.eval_context_mut(); this.atomic_access_check(place, AtomicAccessType::Rmw)?; @@ -920,7 +917,7 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { if let Some(new_value) = new_value { this.allow_data_races_mut(|this| this.write_scalar(new_value, place))?; } - return interp_ok(Immediate::ScalarPair(old_value, Scalar::from_bool(cmpxchg_success))); + return interp_ok((old_value, cmpxchg_success)); } // `binary_op` will bail if either of them is not a scalar. @@ -934,7 +931,7 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { } else { true }; - let res = Immediate::ScalarPair(old.to_scalar(), Scalar::from_bool(cmpxchg_success)); + let res = (old.to_scalar(), cmpxchg_success); trace!( "atomic_compare_exchange_scalar({:?}, {} bytes, success = {})", @@ -964,10 +961,10 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> { } /// Update the data-race detector for an atomic fence on the current thread. - fn atomic_fence(&mut self, atomic: AtomicFenceOrd) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); + fn atomic_fence(&self, atomic: AtomicFenceOrd) -> InterpResult<'tcx> { + let this = self.eval_context_ref(); let machine = &this.machine; - match &this.machine.data_race { + match &machine.data_race { GlobalDataRaceHandler::None => interp_ok(()), GlobalDataRaceHandler::Vclocks(data_race) => data_race.atomic_fence(machine, atomic), GlobalDataRaceHandler::Genmc(genmc_ctx) => genmc_ctx.atomic_fence(machine, atomic), diff --git a/src/tools/miri/src/concurrency/genmc/dummy.rs b/src/tools/miri/src/concurrency/genmc/dummy.rs index d643bda28c34e..0ab369f25770a 100644 --- a/src/tools/miri/src/concurrency/genmc/dummy.rs +++ b/src/tools/miri/src/concurrency/genmc/dummy.rs @@ -3,10 +3,9 @@ use rustc_const_eval::interpret::{AllocId, InterpCx, InterpResult}; pub use self::intercept::EvalContextExt as GenmcEvalContextExt; pub use self::run::run_genmc_mode; -use crate::intrinsics::AtomicRmwOp; use crate::{ - AtomicFenceOrd, AtomicReadOrd, AtomicRwOrd, AtomicWriteOrd, MemoryKind, MiriMachine, OpTy, - Scalar, ThreadId, ThreadManager, VisitProvenance, VisitWith, + AtomicFenceOrd, AtomicReadOrd, AtomicRmwOp, AtomicRwOrd, AtomicWriteOrd, MemoryKind, + MiriMachine, OpTy, Scalar, ThreadId, ThreadManager, VisitProvenance, VisitWith, }; #[derive(Clone, Copy, Debug)] @@ -106,7 +105,7 @@ impl GenmcCtx { unreachable!() } - pub(crate) fn atomic_rmw_op<'tcx>( + pub(crate) fn atomic_rmw<'tcx>( &self, _ecx: &InterpCx<'tcx, MiriMachine<'tcx>>, _address: Size, @@ -120,18 +119,6 @@ impl GenmcCtx { unreachable!() } - pub(crate) fn atomic_exchange<'tcx>( - &self, - _ecx: &InterpCx<'tcx, MiriMachine<'tcx>>, - _address: Size, - _size: Size, - _rhs_scalar: Scalar, - _ordering: AtomicRwOrd, - _old_value: Scalar, - ) -> InterpResult<'tcx, (Scalar, Option)> { - unreachable!() - } - pub(crate) fn atomic_compare_exchange<'tcx>( &self, _ecx: &InterpCx<'tcx, MiriMachine<'tcx>>, diff --git a/src/tools/miri/src/concurrency/genmc/helper.rs b/src/tools/miri/src/concurrency/genmc/helper.rs index e20f001185198..1870a8e4e2f3a 100644 --- a/src/tools/miri/src/concurrency/genmc/helper.rs +++ b/src/tools/miri/src/concurrency/genmc/helper.rs @@ -7,7 +7,6 @@ use rustc_middle::{mir, throw_ub_format}; use super::GenmcScalar; use crate::alloc_addresses::EvalContextExt as _; -use crate::intrinsics::AtomicRmwOp; use crate::*; /// Maximum size memory access in bytes that GenMC supports. @@ -177,6 +176,7 @@ pub(super) fn to_genmc_rmw_op(atomic_op: AtomicRmwOp, is_signed: bool) -> RMWBin (AtomicRmwOp::Max, true) => RMWBinOp::Max, (AtomicRmwOp::Min, false) => RMWBinOp::UMin, (AtomicRmwOp::Max, false) => RMWBinOp::UMax, + (AtomicRmwOp::Swap, _is_signed) => RMWBinOp::Xchg, (AtomicRmwOp::MirOp { op, neg }, _is_signed) => match (op, neg) { (mir::BinOp::Add, false) => RMWBinOp::Add, diff --git a/src/tools/miri/src/concurrency/genmc/mod.rs b/src/tools/miri/src/concurrency/genmc/mod.rs index e3a76363df7e8..dfd7d4b52678d 100644 --- a/src/tools/miri/src/concurrency/genmc/mod.rs +++ b/src/tools/miri/src/concurrency/genmc/mod.rs @@ -20,7 +20,6 @@ use self::helper::{ use self::run::GenmcMode; use self::thread_id_map::ThreadIdMap; use crate::diagnostics::SpanDedupDiagnostic; -use crate::intrinsics::AtomicRmwOp; use crate::*; mod config; @@ -325,7 +324,7 @@ impl GenmcCtx { /// Returns `(old_val, Option)`. `new_val` might not be the latest write in coherence order, which is indicated by `None`. /// /// `old_value` is the value that a non-atomic load would read here, or `None` if the memory is uninitialized. - pub(crate) fn atomic_rmw_op<'tcx>( + pub(crate) fn atomic_rmw<'tcx>( &self, ecx: &InterpCx<'tcx, MiriMachine<'tcx>>, address: Size, @@ -347,29 +346,6 @@ impl GenmcCtx { ) } - /// Returns `(old_val, Option)`. `new_val` might not be the latest write in coherence order, which is indicated by `None`. - /// - /// `old_value` is the value that a non-atomic load would read here, or `None` if the memory is uninitialized. - pub(crate) fn atomic_exchange<'tcx>( - &self, - ecx: &InterpCx<'tcx, MiriMachine<'tcx>>, - address: Size, - size: Size, - rhs_scalar: Scalar, - ordering: AtomicRwOrd, - old_value: Scalar, - ) -> InterpResult<'tcx, (Scalar, Option)> { - self.handle_atomic_rmw_op( - ecx, - address, - size, - ordering, - /* genmc_rmw_op */ RMWBinOp::Xchg, - scalar_to_genmc_scalar(ecx, self, rhs_scalar)?, - scalar_to_genmc_scalar(ecx, self, old_value)?, - ) - } - /// Inform GenMC about an atomic compare-exchange operation. /// /// Returns the old value read by the compare exchange, optionally the value that Miri should write back to its memory, and whether the compare-exchange was a success or not. diff --git a/src/tools/miri/src/concurrency/sync.rs b/src/tools/miri/src/concurrency/sync.rs index de2e0d8e23924..dda3306d0208d 100644 --- a/src/tools/miri/src/concurrency/sync.rs +++ b/src/tools/miri/src/concurrency/sync.rs @@ -353,17 +353,15 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // There's no sync object there yet. Create one, and try a CAS for uninit_val to init_val. let meta_obj = new_meta_obj(this)?; - let (old_init, success) = this - .atomic_compare_exchange_scalar( - &init_field, - &ImmTy::from_scalar(Scalar::from_u8(uninit_val), this.machine.layouts.u8), - Scalar::from_u8(init_val), - AtomicRwOrd::Relaxed, - AtomicReadOrd::Relaxed, - /* can_fail_spuriously */ false, - )? - .to_scalar_pair(); - if !success.to_bool()? { + let (old_init, success) = this.atomic_compare_exchange( + &init_field, + &ImmTy::from_scalar(Scalar::from_u8(uninit_val), this.machine.layouts.u8), + Scalar::from_u8(init_val), + AtomicRwOrd::Relaxed, + AtomicReadOrd::Relaxed, + /* can_fail_spuriously */ false, + )?; + if !success { // This can happen for the macOS lock if it is already marked as initialized. assert_eq!( old_init.to_u8()?, diff --git a/src/tools/miri/src/intrinsics/atomic.rs b/src/tools/miri/src/intrinsics/atomic.rs deleted file mode 100644 index 481b6716c70c7..0000000000000 --- a/src/tools/miri/src/intrinsics/atomic.rs +++ /dev/null @@ -1,347 +0,0 @@ -use rustc_middle::mir::BinOp; -use rustc_middle::ty::AtomicOrdering; -use rustc_middle::{mir, ty}; - -use super::check_intrinsic_arg_count; -use crate::*; - -pub enum AtomicRmwOp { - MirOp { - op: mir::BinOp, - /// Indicates whether the result of the operation should be negated (`UnOp::Not`, must be a - /// boolean-typed operation). - neg: bool, - }, - Max, - Min, -} - -impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} -pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { - /// Calls the atomic intrinsic `intrinsic`; the `atomic_` prefix has already been removed. - /// Returns `Ok(true)` if the intrinsic was handled. - fn emulate_atomic_intrinsic( - &mut self, - intrinsic_name: &str, - generic_args: ty::GenericArgsRef<'tcx>, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { - let this = self.eval_context_mut(); - - let get_ord_at = |i: usize| { - let ordering = generic_args.const_at(i).to_value(); - ordering.to_branch()[0].to_value().to_leaf().to_atomic_ordering() - }; - - fn read_ord(ord: AtomicOrdering) -> AtomicReadOrd { - match ord { - AtomicOrdering::SeqCst => AtomicReadOrd::SeqCst, - AtomicOrdering::Acquire => AtomicReadOrd::Acquire, - AtomicOrdering::Relaxed => AtomicReadOrd::Relaxed, - _ => panic!("invalid read ordering `{ord:?}`"), - } - } - - fn write_ord(ord: AtomicOrdering) -> AtomicWriteOrd { - match ord { - AtomicOrdering::SeqCst => AtomicWriteOrd::SeqCst, - AtomicOrdering::Release => AtomicWriteOrd::Release, - AtomicOrdering::Relaxed => AtomicWriteOrd::Relaxed, - _ => panic!("invalid write ordering `{ord:?}`"), - } - } - - fn rw_ord(ord: AtomicOrdering) -> AtomicRwOrd { - match ord { - AtomicOrdering::SeqCst => AtomicRwOrd::SeqCst, - AtomicOrdering::AcqRel => AtomicRwOrd::AcqRel, - AtomicOrdering::Acquire => AtomicRwOrd::Acquire, - AtomicOrdering::Release => AtomicRwOrd::Release, - AtomicOrdering::Relaxed => AtomicRwOrd::Relaxed, - } - } - - fn fence_ord(ord: AtomicOrdering) -> AtomicFenceOrd { - match ord { - AtomicOrdering::SeqCst => AtomicFenceOrd::SeqCst, - AtomicOrdering::AcqRel => AtomicFenceOrd::AcqRel, - AtomicOrdering::Acquire => AtomicFenceOrd::Acquire, - AtomicOrdering::Release => AtomicFenceOrd::Release, - _ => panic!("invalid fence ordering `{ord:?}`"), - } - } - - match intrinsic_name { - "load" => { - let ord = get_ord_at(1); - this.atomic_load(args, dest, read_ord(ord))?; - } - - "store" => { - let ord = get_ord_at(1); - this.atomic_store(args, write_ord(ord))? - } - - "fence" => { - let ord = get_ord_at(0); - this.atomic_fence_intrinsic(args, fence_ord(ord))? - } - "singlethreadfence" => { - let ord = get_ord_at(0); - this.compiler_fence_intrinsic(args, fence_ord(ord))?; - } - - "xchg" => { - let ord = get_ord_at(1); - this.atomic_exchange(args, dest, rw_ord(ord))?; - } - "cxchg" => { - let ord1 = get_ord_at(1); - let ord2 = get_ord_at(2); - this.atomic_compare_exchange(args, dest, rw_ord(ord1), read_ord(ord2))?; - } - "cxchgweak" => { - let ord1 = get_ord_at(1); - let ord2 = get_ord_at(2); - this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1), read_ord(ord2))?; - } - - "or" => { - let ord = get_ord_at(2); - this.atomic_rmw_op( - args, - dest, - AtomicRmwOp::MirOp { op: BinOp::BitOr, neg: false }, - rw_ord(ord), - )?; - } - "xor" => { - let ord = get_ord_at(2); - this.atomic_rmw_op( - args, - dest, - AtomicRmwOp::MirOp { op: BinOp::BitXor, neg: false }, - rw_ord(ord), - )?; - } - "and" => { - let ord = get_ord_at(2); - this.atomic_rmw_op( - args, - dest, - AtomicRmwOp::MirOp { op: BinOp::BitAnd, neg: false }, - rw_ord(ord), - )?; - } - "nand" => { - let ord = get_ord_at(2); - this.atomic_rmw_op( - args, - dest, - AtomicRmwOp::MirOp { op: BinOp::BitAnd, neg: true }, - rw_ord(ord), - )?; - } - "xadd" => { - let ord = get_ord_at(2); - this.atomic_rmw_op( - args, - dest, - AtomicRmwOp::MirOp { op: BinOp::Add, neg: false }, - rw_ord(ord), - )?; - } - "xsub" => { - let ord = get_ord_at(2); - this.atomic_rmw_op( - args, - dest, - AtomicRmwOp::MirOp { op: BinOp::Sub, neg: false }, - rw_ord(ord), - )?; - } - "min" => { - let ord = get_ord_at(1); - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Int(_))); - this.atomic_rmw_op(args, dest, AtomicRmwOp::Min, rw_ord(ord))?; - } - "umin" => { - let ord = get_ord_at(1); - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_))); - this.atomic_rmw_op(args, dest, AtomicRmwOp::Min, rw_ord(ord))?; - } - "max" => { - let ord = get_ord_at(1); - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Int(_))); - this.atomic_rmw_op(args, dest, AtomicRmwOp::Max, rw_ord(ord))?; - } - "umax" => { - let ord = get_ord_at(1); - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_))); - this.atomic_rmw_op(args, dest, AtomicRmwOp::Max, rw_ord(ord))?; - } - - _ => return interp_ok(EmulateItemResult::NotSupported), - } - interp_ok(EmulateItemResult::NeedsReturn) - } -} - -impl<'tcx> EvalContextPrivExt<'tcx> for MiriInterpCx<'tcx> {} -trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> { - fn atomic_load( - &mut self, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - atomic: AtomicReadOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place] = check_intrinsic_arg_count(args)?; - let place = this.deref_pointer(place)?; - - // Perform atomic load. - let val = this.read_scalar_atomic(&place, atomic)?; - // Perform regular store. - this.write_scalar(val, dest)?; - interp_ok(()) - } - - fn atomic_store(&mut self, args: &[OpTy<'tcx>], atomic: AtomicWriteOrd) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, val] = check_intrinsic_arg_count(args)?; - let place = this.deref_pointer(place)?; - - // Perform regular load. - let val = this.read_scalar(val)?; - // Perform atomic store. - this.write_scalar_atomic(val, &place, atomic)?; - interp_ok(()) - } - - fn compiler_fence_intrinsic( - &mut self, - args: &[OpTy<'tcx>], - atomic: AtomicFenceOrd, - ) -> InterpResult<'tcx> { - let [] = check_intrinsic_arg_count(args)?; - let _ = atomic; - // FIXME, FIXME(GenMC): compiler fences are currently ignored (also ignored in GenMC mode) - interp_ok(()) - } - - fn atomic_fence_intrinsic( - &mut self, - args: &[OpTy<'tcx>], - atomic: AtomicFenceOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - let [] = check_intrinsic_arg_count(args)?; - this.atomic_fence(atomic)?; - interp_ok(()) - } - - fn atomic_rmw_op( - &mut self, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - atomic_op: AtomicRmwOp, - ord: AtomicRwOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, rhs] = check_intrinsic_arg_count(args)?; - let place = this.deref_pointer(place)?; - let rhs = this.read_immediate(rhs)?; - - // The LHS can be a pointer, the RHS must be an integer. - if !(place.layout.ty.is_integral() || place.layout.ty.is_raw_ptr()) - || !rhs.layout.ty.is_integral() - { - span_bug!( - this.cur_span(), - "atomic arithmetic operations only work on integer and raw pointer types", - ); - } - - let old = this.atomic_rmw_op_immediate(&place, &rhs, atomic_op, ord)?; - this.write_immediate(*old, dest)?; // old value is returned - interp_ok(()) - } - - fn atomic_exchange( - &mut self, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - atomic: AtomicRwOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, new] = check_intrinsic_arg_count(args)?; - let place = this.deref_pointer(place)?; - let new = this.read_scalar(new)?; - - let old = this.atomic_exchange_scalar(&place, new, atomic)?; - this.write_scalar(old, dest)?; // old value is returned - interp_ok(()) - } - - fn atomic_compare_exchange_impl( - &mut self, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - success: AtomicRwOrd, - fail: AtomicReadOrd, - can_fail_spuriously: bool, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, expect_old, new] = check_intrinsic_arg_count(args)?; - let place = this.deref_pointer(place)?; - let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()` - let new = this.read_scalar(new)?; - - let old = this.atomic_compare_exchange_scalar( - &place, - &expect_old, - new, - success, - fail, - can_fail_spuriously, - )?; - - // Return old value. - this.write_immediate(old, dest)?; - interp_ok(()) - } - - fn atomic_compare_exchange( - &mut self, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - success: AtomicRwOrd, - fail: AtomicReadOrd, - ) -> InterpResult<'tcx> { - self.atomic_compare_exchange_impl(args, dest, success, fail, false) - } - - fn atomic_compare_exchange_weak( - &mut self, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - success: AtomicRwOrd, - fail: AtomicReadOrd, - ) -> InterpResult<'tcx> { - self.atomic_compare_exchange_impl(args, dest, success, fail, true) - } -} diff --git a/src/tools/miri/src/intrinsics/mod.rs b/src/tools/miri/src/intrinsics/mod.rs index 858e035a8c1f7..0f55009db790b 100644 --- a/src/tools/miri/src/intrinsics/mod.rs +++ b/src/tools/miri/src/intrinsics/mod.rs @@ -1,14 +1,11 @@ #![warn(clippy::arithmetic_side_effects)] mod aarch64; -mod atomic; mod loongarch; mod math; mod simd; mod x86; -pub use self::atomic::AtomicRmwOp; - #[rustfmt::skip] // prevent `use` reordering use rand::RngExt; use rustc_abi::{Endian, Size}; @@ -16,7 +13,6 @@ use rustc_middle::{mir, ty}; use rustc_span::{Symbol, sym}; use rustc_target::spec::Arch; -use self::atomic::EvalContextExt as _; use self::math::EvalContextExt as _; use self::simd::EvalContextExt as _; use crate::*; @@ -97,9 +93,6 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx, EmulateItemResult> { let this = self.eval_context_mut(); - if let Some(name) = intrinsic_name.strip_prefix("atomic_") { - return this.emulate_atomic_intrinsic(name, generic_args, args, dest); - } if let Some(name) = intrinsic_name.strip_prefix("simd_") { return this.emulate_simd_intrinsic(name, args, dest); } diff --git a/src/tools/miri/src/machine.rs b/src/tools/miri/src/machine.rs index 2156efa9179f4..f476614992041 100644 --- a/src/tools/miri/src/machine.rs +++ b/src/tools/miri/src/machine.rs @@ -25,7 +25,7 @@ use rustc_middle::query::TyCtxtAt; use rustc_middle::ty::layout::{ HasTyCtxt, HasTypingEnv, LayoutCx, LayoutError, LayoutOf, TyAndLayout, }; -use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; +use rustc_middle::ty::{self, AtomicOrdering, Instance, Ty, TyCtxt}; use rustc_session::config::InliningThreshold; use rustc_span::def_id::{CrateNum, DefId}; use rustc_span::{Span, SpanData, Symbol}; @@ -1346,6 +1346,64 @@ impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> { ecx.binary_ptr_op(bin_op, left, right) } + fn atomic_load( + ecx: &MiriInterpCx<'tcx>, + place: &MPlaceTy<'tcx>, + ordering: AtomicOrdering, + ) -> InterpResult<'tcx, Scalar> { + ecx.read_scalar_atomic(place, AtomicReadOrd::from(ordering)) + } + + fn atomic_store( + ecx: &mut MiriInterpCx<'tcx>, + place: &MPlaceTy<'tcx>, + val: &ImmTy<'tcx>, + ordering: AtomicOrdering, + ) -> InterpResult<'tcx> { + ecx.write_scalar_atomic(val.to_scalar(), place, AtomicWriteOrd::from(ordering)) + } + + fn atomic_rmw( + ecx: &mut MiriInterpCx<'tcx>, + place: &MPlaceTy<'tcx>, + op: AtomicRmwOp, + operand: &ImmTy<'tcx>, + ordering: AtomicOrdering, + ) -> InterpResult<'tcx, Scalar> { + ecx.atomic_rmw(place, operand, op, AtomicRwOrd::from(ordering)) + } + + fn atomic_compare_exchange( + ecx: &mut MiriInterpCx<'tcx>, + place: &MPlaceTy<'tcx>, + expected_old: &ImmTy<'tcx>, + new: &ImmTy<'tcx>, + can_fail_spuriously: bool, + success_ordering: AtomicOrdering, + failure_ordering: AtomicOrdering, + ) -> InterpResult<'tcx, (Scalar, bool)> { + ecx.atomic_compare_exchange( + place, + expected_old, + new.to_scalar(), + AtomicRwOrd::from(success_ordering), + AtomicReadOrd::from(failure_ordering), + can_fail_spuriously, + ) + } + + fn atomic_fence( + ecx: &MiriInterpCx<'tcx>, + ordering: AtomicOrdering, + singlethread: bool, + ) -> InterpResult<'tcx> { + if singlethread { + // We don't support signal handlers or interrupts so this is a NOP. + return interp_ok(()); + } + ecx.atomic_fence(AtomicFenceOrd::from(ordering)) + } + #[inline(always)] fn generate_nan, F2: Float>( ecx: &InterpCx<'tcx, Self>, diff --git a/src/tools/miri/tests/pass/atomic.rs b/src/tools/miri/tests/pass/atomic.rs index 0b7fff69d8ce2..458d3bd09fb50 100644 --- a/src/tools/miri/tests/pass/atomic.rs +++ b/src/tools/miri/tests/pass/atomic.rs @@ -7,7 +7,7 @@ #![allow(static_mut_refs)] use std::sync::atomic::Ordering::*; -use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicPtr, AtomicUsize, compiler_fence, fence}; +use std::sync::atomic::*; fn main() { atomic_bool(); @@ -27,13 +27,13 @@ fn atomic_bool() { assert_eq!(*ATOMIC.get_mut(), false); ATOMIC.store(true, SeqCst); assert_eq!(*ATOMIC.get_mut(), true); - ATOMIC.fetch_or(false, SeqCst); + assert_eq!(ATOMIC.fetch_or(false, SeqCst), true); assert_eq!(*ATOMIC.get_mut(), true); - ATOMIC.fetch_and(false, SeqCst); + assert_eq!(ATOMIC.fetch_and(false, SeqCst), true); assert_eq!(*ATOMIC.get_mut(), false); - ATOMIC.fetch_nand(true, SeqCst); + assert_eq!(ATOMIC.fetch_nand(true, SeqCst), false); assert_eq!(*ATOMIC.get_mut(), true); - ATOMIC.fetch_xor(true, SeqCst); + assert_eq!(ATOMIC.fetch_xor(true, SeqCst), true); assert_eq!(*ATOMIC.get_mut(), false); } } @@ -124,6 +124,19 @@ fn atomic_u64() { assert_eq!(ATOMIC.fetch_min(0x1000, SeqCst), 0x1000); assert_eq!(ATOMIC.fetch_min(0x100, SeqCst), 0x1000); assert_eq!(ATOMIC.fetch_min(0x10, SeqCst), 0x100); + + assert_eq!(ATOMIC.swap(1, SeqCst), 0x10); + assert_eq!(ATOMIC.load(Relaxed), 1); + + let atomic_signed = AtomicI64::new(0); + assert_eq!(atomic_signed.fetch_min(-1, SeqCst), 0); + assert_eq!(atomic_signed.load(SeqCst), -1); + assert_eq!(atomic_signed.fetch_min(1, SeqCst), -1); + assert_eq!(atomic_signed.load(SeqCst), -1); + assert_eq!(atomic_signed.fetch_max(1, SeqCst), -1); + assert_eq!(atomic_signed.load(SeqCst), 1); + assert_eq!(atomic_signed.fetch_max(-1, SeqCst), 1); + assert_eq!(atomic_signed.load(SeqCst), 1); } fn atomic_fences() { diff --git a/tests/ui/consts/const-eval/atomic.rs b/tests/ui/consts/const-eval/atomic.rs new file mode 100644 index 0000000000000..c8501851653d0 --- /dev/null +++ b/tests/ui/consts/const-eval/atomic.rs @@ -0,0 +1,99 @@ +//@ check-pass +#![feature(const_atomic, const_raw_ptr_comparison)] + +use std::sync::atomic::*; +use std::sync::atomic::Ordering::*; + +const BOOL: () = { + let mut atomic: AtomicBool = AtomicBool::new(false); + + unsafe { + assert!(*atomic.get_mut() == false); + atomic.store(true, SeqCst); + assert!(*atomic.get_mut() == true); + atomic.fetch_or(false, SeqCst); + assert!(*atomic.get_mut() == true); + atomic.fetch_and(false, SeqCst); + assert!(*atomic.get_mut() == false); + atomic.fetch_nand(true, SeqCst); + assert!(*atomic.get_mut() == true); + atomic.fetch_xor(true, SeqCst); + assert!(*atomic.get_mut() == false); + } +}; + +const FENCES: () = { + fence(SeqCst); + fence(Release); + fence(Acquire); + fence(AcqRel); + compiler_fence(SeqCst); + compiler_fence(Release); + compiler_fence(Acquire); + compiler_fence(AcqRel); +}; + +const fn compare_result_u64(a: Result, b: Result) -> bool { + match (a, b) { + (Ok(a), Ok(b)) => a == b, + (Err(a), Err(b)) => a == b, + _ => false, + } +} + +#[cfg(target_has_atomic = "64")] +const ATOMIC_INT: () = { + let atomic = AtomicU64::new(0); + + atomic.store(1, SeqCst); + assert!(compare_result_u64(atomic.compare_exchange(0, 0x100, AcqRel, Acquire), Err(1))); + assert!(compare_result_u64(atomic.compare_exchange(0, 1, Release, Relaxed), Err(1))); + assert!(compare_result_u64(atomic.compare_exchange(1, 0, AcqRel, Relaxed), Ok(1))); + assert!(compare_result_u64(atomic.compare_exchange(0, 1, Relaxed, Relaxed), Ok(0))); + // compare_exchange_weak always succeeds when possible, but that is not a guarantee. + assert!(compare_result_u64(atomic.compare_exchange_weak(1, 0x100, AcqRel, Acquire), Ok(1))); + assert!(compare_result_u64(atomic.compare_exchange_weak(0, 2, Acquire, Relaxed), Err(0x100))); + assert!(compare_result_u64(atomic.compare_exchange_weak(0, 1, Release, Relaxed), Err(0x100))); + assert!(atomic.load(Relaxed) == 0x100); + + assert!(atomic.fetch_max(0x10, SeqCst) == 0x100); + assert!(atomic.fetch_max(0x100, SeqCst) == 0x100); + assert!(atomic.fetch_max(0x1000, SeqCst) == 0x100); + assert!(atomic.fetch_max(0x1000, SeqCst) == 0x1000); + assert!(atomic.fetch_max(0x2000, SeqCst) == 0x1000); + assert!(atomic.fetch_max(0x2000, SeqCst) == 0x2000); + + assert!(atomic.fetch_min(0x2000, SeqCst) == 0x2000); + assert!(atomic.fetch_min(0x2000, SeqCst) == 0x2000); + assert!(atomic.fetch_min(0x1000, SeqCst) == 0x2000); + assert!(atomic.fetch_min(0x1000, SeqCst) == 0x1000); + assert!(atomic.fetch_min(0x100, SeqCst) == 0x1000); + assert!(atomic.fetch_min(0x10, SeqCst) == 0x100); + + assert!(atomic.swap(1, SeqCst) == 0x10); + assert!(atomic.load(Relaxed) == 1); + + let atomic_signed = AtomicI64::new(0); + assert!(atomic_signed.fetch_min(-1, SeqCst) == 0); + assert!(atomic_signed.load(SeqCst) == -1); + assert!(atomic_signed.fetch_min(1, SeqCst) == -1); + assert!(atomic_signed.load(SeqCst) == -1); + assert!(atomic_signed.fetch_max(1, SeqCst) == -1); + assert!(atomic_signed.load(SeqCst) == 1); + assert!(atomic_signed.fetch_max(-1, SeqCst) == 1); + assert!(atomic_signed.load(SeqCst) == 1); +}; + +const ATOMIC_PTR: () = { + use std::ptr; + let array = [0i32; 4]; // a target to point to, to test provenance things + let x = array.as_ptr() as *mut i32; + + let ptr = AtomicPtr::::new(ptr::null_mut()); + assert!(ptr.load(Relaxed).guaranteed_eq(0 as *mut i32).unwrap()); + ptr.store(ptr::without_provenance_mut(13), SeqCst); + assert!(ptr.swap(x, Relaxed).guaranteed_eq(13 as *mut i32).unwrap()); + unsafe { assert!(ptr.load(Acquire).read() == 0) }; +}; + +fn main() {} diff --git a/tests/ui/consts/miri_unleashed/const_refers_to_static.rs b/tests/ui/consts/miri_unleashed/const_refers_to_static.rs index 9546f34792ad4..205eb1d7fe935 100644 --- a/tests/ui/consts/miri_unleashed/const_refers_to_static.rs +++ b/tests/ui/consts/miri_unleashed/const_refers_to_static.rs @@ -8,7 +8,7 @@ use std::sync::atomic::Ordering; const MUTATE_INTERIOR_MUT: usize = { static FOO: AtomicUsize = AtomicUsize::new(0); - FOO.fetch_add(1, Ordering::Relaxed) //~ERROR calling non-const function `Atomic::::fetch_add` + FOO.fetch_add(1, Ordering::Relaxed) //~ERROR constant accesses mutable global memory }; const READ_INTERIOR_MUT: usize = { diff --git a/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr b/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr index 852b4518a920c..e718aac5329a4 100644 --- a/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr +++ b/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr @@ -1,8 +1,17 @@ -error[E0080]: calling non-const function `Atomic::::fetch_add` +error[E0080]: constant accesses mutable global memory --> $DIR/const_refers_to_static.rs:11:5 | LL | FOO.fetch_add(1, Ordering::Relaxed) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `MUTATE_INTERIOR_MUT` failed here + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `MUTATE_INTERIOR_MUT` failed inside this call + | +note: inside `Atomic::::fetch_add` + --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL + ::: $SRC_DIR/core/src/sync/atomic.rs:LL:COL + | + = note: in this macro invocation +note: inside `atomic::atomic_add::` + --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL + = note: this error originates in the macro `atomic_int` which comes from the expansion of the macro `atomic_int_ptr_sized` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0080]: constant accesses mutable global memory --> $DIR/const_refers_to_static.rs:16:14 @@ -26,7 +35,7 @@ LL | REF_INTERIOR_MUT => {}, warning: skipping const checks | -help: skipping check that does not even have a feature gate +help: skipping check for `const_atomic` feature --> $DIR/const_refers_to_static.rs:11:5 | LL | FOO.fetch_add(1, Ordering::Relaxed)