1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
//! ARM's Transactional Memory Extensions (TME).
//!
//! This CPU feature is available on Aarch64 - A architecture profile.
//! This feature is in the non-neon feature set. TME specific vendor documentation can
//! be found [TME Intrinsics Introduction][tme_intrinsics_intro].
//!
//! The reference is [ACLE Q4 2019][acle_q4_2019_ref].
//!
//! ACLE has a section for TME extensions and state masks for aborts and failure codes.
//! [ARM A64 Architecture Register Datasheet][a_profile_future] also describes possible failure code scenarios.
//!
//! [acle_q4_2019_ref]: https://static.docs.arm.com/101028/0010/ACLE_2019Q4_release-0010.pdf
//! [tme_intrinsics_intro]: https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics
//! [llvm_aarch64_int]: https://github.com/llvm/llvm-project/commit/a36d31478c182903523e04eb271bbf102bfab2cc#diff-ff24e1c35f4d54f1110ce5d90c709319R626-R646
//! [a_profile_future]: https://static.docs.arm.com/ddi0601/a/SysReg_xml_futureA-2019-04.pdf?_ga=2.116560387.441514988.1590524918-1110153136.1588469296
#[cfg(test)]
use stdarch_test::assert_instr;
extern "unadjusted" {
#[link_name = "llvm.aarch64.tstart"]
fn aarch64_tstart() -> u64;
#[link_name = "llvm.aarch64.tcommit"]
fn aarch64_tcommit();
#[link_name = "llvm.aarch64.tcancel"]
fn aarch64_tcancel(imm0: u64);
#[link_name = "llvm.aarch64.ttest"]
fn aarch64_ttest() -> u64;
}
/// Transaction successfully started.
pub const _TMSTART_SUCCESS: u64 = 0x00_u64;
/// Extraction mask for failure reason
pub const _TMFAILURE_REASON: u64 = 0x00007FFF_u64;
/// Transaction retry is possible.
pub const _TMFAILURE_RTRY: u64 = 1 << 15;
/// Transaction executed a TCANCEL instruction
pub const _TMFAILURE_CNCL: u64 = 1 << 16;
/// Transaction aborted because a conflict occurred
pub const _TMFAILURE_MEM: u64 = 1 << 17;
/// Fallback error type for any other reason
pub const _TMFAILURE_IMP: u64 = 1 << 18;
/// Transaction aborted because a non-permissible operation was attempted
pub const _TMFAILURE_ERR: u64 = 1 << 19;
/// Transaction aborted due to read or write set limit was exceeded
pub const _TMFAILURE_SIZE: u64 = 1 << 20;
/// Transaction aborted due to transactional nesting level was exceeded
pub const _TMFAILURE_NEST: u64 = 1 << 21;
/// Transaction aborted due to a debug trap.
pub const _TMFAILURE_DBG: u64 = 1 << 22;
/// Transaction failed from interrupt
pub const _TMFAILURE_INT: u64 = 1 << 23;
/// Indicates a TRIVIAL version of TM is available
pub const _TMFAILURE_TRIVIAL: u64 = 1 << 24;
/// Starts a new transaction. When the transaction starts successfully the return value is 0.
/// If the transaction fails, all state modifications are discarded and a cause of the failure
/// is encoded in the return value.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(tstart))]
pub unsafe fn __tstart() -> u64 {
aarch64_tstart()
}
/// Commits the current transaction. For a nested transaction, the only effect is that the
/// transactional nesting depth is decreased. For an outer transaction, the state modifications
/// performed transactionally are committed to the architectural state.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(tcommit))]
pub unsafe fn __tcommit() {
aarch64_tcommit()
}
/// Cancels the current transaction and discards all state modifications that were performed transactionally.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(tcancel, IMM16 = 0x0))]
#[rustc_legacy_const_generics(0)]
pub unsafe fn __tcancel<const IMM16: u64>() {
static_assert!(IMM16 <= 65535);
aarch64_tcancel(IMM16);
}
/// Tests if executing inside a transaction. If no transaction is currently executing,
/// the return value is 0. Otherwise, this intrinsic returns the depth of the transaction.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(ttest))]
pub unsafe fn __ttest() -> u64 {
aarch64_ttest()
}
#[cfg(test)]
mod tests {
use stdarch_test::simd_test;
use crate::core_arch::aarch64::*;
const CANCEL_CODE: u64 = (0 | (0x123 & _TMFAILURE_REASON) as u64) as u64;
#[simd_test(enable = "tme")]
unsafe fn test_tstart() {
let mut x = 0;
for i in 0..10 {
let code = tme::__tstart();
if code == _TMSTART_SUCCESS {
x += 1;
assert_eq!(x, i + 1);
break;
}
assert_eq!(x, 0);
}
}
#[simd_test(enable = "tme")]
unsafe fn test_tcommit() {
let mut x = 0;
for i in 0..10 {
let code = tme::__tstart();
if code == _TMSTART_SUCCESS {
x += 1;
assert_eq!(x, i + 1);
tme::__tcommit();
}
assert_eq!(x, i + 1);
}
}
#[simd_test(enable = "tme")]
unsafe fn test_tcancel() {
let mut x = 0;
for i in 0..10 {
let code = tme::__tstart();
if code == _TMSTART_SUCCESS {
x += 1;
assert_eq!(x, i + 1);
tme::__tcancel::<CANCEL_CODE>();
break;
}
}
assert_eq!(x, 0);
}
#[simd_test(enable = "tme")]
unsafe fn test_ttest() {
for _ in 0..10 {
let code = tme::__tstart();
if code == _TMSTART_SUCCESS {
if tme::__ttest() == 2 {
tme::__tcancel::<CANCEL_CODE>();
break;
}
}
}
}
}