diff -r 000000000000 -r 96e5fb8b040d kernel/eka/common/x86/atomics.cia --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kernel/eka/common/x86/atomics.cia Thu Dec 17 09:24:54 2009 +0200 @@ -0,0 +1,1049 @@ +// Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). +// All rights reserved. +// This component and the accompanying materials are made available +// under the terms of the License "Eclipse Public License v1.0" +// which accompanies this distribution, and is available +// at the URL "http://www.eclipse.org/legal/epl-v10.html". +// +// Initial Contributors: +// Nokia Corporation - initial contribution. +// +// Contributors: +// +// Description: +// e32\common\x86\atomics.cia +// +// + +#include +#include + +/* +Versions needed: + WINS/WINSCW Use X86 locked operations. Assume Pentium or above CPU (CMPXCHG8B available) + X86 For Pentium and above use locked operations + For 486 use locked operations for 8, 16, 32 bit. For 64 bit must disable interrupts. + NOTE: 486 not supported at the moment + ARMv4/ARMv5 Must disable interrupts. + ARMv6 LDREX/STREX for 8, 16, 32 bit. For 64 bit must disable interrupts (maybe). + ARMv6K/ARMv7 LDREXB/LDREXH/LDREX/LDREXD + +Need both kernel side and user side versions +*/ + +#if defined(__SMP__) || !defined(__EPOC32__) +#define __BARRIERS_NEEDED__ +#define __LOCK__ "lock " +#else +#define __LOCK__ +#endif + + +extern "C" { + +#undef __TUintX__ +#undef __TIntX__ +#undef __fname__ +#undef __redir__ +#undef __A_REG__ +#undef __C_REG__ +#undef __D_REG__ +#define __TUintX__ TUint32 +#define __TIntX__ TInt32 +#define __fname__(x) x##32 +#define __redir__(x) asm("jmp _"#x "32") +#define __A_REG__ "eax" +#define __C_REG__ "ecx" +#define __D_REG__ "edx" +#include "atomic_skeleton.h" + +#undef __TUintX__ +#undef __TIntX__ +#undef __fname__ +#undef __redir__ +#undef __A_REG__ +#undef __C_REG__ +#undef __D_REG__ +#define __TUintX__ TUint16 +#define __TIntX__ TInt16 +#define __fname__(x) x##16 +#define __redir__(x) asm("jmp _"#x "16") +#define __A_REG__ "ax" +#define __C_REG__ "cx" +#define __D_REG__ "dx" +#include "atomic_skeleton.h" + +#undef __TUintX__ +#undef __TIntX__ +#undef __fname__ +#undef __redir__ +#undef __A_REG__ +#undef __C_REG__ +#undef __D_REG__ +#define __TUintX__ TUint8 +#define __TIntX__ TInt8 +#define __fname__(x) x##8 +#define __redir__(x) asm("jmp _"#x "8") +#define __A_REG__ "al" +#define __C_REG__ "cl" +#define __D_REG__ "dl" +#include "atomic_skeleton.h" + +#undef __TUintX__ +#undef __TIntX__ +#undef __fname__ +#undef __redir__ +#undef __A_REG__ +#undef __C_REG__ +#undef __D_REG__ + +/** Full memory barrier for explicit memory accesses + +*/ +EXPORT_C __NAKED__ void __e32_memory_barrier() + { +#ifdef __BARRIERS_NEEDED__ + asm("lock add dword ptr [esp], 0 "); +#endif + asm("ret "); + } + + +/** Barrier guaranteeing completion as well as ordering + +*/ +EXPORT_C __NAKED__ void __e32_io_completion_barrier() + { + asm("push ebx "); + asm("cpuid "); + asm("pop ebx "); + asm("ret "); + } + + +/** Find the most significant 1 in a 32 bit word + + @param v The word to be scanned + @return The bit number of the most significant 1 if v != 0 + -1 if v == 0 +*/ +EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/) + { + asm("bsr eax, [esp+4] "); + asm("jnz short 1f "); + asm("mov eax, 0xffffffff "); + asm("1: "); + asm("ret "); + } + + +/** Find the least significant 1 in a 32 bit word + + @param v The word to be scanned + @return The bit number of the least significant 1 if v != 0 + -1 if v == 0 +*/ +EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/) + { + asm("bsf eax, [esp+4] "); + asm("jnz short 1f "); + asm("mov eax, 0xffffffff "); + asm("1: "); + asm("ret "); + } + + +/** Count the number of 1's in a 32 bit word + + @param v The word to be scanned + @return The number of 1's +*/ +EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/) + { + asm("mov eax, [esp+4] "); + asm("mov edx, eax "); + asm("and eax, 0xaaaaaaaa "); + asm("and edx, 0x55555555 "); /* edx = even bits of arg */ + asm("shr eax, 1 "); /* eax = odd bits of arg shifted into even bits */ + asm("add eax, edx "); /* eax = 16 groups of 2 bit counts */ + asm("mov edx, eax "); + asm("and eax, 0xcccccccc "); + asm("and edx, 0x33333333 "); /* even groups of 2 */ + asm("shr eax, 2 "); /* odd groups of 2 shifted to even positions */ + asm("add eax, edx "); /* 8 groups of 4 bit counts */ + asm("mov edx, eax "); + asm("shr eax, 4 "); + asm("add eax, edx "); /* even nibbles = sum of 8 bits, odd nibbles garbage */ + asm("and eax, 0x0f0f0f0f "); /* eliminate garbage nibbles */ + asm("add al, ah "); /* AL = bit count of lower 16 bits */ + asm("mov dl, al "); + asm("shr eax, 16 "); + asm("add al, ah "); /* AL = bit count of upper 16 bits */ + asm("xor ah, ah "); /* top 24 bits of EAX now zero */ + asm("add al, dl "); /* AL = bit count of entire 32 bits */ + asm("ret "); + } + + +/** Find the most significant 1 in a 64 bit word + + @param v The word to be scanned + @return The bit number of the most significant 1 if v != 0 + -1 if v == 0 +*/ +EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/) + { + asm("bsr eax, [esp+8] "); + asm("jnz short 2f "); + asm("bsr eax, [esp+4] "); + asm("jnz short 1f "); + asm("mov eax, 0xffffffff "); + asm("2: "); + asm("or eax, 32 "); + asm("1: "); + asm("ret "); + } + + +/** Find the least significant 1 in a 64 bit word + + @param v The word to be scanned + @return The bit number of the least significant 1 if v != 0 + -1 if v == 0 +*/ +EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/) + { + asm("bsf eax, [esp+4] "); + asm("jnz short 1f "); + asm("bsf eax, [esp+8] "); + asm("jnz short 2f "); + asm("mov eax, 0xffffffff "); + asm("2: "); + asm("or eax, 32 "); + asm("1: "); + asm("ret "); + } + + +/** Count the number of 1's in a 64 bit word + + @param v The word to be scanned + @return The number of 1's +*/ +EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/) + { + asm("mov eax, [esp+4] "); + asm("mov edx, [esp+8] "); + + asm("mov ecx, eax "); + asm("and eax, 0xaaaaaaaa "); + asm("and ecx, 0x55555555 "); + asm("shr eax, 1 "); + asm("add eax, ecx "); + asm("mov ecx, eax "); + asm("and eax, 0xcccccccc "); + asm("and ecx, 0x33333333 "); + asm("shr eax, 2 "); + asm("add ecx, eax "); + + asm("mov eax, edx "); + asm("and eax, 0xaaaaaaaa "); + asm("and edx, 0x55555555 "); + asm("shr eax, 1 "); + asm("add eax, edx "); + asm("mov edx, eax "); + asm("and eax, 0xcccccccc "); + asm("and edx, 0x33333333 "); + asm("shr eax, 2 "); + asm("add eax, edx "); + + asm("add eax, ecx "); + asm("mov edx, eax "); + asm("and eax, 0xf0f0f0f0 "); + asm("and edx, 0x0f0f0f0f "); + asm("shr eax, 4 "); + asm("add eax, edx "); + asm("add al, ah "); + asm("mov dl, al "); + asm("shr eax, 16 "); + asm("add al, ah "); + asm("xor ah, ah "); + asm("add al, dl "); + asm("ret "); + } + + + + +/** Read a 64 bit word with acquire semantics + + @param a Address of word to be read - must be a multiple of 8 + @return The value read +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_load_acq64(const volatile TAny* /*a*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); + asm("mov eax, 0x0badbeef "); + asm("mov edx, eax "); + asm("mov ebx, eax "); + asm("mov ecx, eax "); + asm(__LOCK__ "cmpxchg8b [edi] "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** Write a 64 bit word with release semantics + + @param a Address of word to be written - must be a multiple of 8 + @param v The value to be written + @return The value written +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_store_rel64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); + asm("mov ebx, [esp+16] "); + asm("mov ecx, [esp+20] "); + asm("mov eax, [edi] "); + asm("mov edx, [edi+4] "); + asm("1: "); + asm(__LOCK__ "cmpxchg8b [edi] " ); + asm("jne short 1b "); + asm("mov eax, ebx "); + asm("mov edx, ecx "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** Write a 64 bit word with full barrier semantics + + @param a Address of word to be written - must be a multiple of 8 + @param v The value to be written + @return The value written +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_store_ord64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_store_rel64 "); + } + + +/** Write a 64 bit word to memory and return the original value of the memory. + Relaxed ordering. + + @param a Address of word to be written - must be a multiple of 8 + @param v The value to be written + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_swp_rlx64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_swp_ord64 "); + } + + +/** Write a 64 bit word to memory and return the original value of the memory. + Acquire semantics. + + @param a Address of word to be written - must be a multiple of 8 + @param v The value to be written + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_swp_acq64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_swp_ord64 "); + } + + +/** Write a 64 bit word to memory and return the original value of the memory. + Release semantics. + + @param a Address of word to be written - must be a multiple of 8 + @param v The value to be written + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_swp_rel64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_swp_ord64 "); + } + + +/** Write a 64 bit word to memory and return the original value of the memory. + Full barrier semantics. + + @param a Address of word to be written - must be a multiple of 8 + @param v The value to be written + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_swp_ord64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); + asm("mov ebx, [esp+16] "); + asm("mov ecx, [esp+20] "); + asm("mov eax, [edi] "); + asm("mov edx, [edi+4] "); + asm("1: "); + asm(__LOCK__ "cmpxchg8b [edi] "); + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit compare and swap, relaxed ordering. + + Atomically performs the following operation: + if (*a == *q) { *a = v; return TRUE; } + else { *q = *a; return FALSE; } + + @param a Address of word to be written - must be a multiple of 8 + @param q Address of location containing expected value + @param v The new value to be written if the old value is as expected + @return TRUE if *a was updated, FALSE otherwise +*/ +EXPORT_C __NAKED__ TBool __e32_atomic_cas_rlx64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_cas_ord64 "); + } + + +/** 64 bit compare and swap, acquire semantics. + + Atomically performs the following operation: + if (*a == *q) { *a = v; return TRUE; } + else { *q = *a; return FALSE; } + + @param a Address of word to be written - must be a multiple of 8 + @param q Address of location containing expected value + @param v The new value to be written if the old value is as expected + @return TRUE if *a was updated, FALSE otherwise +*/ +EXPORT_C __NAKED__ TBool __e32_atomic_cas_acq64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_cas_ord64 "); + } + + +/** 64 bit compare and swap, release semantics. + + Atomically performs the following operation: + if (*a == *q) { *a = v; return TRUE; } + else { *q = *a; return FALSE; } + + @param a Address of word to be written - must be a multiple of 8 + @param q Address of location containing expected value + @param v The new value to be written if the old value is as expected + @return TRUE if *a was updated, FALSE otherwise +*/ +EXPORT_C __NAKED__ TBool __e32_atomic_cas_rel64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_cas_ord64 "); + } + + +/** 64 bit compare and swap, full barrier semantics. + + Atomically performs the following operation: + if (*a == *q) { *a = v; return TRUE; } + else { *q = *a; return FALSE; } + + @param a Address of word to be written - must be a multiple of 8 + @param q Address of location containing expected value + @param v The new value to be written if the old value is as expected + @return TRUE if *a was updated, FALSE otherwise +*/ +EXPORT_C __NAKED__ TBool __e32_atomic_cas_ord64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("push esi "); + asm("mov edi, [esp+16] "); // edi = a + asm("mov esi, [esp+20] "); // esi = q + asm("mov ebx, [esp+24] "); // ecx:ebx = v + asm("mov ecx, [esp+28] "); + asm("mov eax, [esi] "); // edx:eax = *q + asm("mov edx, [esi+4] "); + asm(__LOCK__ "cmpxchg8b [edi] "); // if (*a==*q) *a=v, ZF=1 else edx:eax=*a, ZF=0 + asm("jne short 2f "); + asm("mov eax, 1 "); + asm("pop esi "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + asm("2: "); + asm("mov [esi], eax "); // *q = edx:eax + asm("mov [esi+4], edx "); + asm("xor eax, eax "); + asm("pop esi "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit atomic add, relaxed ordering. + + Atomically performs the following operation: + oldv = *a; *a = oldv + v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be added + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_add_rlx64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_add_ord64 "); + } + + +/** 64 bit atomic add, acquire semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv + v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be added + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_add_acq64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_add_ord64 "); + } + + +/** 64 bit atomic add, release semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv + v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be added + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_add_rel64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_add_ord64 "); + } + + +/** 64 bit atomic add, full barrier semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv + v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be added + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_add_ord64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); // edi = a + asm("mov eax, [edi] "); // edx:eax = oldv + asm("mov edx, [edi+4] "); + asm("1: "); + asm("mov ebx, eax "); + asm("mov ecx, edx "); + asm("add ebx, [esp+16] "); // ecx:ebx = oldv + v + asm("adc ecx, [esp+20] "); + asm(__LOCK__ "cmpxchg8b [edi] "); // if (*a==oldv) *a=oldv+v, ZF=1 else edx:eax=*a, ZF=0 + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit atomic bitwise logical AND, relaxed ordering. + + Atomically performs the following operation: + oldv = *a; *a = oldv & v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ANDed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_and_rlx64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_and_ord64 "); + } + + +/** 64 bit atomic bitwise logical AND, acquire semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv & v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ANDed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_and_acq64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_and_ord64 "); + } + + +/** 64 bit atomic bitwise logical AND, release semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv & v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ANDed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_and_rel64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_and_ord64 "); + } + + +/** 64 bit atomic bitwise logical AND, full barrier semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv & v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ANDed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_and_ord64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); // edi = a + asm("mov eax, [edi] "); // edx:eax = oldv + asm("mov edx, [edi+4] "); + asm("1: "); + asm("mov ebx, eax "); + asm("mov ecx, edx "); + asm("and ebx, [esp+16] "); // ecx:ebx = oldv & v + asm("and ecx, [esp+20] "); + asm(__LOCK__ "cmpxchg8b [edi] "); // if (*a==oldv) *a=oldv&v, ZF=1 else edx:eax=*a, ZF=0 + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit atomic bitwise logical inclusive OR, relaxed ordering. + + Atomically performs the following operation: + oldv = *a; *a = oldv | v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_ior_rlx64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_ior_ord64 "); + } + + +/** 64 bit atomic bitwise logical inclusive OR, acquire semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv | v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_ior_acq64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_ior_ord64 "); + } + + +/** 64 bit atomic bitwise logical inclusive OR, release semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv | v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_ior_rel64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_ior_ord64 "); + } + + +/** 64 bit atomic bitwise logical inclusive OR, full barrier semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv | v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be ORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_ior_ord64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); // edi = a + asm("mov eax, [edi] "); // edx:eax = oldv + asm("mov edx, [edi+4] "); + asm("1: "); + asm("mov ebx, eax "); + asm("mov ecx, edx "); + asm("or ebx, [esp+16] "); // ecx:ebx = oldv | v + asm("or ecx, [esp+20] "); + asm(__LOCK__ "cmpxchg8b [edi] "); // if (*a==oldv) *a=oldv|v, ZF=1 else edx:eax=*a, ZF=0 + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit atomic bitwise logical exclusive OR, relaxed ordering. + + Atomically performs the following operation: + oldv = *a; *a = oldv ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be XORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_xor_rlx64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_xor_ord64 "); + } + + +/** 64 bit atomic bitwise logical exclusive OR, acquire semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be XORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_xor_acq64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_xor_ord64 "); + } + + +/** 64 bit atomic bitwise logical exclusive OR, release semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be XORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_xor_rel64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_xor_ord64 "); + } + + +/** 64 bit atomic bitwise logical exclusive OR, full barrier semantics. + + Atomically performs the following operation: + oldv = *a; *a = oldv ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param v The value to be XORed with *a + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_xor_ord64(volatile TAny* /*a*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); // edi = a + asm("mov eax, [edi] "); // edx:eax = oldv + asm("mov edx, [edi+4] "); + asm("1: "); + asm("mov ebx, eax "); + asm("mov ecx, edx "); + asm("xor ebx, [esp+16] "); // ecx:ebx = oldv ^ v + asm("xor ecx, [esp+20] "); + asm(__LOCK__ "cmpxchg8b [edi] "); // if (*a==oldv) *a=oldv^v, ZF=1 else edx:eax=*a, ZF=0 + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit atomic bitwise universal function, relaxed ordering. + + Atomically performs the following operation: + oldv = *a; *a = (oldv & u) ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param u The value to be ANDed with *a + @param v The value to be XORed with (*a&u) + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_axo_rlx64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_axo_ord64 "); + } + + +/** 64 bit atomic bitwise universal function, acquire semantics. + + Atomically performs the following operation: + oldv = *a; *a = (oldv & u) ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param u The value to be ANDed with *a + @param v The value to be XORed with (*a&u) + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_axo_acq64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_axo_ord64 "); + } + + +/** 64 bit atomic bitwise universal function, release semantics. + + Atomically performs the following operation: + oldv = *a; *a = (oldv & u) ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param u The value to be ANDed with *a + @param v The value to be XORed with (*a&u) + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_axo_rel64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_axo_ord64 "); + } + + +/** 64 bit atomic bitwise universal function, release semantics. + + Atomically performs the following operation: + oldv = *a; *a = (oldv & u) ^ v; return oldv; + + @param a Address of word to be updated - must be a multiple of 8 + @param u The value to be ANDed with *a + @param v The value to be XORed with (*a&u) + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_axo_ord64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); // edi = a + asm("mov eax, [edi] "); // edx:eax = oldv + asm("mov edx, [edi+4] "); + asm("1: "); + asm("mov ebx, eax "); + asm("mov ecx, edx "); + asm("and ebx, [esp+16] "); // ecx:ebx = oldv & u + asm("and ecx, [esp+20] "); + asm("xor ebx, [esp+24] "); // ecx:ebx = (oldv & u) ^ v + asm("xor ecx, [esp+28] "); + asm(__LOCK__ "cmpxchg8b [edi] "); // if (*a==oldv) *a=(oldv&u)^v, ZF=1 else edx:eax=*a, ZF=0 + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit threshold and add, unsigned, relaxed ordering. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (unsigned compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_tau_rlx64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_tau_ord64 "); + } + + +/** 64 bit threshold and add, unsigned, acquire semantics. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (unsigned compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_tau_acq64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_tau_ord64 "); + } + + +/** 64 bit threshold and add, unsigned, release semantics. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (unsigned compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_tau_rel64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("jmp ___e32_atomic_tau_ord64 "); + } + + +/** 64 bit threshold and add, unsigned, full barrier semantics. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (unsigned compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TUint64 __e32_atomic_tau_ord64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); // edi = a + asm("mov eax, [edi] "); // edx:eax = oldv + asm("mov edx, [edi+4] "); + asm("1: "); + asm("mov ebx, edx "); + asm("cmp eax, [esp+16] "); // eax - t.low, CF=borrow + asm("sbb ebx, [esp+20] "); // CF = borrow from (oldv - t) + asm("jnc short 2f "); // no borrow means oldv>=t so use u + asm("mov ebx, [esp+32] "); // ecx:ebx = v + asm("mov ecx, [esp+36] "); + asm("jmp short 3f "); + asm("2: "); + asm("mov ebx, [esp+24] "); // ecx:ebx = u + asm("mov ecx, [esp+28] "); + asm("3: "); + asm("add ebx, eax "); // ecx:ebx = oldv + u or v + asm("adc ecx, edx "); + asm(__LOCK__ "cmpxchg8b [edi] "); + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + + +/** 64 bit threshold and add, signed, relaxed ordering. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (signed compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TInt64 __e32_atomic_tas_rlx64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/) + { + asm("jmp ___e32_atomic_tas_ord64 "); + } + + +/** 64 bit threshold and add, signed, acquire semantics. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (signed compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TInt64 __e32_atomic_tas_acq64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/) + { + asm("jmp ___e32_atomic_tas_ord64 "); + } + + +/** 64 bit threshold and add, signed, release semantics. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (signed compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TInt64 __e32_atomic_tas_rel64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/) + { + asm("jmp ___e32_atomic_tas_ord64 "); + } + + +/** 64 bit threshold and add, signed, full barrier semantics. + + Atomically performs the following operation: + oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv; + + @param a Address of data to be updated - must be naturally aligned + @param t The threshold to compare *a to (signed compare) + @param u The value to be added to *a if it is originally >= t + @param u The value to be added to *a if it is originally < t + @return The original value of *a +*/ +EXPORT_C __NAKED__ TInt64 __e32_atomic_tas_ord64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/) + { + asm("push ebx "); + asm("push edi "); + asm("mov edi, [esp+12] "); // edi = a + asm("mov eax, [edi] "); // edx:eax = oldv + asm("mov edx, [edi+4] "); + asm("1: "); + asm("mov ebx, edx "); + asm("cmp eax, [esp+16] "); // eax - t.low, CF=borrow + asm("sbb ebx, [esp+20] "); // SF=sign, OF=overflow from (oldv - t) + asm("jge short 2f "); // SF==OF (GE condition) means oldv>=t so use u + asm("mov ebx, [esp+32] "); // ecx:ebx = v + asm("mov ecx, [esp+36] "); + asm("jmp short 3f "); + asm("2: "); + asm("mov ebx, [esp+24] "); // ecx:ebx = u + asm("mov ecx, [esp+28] "); + asm("3: "); + asm("add ebx, eax "); // ecx:ebx = oldv + u or v + asm("adc ecx, edx "); + asm(__LOCK__ "cmpxchg8b [edi] "); + asm("jne short 1b "); + asm("pop edi "); + asm("pop ebx "); + asm("ret "); + } + +} // extern "C"