From ba660ea7548d86147afe30f314f67527c22c267b Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Sun, 31 Jul 2011 18:38:00 -0300 Subject: [PATCH] Port the i386 and x86-64 atomics to the new QBasicAtomic architecture Both implementations now are very similar to one another, to the point we could share the code if we wanted to. They are based on assembly code for the Relaxed functions only, as the i386 and x86-64 architectures only allow for full memory ordering or something that closely resembles it (see 8.2 "Memory ordering" in the Intel 64 and IA-32 Architectures Software Developer's Manual Volume 3A). We could add "lfence/mfence/sfence" in future versions if we wanted to (SSE2+). Change-Id: I76966d9f8694edfece2c5ebd3387348fac721447 Reviewed-by: Bradley T. Hughes Reviewed-by: Thiago Macieira --- src/corelib/arch/qatomic_arch.h | 4 - src/corelib/arch/qatomic_i386.h | 452 ++++++++++++++++-------------- src/corelib/arch/qatomic_x86_64.h | 440 ++++++++++++++++------------- src/corelib/thread/qbasicatomic.h | 7 + 4 files changed, 494 insertions(+), 409 deletions(-) diff --git a/src/corelib/arch/qatomic_arch.h b/src/corelib/arch/qatomic_arch.h index d84de5643a..cda824498c 100644 --- a/src/corelib/arch/qatomic_arch.h +++ b/src/corelib/arch/qatomic_arch.h @@ -62,8 +62,6 @@ QT_BEGIN_HEADER # include "QtCore/qatomic_bfin.h" #elif defined(QT_ARCH_GENERIC) # include "QtCore/qatomic_generic.h" -#elif defined(QT_ARCH_I386) -# include "QtCore/qatomic_i386.h" #elif defined(QT_ARCH_IA64) # include "QtCore/qatomic_ia64.h" #elif defined(QT_ARCH_MACOSX) @@ -82,8 +80,6 @@ QT_BEGIN_HEADER # include "QtCore/qatomic_windows.h" #elif defined(QT_ARCH_WINDOWSCE) # include "QtCore/qatomic_windowsce.h" -#elif defined(QT_ARCH_X86_64) -# include "QtCore/qatomic_x86_64.h" #elif defined(QT_ARCH_SH) # include "QtCore/qatomic_sh.h" #elif defined(QT_ARCH_SH4A) diff --git a/src/corelib/arch/qatomic_i386.h b/src/corelib/arch/qatomic_i386.h index b625dc85ff..0c75d3540b 100644 --- a/src/corelib/arch/qatomic_i386.h +++ b/src/corelib/arch/qatomic_i386.h @@ -1,7 +1,7 @@ /**************************************************************************** ** ** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). -** All rights reserved. +** Copyright (C) 2011 Thiago Macieira ** Contact: http://www.qt-project.org/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -42,102 +42,175 @@ #ifndef QATOMIC_I386_H #define QATOMIC_I386_H +#include + QT_BEGIN_HEADER QT_BEGIN_NAMESPACE +#if 0 +#pragma qt_sync_stop_processing +#endif + +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; + #define Q_ATOMIC_INT_REFERENCE_COUNTING_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_REFERENCE_COUNTING_IS_WAIT_FREE -inline bool QBasicAtomicInt::isReferenceCountingNative() -{ return true; } -inline bool QBasicAtomicInt::isReferenceCountingWaitFree() -{ return true; } - #define Q_ATOMIC_INT_TEST_AND_SET_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_TEST_AND_SET_IS_WAIT_FREE -inline bool QBasicAtomicInt::isTestAndSetNative() -{ return true; } -inline bool QBasicAtomicInt::isTestAndSetWaitFree() -{ return true; } - #define Q_ATOMIC_INT_FETCH_AND_STORE_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_FETCH_AND_STORE_IS_WAIT_FREE -inline bool QBasicAtomicInt::isFetchAndStoreNative() -{ return true; } -inline bool QBasicAtomicInt::isFetchAndStoreWaitFree() -{ return true; } - #define Q_ATOMIC_INT_FETCH_AND_ADD_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_FETCH_AND_ADD_IS_WAIT_FREE -inline bool QBasicAtomicInt::isFetchAndAddNative() -{ return true; } -inline bool QBasicAtomicInt::isFetchAndAddWaitFree() -{ return true; } +#define Q_ATOMIC_INT32_IS_SUPPORTED + +#define Q_ATOMIC_INT32_REFERENCE_COUNTING_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_REFERENCE_COUNTING_IS_WAIT_FREE + +#define Q_ATOMIC_INT32_TEST_AND_SET_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT32_FETCH_AND_STORE_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_FETCH_AND_STORE_IS_WAIT_FREE + +#define Q_ATOMIC_INT32_FETCH_AND_ADD_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_FETCH_AND_ADD_IS_WAIT_FREE #define Q_ATOMIC_POINTER_TEST_AND_SET_IS_ALWAYS_NATIVE #define Q_ATOMIC_POINTER_TEST_AND_SET_IS_WAIT_FREE -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isTestAndSetNative() -{ return true; } -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isTestAndSetWaitFree() -{ return true; } - #define Q_ATOMIC_POINTER_FETCH_AND_STORE_IS_ALWAYS_NATIVE #define Q_ATOMIC_POINTER_FETCH_AND_STORE_IS_WAIT_FREE -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndStoreNative() -{ return true; } -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndStoreWaitFree() -{ return true; } - #define Q_ATOMIC_POINTER_FETCH_AND_ADD_IS_ALWAYS_NATIVE #define Q_ATOMIC_POINTER_FETCH_AND_ADD_IS_WAIT_FREE -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndAddNative() -{ return true; } -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndAddWaitFree() -{ return true; } +template struct QBasicAtomicOps: QGenericAtomicOps > +{ + static inline bool isReferenceCountingNative() { return true; } + static inline bool isReferenceCountingWaitFree() { return true; } + template static bool ref(T &_q_value); + template static bool deref(T &_q_value); + + static inline bool isTestAndSetNative() { return true; } + static inline bool isTestAndSetWaitFree() { return true; } + template static bool testAndSetRelaxed(T &_q_value, T expectedValue, T newValue); + + static inline bool isFetchAndStoreNative() { return true; } + static inline bool isFetchAndStoreWaitFree() { return true; } + template static T fetchAndStoreRelaxed(T &_q_value, T newValue); + + static inline bool isFetchAndAddNative() { return true; } + static inline bool isFetchAndAddWaitFree() { return true; } + template static + T fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd); +}; + +template struct QAtomicOps : QBasicAtomicOps +{ + typedef T Type; +}; #if defined(Q_CC_GNU) || defined(Q_CC_INTEL) -inline bool QBasicAtomicInt::ref() +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; + +template<> template inline +bool QBasicAtomicOps<1>::ref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "incb %0\n" + "setne %1" + : "+m" (_q_value), "=qm" (ret) + : + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<2>::ref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "incw %0\n" + "setne %1" + : "+m" (_q_value), "=qm" (ret) + : + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<4>::ref(T &_q_value) { unsigned char ret; asm volatile("lock\n" "incl %0\n" "setne %1" - : "=m" (_q_value), "=qm" (ret) - : "m" (_q_value) + : "+m" (_q_value), "=qm" (ret) + : : "memory"); return ret != 0; } -inline bool QBasicAtomicInt::deref() +template<> template inline +bool QBasicAtomicOps<1>::deref(T &_q_value) { - unsigned char ret; + unsigned char ret; + asm volatile("lock\n" + "decb %0\n" + "setne %1" + : "+m" (_q_value), "=qm" (ret) + : + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<2>::deref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "decw %0\n" + "setne %1" + : "+m" (_q_value), "=qm" (ret) + : + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<4>::deref(T &_q_value) +{ + unsigned char ret; asm volatile("lock\n" "decl %0\n" "setne %1" - : "=m" (_q_value), "=qm" (ret) - : "m" (_q_value) + : "+m" (_q_value), "=qm" (ret) + : : "memory"); return ret != 0; } -inline bool QBasicAtomicInt::testAndSetOrdered(int expectedValue, int newValue) +template template inline +bool QBasicAtomicOps::testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) { unsigned char ret; asm volatile("lock\n" - "cmpxchgl %3,%2\n" + "cmpxchg %3,%2\n" "sete %1\n" : "=a" (newValue), "=qm" (ret), "+m" (_q_value) : "r" (newValue), "0" (expectedValue) @@ -145,59 +218,131 @@ inline bool QBasicAtomicInt::testAndSetOrdered(int expectedValue, int newValue) return ret != 0; } -inline int QBasicAtomicInt::fetchAndStoreOrdered(int newValue) -{ - asm volatile("xchgl %0,%1" - : "=r" (newValue), "+m" (_q_value) - : "0" (newValue) - : "memory"); - return newValue; -} - -inline int QBasicAtomicInt::fetchAndAddOrdered(int valueToAdd) -{ - asm volatile("lock\n" - "xaddl %0,%1" - : "=r" (valueToAdd), "+m" (_q_value) - : "0" (valueToAdd) - : "memory"); - return valueToAdd; -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetOrdered(T *expectedValue, T *newValue) +template<> template inline +bool QBasicAtomicOps<1>::testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) { unsigned char ret; asm volatile("lock\n" - "cmpxchgl %3,%2\n" + "cmpxchg %3,%2\n" "sete %1\n" : "=a" (newValue), "=qm" (ret), "+m" (_q_value) - : "r" (newValue), "0" (expectedValue) + : "q" (newValue), "0" (expectedValue) : "memory"); return ret != 0; } -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreOrdered(T *newValue) +template template inline +T QBasicAtomicOps::fetchAndStoreRelaxed(T &_q_value, T newValue) { - asm volatile("xchgl %0,%1" + asm volatile("xchg %0,%1" : "=r" (newValue), "+m" (_q_value) : "0" (newValue) : "memory"); return newValue; } -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddOrdered(qptrdiff valueToAdd) +template<> template inline +T QBasicAtomicOps<1>::fetchAndStoreRelaxed(T &_q_value, T newValue) { - asm volatile("lock\n" - "xaddl %0,%1" - : "=r" (valueToAdd), "+m" (_q_value) - : "0" (valueToAdd * sizeof(T)) + asm volatile("xchg %0,%1" + : "=q" (newValue), "+m" (_q_value) + : "0" (newValue) : "memory"); - return reinterpret_cast(valueToAdd); + return newValue; } +template template inline +T QBasicAtomicOps::fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd) +{ + T result; + asm volatile("lock\n" + "xadd %0,%1" + : "=r" (result), "+m" (_q_value) + : "0" (valueToAdd * QAtomicAdditiveType::AddScale) + : "memory"); + return result; +} + +template<> template inline +T QBasicAtomicOps<1>::fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd) +{ + T result; + asm volatile("lock\n" + "xadd %0,%1" + : "=q" (result), "+m" (_q_value) + : "0" (valueToAdd * QAtomicAdditiveType::AddScale) + : "memory"); + return result; +} + +#define Q_ATOMIC_INT8_IS_SUPPORTED + +#define Q_ATOMIC_INT8_REFERENCE_COUNTING_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_REFERENCE_COUNTING_IS_WAIT_FREE + +#define Q_ATOMIC_INT8_TEST_AND_SET_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT8_FETCH_AND_STORE_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_FETCH_AND_STORE_IS_WAIT_FREE + +#define Q_ATOMIC_INT8_FETCH_AND_ADD_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_FETCH_AND_ADD_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_IS_SUPPORTED + +#define Q_ATOMIC_INT16_REFERENCE_COUNTING_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_REFERENCE_COUNTING_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_TEST_AND_SET_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_FETCH_AND_STORE_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_FETCH_AND_STORE_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_FETCH_AND_ADD_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_FETCH_AND_ADD_IS_WAIT_FREE + +template <> struct QBasicAtomicOps<8>: QGenericAtomicOps > +{ + static inline bool isTestAndSetNative() { return true; } + static inline bool isTestAndSetWaitFree() { return true; } + template static inline + bool testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) + { +#ifdef __PIC__ +# define EBX_reg "r" +# define EBX_load(reg) "xchg " reg ", %%ebx\n" +#else +# define EBX_reg "b" +# define EBX_load(reg) +#endif + unsigned char ret; + asm volatile(EBX_load("%3") + "lock\n" + "cmpxchg8b %0\n" + EBX_load("%3") + "sete %1\n" + : "+m" (_q_value), "=qm" (ret), + "+A" (expectedValue) + : EBX_reg (quint32(newValue & 0xffffffff)), "c" (quint32(newValue >> 32)) + : "memory"); + return ret != 0; +#undef EBX_reg +#undef EBX_load + } +}; +#define Q_ATOMIC_INT64_IS_SUPPORTED + +#define Q_ATOMIC_INT64_REFERENCE_COUNTING_IS_NOT_NATIVE + +#define Q_ATOMIC_INT64_TEST_AND_SET_IS_NOT_NATIVE +#define Q_ATOMIC_INT64_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT64_FETCH_AND_STORE_IS_NATIVE + +#define Q_ATOMIC_INT64_FETCH_AND_ADD_IS_NOT_NATIVE + #else extern "C" { @@ -211,150 +356,37 @@ extern "C" { Q_CORE_EXPORT void *q_atomic_fetch_and_add_ptr(volatile void *ptr, int value); } // extern "C" -inline bool QBasicAtomicInt::ref() +template<> template inline +bool QBasicAtomicOps<4>::ref(T &_q_value) { - return q_atomic_increment(&_q_value) != 0; + return q_atomic_increment((int *)&_q_value) != 0; } -inline bool QBasicAtomicInt::deref() +template<> template inline +bool QBasicAtomicOps<4>::deref(T &_q_value) { - return q_atomic_decrement(&_q_value) != 0; + return q_atomic_decrement((int *)&_q_value) != 0; } -inline bool QBasicAtomicInt::testAndSetOrdered(int expectedValue, int newValue) +template<> template inline +bool QBasicAtomicOps<4>::testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) { - return q_atomic_test_and_set_int(&_q_value, expectedValue, newValue) != 0; + return q_atomic_test_and_set_int((int*)&_q_value, int(expectedValue), int(newValue)); } -inline int QBasicAtomicInt::fetchAndStoreOrdered(int newValue) +template<> template inline +T QBasicAtomicOps<4>::fetchAndStoreRelaxed(T &_q_value, T newValue) { - return q_atomic_set_int(&_q_value, newValue); + return T(q_atomic_set_int((int*)&_q_value, int(newValue)); } -inline int QBasicAtomicInt::fetchAndAddOrdered(int valueToAdd) +template<> template inline +T QBasicAtomicOps<4>::fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd) { - return q_atomic_fetch_and_add_int(&_q_value, valueToAdd); + return T(q_atomic_fetch_and_add_int((int *)&_q_value, valueToAdd * QAtomicAdditiveType::AddScale)); } - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetOrdered(T *expectedValue, T *newValue) -{ - return q_atomic_test_and_set_ptr(&_q_value, expectedValue, newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreOrdered(T *newValue) -{ - return reinterpret_cast(q_atomic_set_ptr(&_q_value, newValue)); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddOrdered(qptrdiff valueToAdd) -{ - return reinterpret_cast(q_atomic_fetch_and_add_ptr(&_q_value, valueToAdd * sizeof(T))); -} - #endif -inline bool QBasicAtomicInt::testAndSetRelaxed(int expectedValue, int newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -inline bool QBasicAtomicInt::testAndSetAcquire(int expectedValue, int newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -inline bool QBasicAtomicInt::testAndSetRelease(int expectedValue, int newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -inline int QBasicAtomicInt::fetchAndStoreRelaxed(int newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -inline int QBasicAtomicInt::fetchAndStoreAcquire(int newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -inline int QBasicAtomicInt::fetchAndStoreRelease(int newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -inline int QBasicAtomicInt::fetchAndAddRelaxed(int valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -inline int QBasicAtomicInt::fetchAndAddAcquire(int valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -inline int QBasicAtomicInt::fetchAndAddRelease(int valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetRelaxed(T *expectedValue, T *newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetAcquire(T *expectedValue, T *newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetRelease(T *expectedValue, T *newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreRelaxed(T *newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreAcquire(T *newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreRelease(T *newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddRelaxed(qptrdiff valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddAcquire(qptrdiff valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddRelease(qptrdiff valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - QT_END_NAMESPACE QT_END_HEADER diff --git a/src/corelib/arch/qatomic_x86_64.h b/src/corelib/arch/qatomic_x86_64.h index 418248a43e..342cd12294 100644 --- a/src/corelib/arch/qatomic_x86_64.h +++ b/src/corelib/arch/qatomic_x86_64.h @@ -1,7 +1,7 @@ /**************************************************************************** ** ** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). -** All rights reserved. +** Copyright (C) 2011 Thiago Macieira ** Contact: http://www.qt-project.org/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -42,75 +42,120 @@ #ifndef QATOMIC_X86_64_H #define QATOMIC_X86_64_H +#include + QT_BEGIN_HEADER QT_BEGIN_NAMESPACE +#if 0 +#pragma qt_sync_stop_processing +#endif + #define Q_ATOMIC_INT_REFERENCE_COUNTING_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_REFERENCE_COUNTING_IS_WAIT_FREE -inline bool QBasicAtomicInt::isReferenceCountingNative() -{ return true; } -inline bool QBasicAtomicInt::isReferenceCountingWaitFree() -{ return true; } - #define Q_ATOMIC_INT_TEST_AND_SET_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_TEST_AND_SET_IS_WAIT_FREE -inline bool QBasicAtomicInt::isTestAndSetNative() -{ return true; } -inline bool QBasicAtomicInt::isTestAndSetWaitFree() -{ return true; } - #define Q_ATOMIC_INT_FETCH_AND_STORE_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_FETCH_AND_STORE_IS_WAIT_FREE -inline bool QBasicAtomicInt::isFetchAndStoreNative() -{ return true; } -inline bool QBasicAtomicInt::isFetchAndStoreWaitFree() -{ return true; } - #define Q_ATOMIC_INT_FETCH_AND_ADD_IS_ALWAYS_NATIVE #define Q_ATOMIC_INT_FETCH_AND_ADD_IS_WAIT_FREE -inline bool QBasicAtomicInt::isFetchAndAddNative() -{ return true; } -inline bool QBasicAtomicInt::isFetchAndAddWaitFree() -{ return true; } +#define Q_ATOMIC_INT32_IS_SUPPORTED + +#define Q_ATOMIC_INT32_REFERENCE_COUNTING_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_REFERENCE_COUNTING_IS_WAIT_FREE + +#define Q_ATOMIC_INT32_TEST_AND_SET_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT32_FETCH_AND_STORE_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_FETCH_AND_STORE_IS_WAIT_FREE + +#define Q_ATOMIC_INT32_FETCH_AND_ADD_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT32_FETCH_AND_ADD_IS_WAIT_FREE #define Q_ATOMIC_POINTER_TEST_AND_SET_IS_ALWAYS_NATIVE #define Q_ATOMIC_POINTER_TEST_AND_SET_IS_WAIT_FREE -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isTestAndSetNative() -{ return true; } -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isTestAndSetWaitFree() -{ return true; } - #define Q_ATOMIC_POINTER_FETCH_AND_STORE_IS_ALWAYS_NATIVE #define Q_ATOMIC_POINTER_FETCH_AND_STORE_IS_WAIT_FREE -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndStoreNative() -{ return true; } -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndStoreWaitFree() -{ return true; } - #define Q_ATOMIC_POINTER_FETCH_AND_ADD_IS_ALWAYS_NATIVE #define Q_ATOMIC_POINTER_FETCH_AND_ADD_IS_WAIT_FREE -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndAddNative() -{ return true; } -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::isFetchAndAddWaitFree() -{ return true; } +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; + +template struct QBasicAtomicOps: QGenericAtomicOps > +{ + static inline bool isReferenceCountingNative() { return true; } + static inline bool isReferenceCountingWaitFree() { return true; } + template static bool ref(T &_q_value); + template static bool deref(T &_q_value); + + static inline bool isTestAndSetNative() { return true; } + static inline bool isTestAndSetWaitFree() { return true; } + template static bool testAndSetRelaxed(T &_q_value, T expectedValue, T newValue); + + static inline bool isFetchAndStoreNative() { return true; } + static inline bool isFetchAndStoreWaitFree() { return true; } + template static T fetchAndStoreRelaxed(T &_q_value, T newValue); + + static inline bool isFetchAndAddNative() { return true; } + static inline bool isFetchAndAddWaitFree() { return true; } + template static + T fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd); +}; + +template struct QAtomicOps : QBasicAtomicOps +{ + typedef T Type; +}; #if defined(Q_CC_GNU) || defined(Q_CC_INTEL) -inline bool QBasicAtomicInt::ref() +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; +template<> struct QAtomicIntegerTraits { enum { IsInteger = 1 }; }; + +template<> template inline +bool QBasicAtomicOps<1>::ref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "incb %0\n" + "setne %1" + : "=m" (_q_value), "=qm" (ret) + : "m" (_q_value) + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<2>::ref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "incw %0\n" + "setne %1" + : "=m" (_q_value), "=qm" (ret) + : "m" (_q_value) + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<4>::ref(T &_q_value) { unsigned char ret; asm volatile("lock\n" @@ -122,7 +167,46 @@ inline bool QBasicAtomicInt::ref() return ret != 0; } -inline bool QBasicAtomicInt::deref() +template<> template inline +bool QBasicAtomicOps<8>::ref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "incq %0\n" + "setne %1" + : "=m" (_q_value), "=qm" (ret) + : "m" (_q_value) + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<1>::deref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "decb %0\n" + "setne %1" + : "=m" (_q_value), "=qm" (ret) + : "m" (_q_value) + : "memory"); + return ret != 0; +} + +template<> template inline +bool QBasicAtomicOps<2>::deref(T &_q_value) +{ + unsigned char ret; + asm volatile("lock\n" + "decw %0\n" + "setne %1" + : "=m" (_q_value), "=qm" (ret) + : "m" (_q_value) + : "memory"); + return ret != 0; +} +template<> template inline +bool QBasicAtomicOps<4>::deref(T &_q_value) { unsigned char ret; asm volatile("lock\n" @@ -134,11 +218,25 @@ inline bool QBasicAtomicInt::deref() return ret != 0; } -inline bool QBasicAtomicInt::testAndSetOrdered(int expectedValue, int newValue) +template<> template inline +bool QBasicAtomicOps<8>::deref(T &_q_value) { unsigned char ret; asm volatile("lock\n" - "cmpxchgl %3,%2\n" + "decq %0\n" + "setne %1" + : "=m" (_q_value), "=qm" (ret) + : "m" (_q_value) + : "memory"); + return ret != 0; +} + +template template inline +bool QBasicAtomicOps::testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) +{ + unsigned char ret; + asm volatile("lock\n" + "cmpxchg %3,%2\n" "sete %1\n" : "=a" (newValue), "=qm" (ret), "+m" (_q_value) : "r" (newValue), "0" (expectedValue) @@ -146,59 +244,105 @@ inline bool QBasicAtomicInt::testAndSetOrdered(int expectedValue, int newValue) return ret != 0; } -inline int QBasicAtomicInt::fetchAndStoreOrdered(int newValue) -{ - asm volatile("xchgl %0,%1" - : "=r" (newValue), "+m" (_q_value) - : "0" (newValue) - : "memory"); - return newValue; -} - -inline int QBasicAtomicInt::fetchAndAddOrdered(int valueToAdd) -{ - asm volatile("lock\n" - "xaddl %0,%1" - : "=r" (valueToAdd), "+m" (_q_value) - : "0" (valueToAdd) - : "memory"); - return valueToAdd; -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetOrdered(T *expectedValue, T *newValue) +template<> template inline +bool QBasicAtomicOps<1>::testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) { unsigned char ret; asm volatile("lock\n" - "cmpxchgq %3,%2\n" + "cmpxchg %3,%2\n" "sete %1\n" : "=a" (newValue), "=qm" (ret), "+m" (_q_value) - : "r" (newValue), "0" (expectedValue) + : "q" (newValue), "0" (expectedValue) : "memory"); return ret != 0; } -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreOrdered(T *newValue) +template template inline +T QBasicAtomicOps::fetchAndStoreRelaxed(T &_q_value, T newValue) { - asm volatile("xchgq %0,%1" + asm volatile("xchg %0,%1" : "=r" (newValue), "+m" (_q_value) : "0" (newValue) : "memory"); return newValue; } -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddOrdered(qptrdiff valueToAdd) +template<> template inline +T QBasicAtomicOps<1>::fetchAndStoreRelaxed(T &_q_value, T newValue) { - asm volatile("lock\n" - "xaddq %0,%1" - : "=r" (valueToAdd), "+m" (_q_value) - : "0" (valueToAdd * sizeof(T)) + asm volatile("xchg %0,%1" + : "=q" (newValue), "+m" (_q_value) + : "0" (newValue) : "memory"); - return reinterpret_cast(valueToAdd); + return newValue; } +template template inline +T QBasicAtomicOps::fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd) +{ + T result; + asm volatile("lock\n" + "xadd %0,%1" + : "=r" (result), "+m" (_q_value) + : "0" (valueToAdd * QAtomicAdditiveType::AddScale) + : "memory"); + return result; +} + +template<> template inline +T QBasicAtomicOps<1>::fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd) +{ + T result; + asm volatile("lock\n" + "xadd %0,%1" + : "=q" (result), "+m" (_q_value) + : "0" (valueToAdd * QAtomicAdditiveType::AddScale) + : "memory"); + return result; +} + +#define Q_ATOMIC_INT8_IS_SUPPORTED + +#define Q_ATOMIC_INT8_REFERENCE_COUNTING_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_REFERENCE_COUNTING_IS_WAIT_FREE + +#define Q_ATOMIC_INT8_TEST_AND_SET_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT8_FETCH_AND_STORE_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_FETCH_AND_STORE_IS_WAIT_FREE + +#define Q_ATOMIC_INT8_FETCH_AND_ADD_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT8_FETCH_AND_ADD_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_IS_SUPPORTED + +#define Q_ATOMIC_INT16_REFERENCE_COUNTING_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_REFERENCE_COUNTING_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_TEST_AND_SET_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_FETCH_AND_STORE_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_FETCH_AND_STORE_IS_WAIT_FREE + +#define Q_ATOMIC_INT16_FETCH_AND_ADD_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT16_FETCH_AND_ADD_IS_WAIT_FREE + +#define Q_ATOMIC_INT64_IS_SUPPORTED + +#define Q_ATOMIC_INT64_REFERENCE_COUNTING_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT64_REFERENCE_COUNTING_IS_WAIT_FREE + +#define Q_ATOMIC_INT64_TEST_AND_SET_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT64_TEST_AND_SET_IS_WAIT_FREE + +#define Q_ATOMIC_INT64_FETCH_AND_STORE_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT64_FETCH_AND_STORE_IS_WAIT_FREE + +#define Q_ATOMIC_INT64_FETCH_AND_ADD_IS_ALWAYS_NATIVE +#define Q_ATOMIC_INT64_FETCH_AND_ADD_IS_WAIT_FREE + #else // !Q_CC_INTEL && !Q_CC_GNU extern "C" { @@ -212,150 +356,56 @@ extern "C" { Q_CORE_EXPORT void *q_atomic_fetch_and_add_ptr(volatile void *ptr, qptrdiff value); } // extern "C" -inline bool QBasicAtomicInt::ref() +template<> template inline +bool QBasicAtomicOps<4>::ref(T &_q_value) { - return q_atomic_increment(&_q_value) != 0; + return q_atomic_increment((int *)&_q_value) != 0; } -inline bool QBasicAtomicInt::deref() +template<> template inline +bool QBasicAtomicOps<4>::deref(T &_q_value) { - return q_atomic_decrement(&_q_value) != 0; + return q_atomic_decrement((int *)&_q_value) != 0; } -inline bool QBasicAtomicInt::testAndSetOrdered(int expected, int newval) +template<> template inline +bool QBasicAtomicOps<4>::testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) { - return q_atomic_test_and_set_int(&_q_value, expected, newval) != 0; + return q_atomic_test_and_set_int((int*)&_q_value, int(expectedValue), int(newValue)); } -inline int QBasicAtomicInt::fetchAndStoreOrdered(int newval) +template<> template inline +T QBasicAtomicOps<4>::fetchAndStoreRelaxed(T &_q_value, T newValue) { - return q_atomic_set_int(&_q_value, newval); + return T(q_atomic_set_int((int*)&_q_value, int(newValue)); } -inline int QBasicAtomicInt::fetchAndAddOrdered(int aValue) +template<> template inline +T QBasicAtomicOps<4>::fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd) { - return q_atomic_fetch_and_add_int(&_q_value, aValue); + return T(q_atomic_fetch_and_add_int((int *)&_q_value, valueToAdd * QAtomicAdditiveType::AddScale)); } -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetOrdered(T *expectedValue, T *newValue) +template<> template inline +bool QBasicAtomicOps<8>::testAndSetRelaxed(T &_q_value, T expectedValue, T newValue) { - return q_atomic_test_and_set_ptr(&_q_value, expectedValue, newValue); + return q_atomic_test_and_set_ptr(&_q_value, (void*)expectedValue, (void*)newValue); } -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreOrdered(T *newValue) +template<> template inline +T QBasicAtomicOps<8>::fetchAndStoreRelaxed(T &_q_value, T newValue) { - return reinterpret_cast(q_atomic_set_ptr(&_q_value, newValue)); + return T(q_atomic_set_ptr(&_q_value, (void*)newValue); } -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddOrdered(qptrdiff valueToAdd) +template<> template inline +T QBasicAtomicOps<8>::fetchAndAddRelaxed(T &_q_value, typename QAtomicAdditiveType::AdditiveT valueToAdd) { - return reinterpret_cast(q_atomic_fetch_and_add_ptr(&_q_value, valueToAdd * sizeof(T))); + return T(q_atomic_fetch_and_add_int(&_q_value, valueToAdd * QAtomicAdditiveType::AddScale)); } #endif // Q_CC_GNU || Q_CC_INTEL -inline bool QBasicAtomicInt::testAndSetRelaxed(int expectedValue, int newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -inline bool QBasicAtomicInt::testAndSetAcquire(int expectedValue, int newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -inline bool QBasicAtomicInt::testAndSetRelease(int expectedValue, int newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -inline int QBasicAtomicInt::fetchAndStoreRelaxed(int newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -inline int QBasicAtomicInt::fetchAndStoreAcquire(int newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -inline int QBasicAtomicInt::fetchAndStoreRelease(int newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -inline int QBasicAtomicInt::fetchAndAddRelaxed(int valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -inline int QBasicAtomicInt::fetchAndAddAcquire(int valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -inline int QBasicAtomicInt::fetchAndAddRelease(int valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetRelaxed(T *expectedValue, T *newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetAcquire(T *expectedValue, T *newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -template -Q_INLINE_TEMPLATE bool QBasicAtomicPointer::testAndSetRelease(T *expectedValue, T *newValue) -{ - return testAndSetOrdered(expectedValue, newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreRelaxed(T *newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreAcquire(T *newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndStoreRelease(T *newValue) -{ - return fetchAndStoreOrdered(newValue); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddRelaxed(qptrdiff valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddAcquire(qptrdiff valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - -template -Q_INLINE_TEMPLATE T *QBasicAtomicPointer::fetchAndAddRelease(qptrdiff valueToAdd) -{ - return fetchAndAddOrdered(valueToAdd); -} - QT_END_NAMESPACE QT_END_HEADER diff --git a/src/corelib/thread/qbasicatomic.h b/src/corelib/thread/qbasicatomic.h index fd32c55ed0..1960361f0f 100644 --- a/src/corelib/thread/qbasicatomic.h +++ b/src/corelib/thread/qbasicatomic.h @@ -45,6 +45,13 @@ #if defined(QT_MOC) || defined(QT_BUILD_QMAKE) || defined(QT_RCC) || defined(QT_UIC) || defined(QT_BOOTSTRAPPED) # include +#elif defined(Q_CC_MSVC) + // not ported yet +# define QT_OLD_ATOMICS +#elif defined(__i386) || defined(__i386__) +# include +#elif defined(__x86_64) || defined(__x86_64__) || defined(__amd64) +# include #else # define QT_OLD_ATOMICS #endif