bionic: Do not use <sys/atomics.h> for platform code.
We're going to modify the __atomic_xxx implementation to provide
full memory barriers, to avoid problems for NDK machine code that
link to these functions.
First step is to remove their usage from our platform code.
We now use inlined versions of the same functions for a slight
performance boost.
+ remove obsolete atomics_x86.c (was never compiled)
NOTE: This improvement was benchmarked on various devices.
Comparing a pthread mutex lock + atomic increment + unlock
we get:
- ARMv7 emulator, running on a 2.4 GHz Xeon:
before: 396 ns after: 288 ns
- x86 emulator in KVM mode on same machine:
before: 27 ns after: 27 ns
- Google Nexus S, in ARMv7 mode (single-core):
before: 82 ns after: 76 ns
- Motorola Xoom, in ARMv7 mode (multi-core):
before: 121 ns after: 120 ns
The code has also been rebuilt in ARMv5TE mode for correctness.
Change-Id: Ic1dc72b173d59b2e7af901dd70d6a72fb2f64b17
diff --git a/libc/bionic/atomics_x86.c b/libc/bionic/atomics_x86.c
deleted file mode 100644
index fd60f4f..0000000
--- a/libc/bionic/atomics_x86.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright (C) 2008 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-#include <sys/atomics.h>
-
-#define FUTEX_SYSCALL 240
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-
-int __futex_wait(volatile void *ftx, int val)
-{
- int ret;
- asm volatile (
- "int $0x80;"
- : "=a" (ret)
- : "0" (FUTEX_SYSCALL),
- "b" (ftx),
- "c" (FUTEX_WAIT),
- "d" (val),
- "S" (0)
- );
- return ret;
-}
-
-int __futex_wake(volatile void *ftx, int count)
-{
- int ret;
- asm volatile (
- "int $0x80;"
- : "=a" (ret)
- : "0" (FUTEX_SYSCALL),
- "b" (ftx),
- "c" (FUTEX_WAKE),
- "d" (count)
- );
- return ret;
-}
-
-int __atomic_cmpxchg(int old, int new, volatile int* addr) {
- int xchg;
- asm volatile (
- "lock;"
- "cmpxchg %%ecx, (%%edx);"
- "setne %%al;"
- : "=a" (xchg)
- : "a" (old),
- "c" (new),
- "d" (addr)
- );
- return xchg;
-}
-
-int __atomic_swap(int new, volatile int* addr) {
- int old;
- asm volatile (
- "lock;"
- "xchg %%ecx, (%%edx);"
- : "=c" (old)
- : "c" (new),
- "d" (addr)
- );
- return old;
-}
-
-int __atomic_dec(volatile int* addr) {
- int old;
- do {
- old = *addr;
- } while (atomic_cmpxchg(old, old-1, addr));
- return old;
-}
diff --git a/libc/bionic/pthread.c b/libc/bionic/pthread.c
index b893a12..b56822f 100644
--- a/libc/bionic/pthread.c
+++ b/libc/bionic/pthread.c
@@ -692,7 +692,7 @@
goto Exit;
}
}
- while ( __atomic_cmpxchg( flags, flags | PTHREAD_ATTR_FLAG_DETACHED,
+ while ( __bionic_cmpxchg( flags, flags | PTHREAD_ATTR_FLAG_DETACHED,
(volatile int*)&thread->attr.flags ) != 0 );
Exit:
pthread_mutex_unlock(&gThreadListLock);
@@ -926,17 +926,17 @@
int shared = mutex->value & MUTEX_SHARED_MASK;
/*
* The common case is an unlocked mutex, so we begin by trying to
- * change the lock's state from 0 to 1. __atomic_cmpxchg() returns 0
+ * change the lock's state from 0 to 1. __bionic_cmpxchg() returns 0
* if it made the swap successfully. If the result is nonzero, this
* lock is already held by another thread.
*/
- if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value ) != 0) {
+ if (__bionic_cmpxchg(shared|0, shared|1, &mutex->value ) != 0) {
/*
* We want to go to sleep until the mutex is available, which
* requires promoting it to state 2. We need to swap in the new
* state value and then wait until somebody wakes us up.
*
- * __atomic_swap() returns the previous value. We swap 2 in and
+ * __bionic_swap() returns the previous value. We swap 2 in and
* see if we got zero back; if so, we have acquired the lock. If
* not, another thread still holds the lock and we wait again.
*
@@ -947,7 +947,7 @@
* that the mutex is in state 2 when we go to sleep on it, which
* guarantees a wake-up call.
*/
- while (__atomic_swap(shared|2, &mutex->value ) != (shared|0))
+ while (__bionic_swap(shared|2, &mutex->value ) != (shared|0))
__futex_wait_ex(&mutex->value, shared, shared|2, 0);
}
ANDROID_MEMBAR_FULL();
@@ -967,10 +967,10 @@
/*
* The mutex state will be 1 or (rarely) 2. We use an atomic decrement
- * to release the lock. __atomic_dec() returns the previous value;
+ * to release the lock. __bionic_atomic_dec() returns the previous value;
* if it wasn't 1 we have to do some additional work.
*/
- if (__atomic_dec(&mutex->value) != (shared|1)) {
+ if (__bionic_atomic_dec(&mutex->value) != (shared|1)) {
/*
* Start by releasing the lock. The decrement changed it from
* "contended lock" to "uncontended lock", which means we still
@@ -1158,7 +1158,7 @@
/* Handle common case first */
if ( __likely(mtype == MUTEX_TYPE_NORMAL) )
{
- if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value) == 0) {
+ if (__bionic_cmpxchg(shared|0, shared|1, &mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
@@ -1256,13 +1256,13 @@
if ( __likely(mtype == MUTEX_TYPE_NORMAL) )
{
/* fast path for uncontended lock */
- if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value) == 0) {
+ if (__bionic_cmpxchg(shared|0, shared|1, &mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
/* loop while needed */
- while (__atomic_swap(shared|2, &mutex->value) != (shared|0)) {
+ while (__bionic_swap(shared|2, &mutex->value) != (shared|0)) {
if (__timespec_to_absolute(&ts, &abstime, clock) < 0)
return EBUSY;
@@ -1431,7 +1431,7 @@
long oldval = cond->value;
long newval = ((oldval - COND_COUNTER_INCREMENT) & COND_COUNTER_MASK)
| flags;
- if (__atomic_cmpxchg(oldval, newval, &cond->value) == 0)
+ if (__bionic_cmpxchg(oldval, newval, &cond->value) == 0)
break;
}
diff --git a/libc/bionic/semaphore.c b/libc/bionic/semaphore.c
index 96819ae..9bc8412 100644
--- a/libc/bionic/semaphore.c
+++ b/libc/bionic/semaphore.c
@@ -174,7 +174,7 @@
new = SEMCOUNT_DECREMENT(old);
}
- while (__atomic_cmpxchg((int)(old|shared),
+ while (__bionic_cmpxchg((int)(old|shared),
(int)(new|shared),
(volatile int *)pvalue) != 0);
return ret;
@@ -198,7 +198,7 @@
new = SEMCOUNT_DECREMENT(old);
}
- while (__atomic_cmpxchg((int)(old|shared),
+ while (__bionic_cmpxchg((int)(old|shared),
(int)(new|shared),
(volatile int *)pvalue) != 0);
@@ -235,7 +235,7 @@
else
new = SEMCOUNT_INCREMENT(old);
}
- while ( __atomic_cmpxchg((int)(old|shared),
+ while ( __bionic_cmpxchg((int)(old|shared),
(int)(new|shared),
(volatile int*)pvalue) != 0);