Inflate contended lock word by suspending owner.
Bug 6961405.
Don't inflate monitors for Notify and NotifyAll.
Tidy lock word, handle recursive lock case alongside unlocked case and move
assembly out of line (except for ARM quick). Also handle null in out-of-line
assembly as the test is quick and the enter/exit code is already a safepoint.
To gain ownership of a monitor on behalf of another thread, monitor contenders
must not hold the monitor_lock_, so they wait on a condition variable.
Reduce size of per mutex contention log.
Be consistent in calling thin lock thread ids just thread ids.
Fix potential thread death races caused by the use of FindThreadByThreadId,
make it invariant that returned threads are either self or suspended now.
Code size reduction on ARM boot.oat 0.2%.
Old nexus 7 speedup 0.25%, new nexus 7 speedup 1.4%, nexus 10 speedup 2.24%,
nexus 4 speedup 2.09% on DeltaBlue.
Change-Id: Id52558b914f160d9c8578fdd7fc8199a9598576a
diff --git a/runtime/arch/arm/asm_support_arm.h b/runtime/arch/arm/asm_support_arm.h
index ed3d476..69fb9c3 100644
--- a/runtime/arch/arm/asm_support_arm.h
+++ b/runtime/arch/arm/asm_support_arm.h
@@ -27,5 +27,7 @@
#define THREAD_FLAGS_OFFSET 0
// Offset of field Thread::exception_ verified in InitCpu
#define THREAD_EXCEPTION_OFFSET 12
+// Offset of field Thread::thin_lock_thread_id_ verified in InitCpu
+#define THREAD_ID_OFFSET 60
#endif // ART_RUNTIME_ARCH_ARM_ASM_SUPPORT_ARM_H_
diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S
index 5b2dd6c..cb61698 100644
--- a/runtime/arch/arm/quick_entrypoints_arm.S
+++ b/runtime/arch/arm/quick_entrypoints_arm.S
@@ -318,22 +318,67 @@
END art_quick_handle_fill_data
/*
- * Entry from managed code that calls artLockObjectFromCode, may block for GC.
+ * Entry from managed code that calls artLockObjectFromCode, may block for GC. r0 holds the
+ * possibly null object to lock.
*/
.extern artLockObjectFromCode
ENTRY art_quick_lock_object
+ cbz r0, slow_lock
+retry_lock:
+ ldrex r1, [r0, #LOCK_WORD_OFFSET]
+ ldrt r2, [r9, #THREAD_ID_OFFSET]
+ cmp r1, #0
+ bmi slow_lock @ lock word contains a monitor
+ bne already_thin
+ @ unlocked case - r2 holds thread id with count of 0
+ strex r3, r2, [r0, #LOCK_WORD_OFFSET]
+ cbnz r3, strex_fail @ store failed, retry
+ bx lr
+strex_fail:
+ b retry_lock @ unlikely forward branch, need to reload and recheck r1/r2
+already_thin:
+ eor r2, r1, r2 @ lock_word.ThreadId() ^ self->ThreadId()
+ uxth r2, r2 @ zero top 16 bits
+ cbnz r2, slow_lock @ lock word and self thread id's match -> recursive lock
+ @ else contention, go to slow path
+ adds r2, r1, #65536 @ increment count in lock word placing in r2 for storing
+ bmi slow_lock @ if we overflow the count go slow
+ str r2, [r0, #LOCK_WORD_OFFSET] @ no need for strex as we hold the lock
+ bx lr
+slow_lock:
SETUP_REF_ONLY_CALLEE_SAVE_FRAME @ save callee saves in case we block
mov r1, r9 @ pass Thread::Current
mov r2, sp @ pass SP
bl artLockObjectFromCode @ (Object* obj, Thread*, SP)
- RESTORE_REF_ONLY_CALLEE_SAVE_FRAME_AND_RETURN
+ RESTORE_REF_ONLY_CALLEE_SAVE_FRAME
+ RETURN_IF_RESULT_IS_ZERO
+ DELIVER_PENDING_EXCEPTION
END art_quick_lock_object
/*
* Entry from managed code that calls artUnlockObjectFromCode and delivers exception on failure.
+ * r0 holds the possibly null object to lock.
*/
.extern artUnlockObjectFromCode
ENTRY art_quick_unlock_object
+ cbz r0, slow_unlock
+ ldr r1, [r0, #LOCK_WORD_OFFSET]
+ ldr r2, [r9, #THREAD_ID_OFFSET]
+ cmp r1, #0
+ bmi slow_unlock @ lock word contains a monitor
+ eor r3, r1, r2 @ lock_word.ThreadId() ^ self->ThreadId()
+ uxth r3, r3 @ zero top 16 bits
+ cbnz r3, slow_unlock @ do lock word and self thread id's match?
+ cmp r1, #65536
+ bpl recursive_thin_unlock
+ @ transition to unlocked, r3 holds 0
+ str r3, [r0, #LOCK_WORD_OFFSET]
+ bx lr
+recursive_thin_unlock:
+ sub r1, r1, #65536
+ str r1, [r0, #LOCK_WORD_OFFSET]
+ bx lr
+slow_unlock:
SETUP_REF_ONLY_CALLEE_SAVE_FRAME @ save callee saves in case exception allocation triggers GC
mov r1, r9 @ pass Thread::Current
mov r2, sp @ pass SP
diff --git a/runtime/arch/arm/thread_arm.cc b/runtime/arch/arm/thread_arm.cc
index ea908be..75eef60 100644
--- a/runtime/arch/arm/thread_arm.cc
+++ b/runtime/arch/arm/thread_arm.cc
@@ -24,6 +24,7 @@
void Thread::InitCpu() {
CHECK_EQ(THREAD_FLAGS_OFFSET, OFFSETOF_MEMBER(Thread, state_and_flags_));
CHECK_EQ(THREAD_EXCEPTION_OFFSET, OFFSETOF_MEMBER(Thread, exception_));
+ CHECK_EQ(THREAD_ID_OFFSET, OFFSETOF_MEMBER(Thread, thin_lock_thread_id_));
}
} // namespace art
diff --git a/runtime/arch/x86/asm_support_x86.h b/runtime/arch/x86/asm_support_x86.h
index 1092910..d4e0927 100644
--- a/runtime/arch/x86/asm_support_x86.h
+++ b/runtime/arch/x86/asm_support_x86.h
@@ -23,5 +23,7 @@
#define THREAD_SELF_OFFSET 40
// Offset of field Thread::exception_ verified in InitCpu
#define THREAD_EXCEPTION_OFFSET 12
+// Offset of field Thread::thin_lock_thread_id_ verified in InitCpu
+#define THREAD_ID_OFFSET 60
#endif // ART_RUNTIME_ARCH_X86_ASM_SUPPORT_X86_H_
diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S
index 06b2203..6be73d1 100644
--- a/runtime/arch/x86/quick_entrypoints_x86.S
+++ b/runtime/arch/x86/quick_entrypoints_x86.S
@@ -401,14 +401,85 @@
TWO_ARG_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode, RETURN_IF_EAX_NOT_ZERO
TWO_ARG_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode, RETURN_IF_EAX_NOT_ZERO
-ONE_ARG_DOWNCALL art_quick_lock_object, artLockObjectFromCode, ret
-ONE_ARG_DOWNCALL art_quick_unlock_object, artUnlockObjectFromCode, RETURN_IF_EAX_ZERO
-
TWO_ARG_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_EAX_ZERO
+DEFINE_FUNCTION art_quick_lock_object
+ testl %eax, %eax // null check object/eax
+ jz slow_lock
+retry_lock:
+ movl LOCK_WORD_OFFSET(%eax), %ecx // ecx := lock word
+ movl %fs:THREAD_ID_OFFSET, %edx // edx := thread id
+ test %ecx, %ecx
+ jb slow_lock // lock word contains a monitor
+ jnz already_thin // lock word contains a thin lock
+ // unlocked case - %edx holds thread id with count of 0
+ movl %eax, %ecx // remember object in case of retry
+ xor %eax, %eax // eax == 0 for comparison with lock word in cmpxchg
+ lock cmpxchg %edx, LOCK_WORD_OFFSET(%ecx)
+ jnz cmpxchg_fail // cmpxchg failed retry
+ ret
+cmpxchg_fail:
+ movl %ecx, %eax // restore eax
+ jmp retry_lock
+already_thin:
+ cmpw %ax, %dx // do we hold the lock already?
+ jne slow_lock
+ addl LITERAL(65536), %eax // increment recursion count
+ jb slow_lock // count overflowed so go slow
+ movl %eax, LOCK_WORD_OFFSET(%ecx) // update lockword, cmpxchg not necessary as we hold lock
+ ret
+slow_lock:
+ SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save ref containing registers for GC
+ mov %esp, %edx // remember SP
+ // Outgoing argument set up
+ PUSH eax // push padding
+ PUSH edx // pass SP
+ pushl %fs:THREAD_SELF_OFFSET // pass Thread::Current()
+ .cfi_adjust_cfa_offset 4
+ PUSH eax // pass object
+ call artLockObjectFromCode // artLockObjectFromCode(object, Thread*, SP)
+ addl MACRO_LITERAL(16), %esp // pop arguments
+ .cfi_adjust_cfa_offset -16
+ RESTORE_REF_ONLY_CALLEE_SAVE_FRAME // restore frame up to return address
+ RETURN_IF_EAX_ZERO
+END_FUNCTION art_quick_lock_object
+
+DEFINE_FUNCTION art_quick_unlock_object
+ testl %eax, %eax // null check object/eax
+ jz slow_unlock
+ movl LOCK_WORD_OFFSET(%eax), %ecx // ecx := lock word
+ movl %fs:THREAD_ID_OFFSET, %edx // edx := thread id
+ test %ecx, %ecx
+ jb slow_unlock // lock word contains a monitor
+ cmpw %cx, %dx // does the thread id match?
+ jne slow_unlock
+ cmpl LITERAL(65536), %ecx
+ jae recursive_thin_unlock
+ movl LITERAL(0), LOCK_WORD_OFFSET(%eax)
+ ret
+recursive_thin_unlock:
+ subl LITERAL(65536), %ecx
+ mov %ecx, LOCK_WORD_OFFSET(%eax)
+ ret
+slow_unlock:
+ SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save ref containing registers for GC
+ mov %esp, %edx // remember SP
+ // Outgoing argument set up
+ PUSH eax // push padding
+ PUSH edx // pass SP
+ pushl %fs:THREAD_SELF_OFFSET // pass Thread::Current()
+ .cfi_adjust_cfa_offset 4
+ PUSH eax // pass object
+ call artUnlockObjectFromCode // artUnlockObjectFromCode(object, Thread*, SP)
+ addl MACRO_LITERAL(16), %esp // pop arguments
+ .cfi_adjust_cfa_offset -16
+ RESTORE_REF_ONLY_CALLEE_SAVE_FRAME // restore frame up to return address
+ RETURN_IF_EAX_ZERO
+END_FUNCTION art_quick_unlock_object
+
DEFINE_FUNCTION art_quick_is_assignable
PUSH eax // alignment padding
- PUSH ecx // pass arg2
+ PUSH ecx // pass arg2
PUSH eax // pass arg1
call SYMBOL(artIsAssignableFromCode) // (Class* a, Class* b, Thread*, SP)
addl LITERAL(12), %esp // pop arguments
diff --git a/runtime/arch/x86/thread_x86.cc b/runtime/arch/x86/thread_x86.cc
index dd3e7dd..7e0aee0 100644
--- a/runtime/arch/x86/thread_x86.cc
+++ b/runtime/arch/x86/thread_x86.cc
@@ -134,6 +134,7 @@
// Sanity check other offsets.
CHECK_EQ(THREAD_EXCEPTION_OFFSET, OFFSETOF_MEMBER(Thread, exception_));
+ CHECK_EQ(THREAD_ID_OFFSET, OFFSETOF_MEMBER(Thread, thin_lock_thread_id_));
}
} // namespace art