blob: 823cb60d9717d825a933c56c4278ae1c6401ea5c [file] [log] [blame]
Matteo Franchin43ec8732014-03-31 15:00:14 +01001/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/* This file contains codegen for the Thumb2 ISA. */
18
Matteo Franchin43ec8732014-03-31 15:00:14 +010019#include "codegen_arm64.h"
Andreas Gampe0b9203e2015-01-22 20:39:27 -080020
21#include "arm64_lir.h"
22#include "base/logging.h"
23#include "dex/mir_graph.h"
Matteo Franchin43ec8732014-03-31 15:00:14 +010024#include "dex/quick/mir_to_lir-inl.h"
Andreas Gampe0b9203e2015-01-22 20:39:27 -080025#include "driver/compiler_driver.h"
Ian Rogers576ca0c2014-06-06 15:58:22 -070026#include "gc/accounting/card_table.h"
Matteo Franchin43ec8732014-03-31 15:00:14 +010027#include "entrypoints/quick/quick_entrypoints.h"
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +010028#include "mirror/art_method.h"
29#include "mirror/object_array-inl.h"
Matteo Franchin43ec8732014-03-31 15:00:14 +010030
31namespace art {
32
33/*
34 * The sparse table in the literal pool is an array of <key,displacement>
Matteo Franchine45fb9e2014-05-06 10:10:30 +010035 * pairs. For each set, we'll load them as a pair using ldp.
Matteo Franchin43ec8732014-03-31 15:00:14 +010036 * The test loop will look something like:
37 *
38 * adr r_base, <table>
Matteo Franchine45fb9e2014-05-06 10:10:30 +010039 * ldr r_val, [rA64_SP, v_reg_off]
Matteo Franchin43ec8732014-03-31 15:00:14 +010040 * mov r_idx, #table_size
Matteo Franchine45fb9e2014-05-06 10:10:30 +010041 * loop:
42 * cbz r_idx, quit
43 * ldp r_key, r_disp, [r_base], #8
Matteo Franchin43ec8732014-03-31 15:00:14 +010044 * sub r_idx, #1
45 * cmp r_val, r_key
Matteo Franchine45fb9e2014-05-06 10:10:30 +010046 * b.ne loop
47 * adr r_base, #0 ; This is the instruction from which we compute displacements
48 * add r_base, r_disp
49 * br r_base
50 * quit:
Matteo Franchin43ec8732014-03-31 15:00:14 +010051 */
Andreas Gampe48971b32014-08-06 10:09:01 -070052void Arm64Mir2Lir::GenLargeSparseSwitch(MIR* mir, uint32_t table_offset, RegLocation rl_src) {
Razvan A Lupusoru8d0d03e2014-06-06 17:04:52 -070053 const uint16_t* table = mir_graph_->GetTable(mir, table_offset);
Matteo Franchin43ec8732014-03-31 15:00:14 +010054 // Add the table to the list - we'll process it later
55 SwitchTable *tab_rec =
56 static_cast<SwitchTable*>(arena_->Alloc(sizeof(SwitchTable), kArenaAllocData));
Chao-ying Fu72f53af2014-11-11 16:48:40 -080057 tab_rec->switch_mir = mir;
Matteo Franchin43ec8732014-03-31 15:00:14 +010058 tab_rec->table = table;
59 tab_rec->vaddr = current_dalvik_offset_;
60 uint32_t size = table[1];
Vladimir Markoe39c54e2014-09-22 14:50:02 +010061 switch_tables_.push_back(tab_rec);
Matteo Franchin43ec8732014-03-31 15:00:14 +010062
63 // Get the switch value
64 rl_src = LoadValue(rl_src, kCoreReg);
Matteo Franchin5acc8b02014-06-05 15:10:35 +010065 RegStorage r_base = AllocTempWide();
Matteo Franchine45fb9e2014-05-06 10:10:30 +010066 // Allocate key and disp temps.
Matteo Franchin43ec8732014-03-31 15:00:14 +010067 RegStorage r_key = AllocTemp();
68 RegStorage r_disp = AllocTemp();
Matteo Franchin43ec8732014-03-31 15:00:14 +010069 // Materialize a pointer to the switch table
Matteo Franchine45fb9e2014-05-06 10:10:30 +010070 NewLIR3(kA64Adr2xd, r_base.GetReg(), 0, WrapPointer(tab_rec));
Matteo Franchin43ec8732014-03-31 15:00:14 +010071 // Set up r_idx
72 RegStorage r_idx = AllocTemp();
73 LoadConstant(r_idx, size);
Matteo Franchine45fb9e2014-05-06 10:10:30 +010074
75 // Entry of loop.
76 LIR* loop_entry = NewLIR0(kPseudoTargetLabel);
77 LIR* branch_out = NewLIR2(kA64Cbz2rt, r_idx.GetReg(), 0);
78
79 // Load next key/disp.
80 NewLIR4(kA64LdpPost4rrXD, r_key.GetReg(), r_disp.GetReg(), r_base.GetReg(), 2);
81 OpRegRegImm(kOpSub, r_idx, r_idx, 1);
82
83 // Go to next case, if key does not match.
Matteo Franchin43ec8732014-03-31 15:00:14 +010084 OpRegReg(kOpCmp, r_key, rl_src.reg);
Matteo Franchine45fb9e2014-05-06 10:10:30 +010085 OpCondBranch(kCondNe, loop_entry);
86
87 // Key does match: branch to case label.
88 LIR* switch_label = NewLIR3(kA64Adr2xd, r_base.GetReg(), 0, -1);
89 tab_rec->anchor = switch_label;
90
91 // Add displacement to base branch address and go!
Andreas Gampe47b31aa2014-06-19 01:10:07 -070092 OpRegRegRegExtend(kOpAdd, r_base, r_base, As64BitReg(r_disp), kA64Sxtw, 0U);
Matteo Franchine45fb9e2014-05-06 10:10:30 +010093 NewLIR1(kA64Br1x, r_base.GetReg());
94
95 // Loop exit label.
96 LIR* loop_exit = NewLIR0(kPseudoTargetLabel);
97 branch_out->target = loop_exit;
Matteo Franchin43ec8732014-03-31 15:00:14 +010098}
99
100
Andreas Gampe48971b32014-08-06 10:09:01 -0700101void Arm64Mir2Lir::GenLargePackedSwitch(MIR* mir, uint32_t table_offset, RegLocation rl_src) {
Razvan A Lupusoru8d0d03e2014-06-06 17:04:52 -0700102 const uint16_t* table = mir_graph_->GetTable(mir, table_offset);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100103 // Add the table to the list - we'll process it later
104 SwitchTable *tab_rec =
105 static_cast<SwitchTable*>(arena_->Alloc(sizeof(SwitchTable), kArenaAllocData));
Chao-ying Fu72f53af2014-11-11 16:48:40 -0800106 tab_rec->switch_mir = mir;
Matteo Franchin43ec8732014-03-31 15:00:14 +0100107 tab_rec->table = table;
108 tab_rec->vaddr = current_dalvik_offset_;
109 uint32_t size = table[1];
Vladimir Markoe39c54e2014-09-22 14:50:02 +0100110 switch_tables_.push_back(tab_rec);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100111
112 // Get the switch value
113 rl_src = LoadValue(rl_src, kCoreReg);
Matteo Franchin5acc8b02014-06-05 15:10:35 +0100114 RegStorage table_base = AllocTempWide();
Matteo Franchin43ec8732014-03-31 15:00:14 +0100115 // Materialize a pointer to the switch table
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100116 NewLIR3(kA64Adr2xd, table_base.GetReg(), 0, WrapPointer(tab_rec));
Matteo Franchin43ec8732014-03-31 15:00:14 +0100117 int low_key = s4FromSwitchData(&table[2]);
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100118 RegStorage key_reg;
Matteo Franchin43ec8732014-03-31 15:00:14 +0100119 // Remove the bias, if necessary
120 if (low_key == 0) {
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100121 key_reg = rl_src.reg;
Matteo Franchin43ec8732014-03-31 15:00:14 +0100122 } else {
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100123 key_reg = AllocTemp();
124 OpRegRegImm(kOpSub, key_reg, rl_src.reg, low_key);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100125 }
126 // Bounds check - if < 0 or >= size continue following switch
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100127 OpRegImm(kOpCmp, key_reg, size - 1);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100128 LIR* branch_over = OpCondBranch(kCondHi, NULL);
129
130 // Load the displacement from the switch table
131 RegStorage disp_reg = AllocTemp();
Andreas Gampe4b537a82014-06-30 22:24:53 -0700132 LoadBaseIndexed(table_base, As64BitReg(key_reg), disp_reg, 2, k32);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100133
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100134 // Get base branch address.
Matteo Franchin5acc8b02014-06-05 15:10:35 +0100135 RegStorage branch_reg = AllocTempWide();
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100136 LIR* switch_label = NewLIR3(kA64Adr2xd, branch_reg.GetReg(), 0, -1);
137 tab_rec->anchor = switch_label;
Matteo Franchin43ec8732014-03-31 15:00:14 +0100138
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100139 // Add displacement to base branch address and go!
Andreas Gampe47b31aa2014-06-19 01:10:07 -0700140 OpRegRegRegExtend(kOpAdd, branch_reg, branch_reg, As64BitReg(disp_reg), kA64Sxtw, 0U);
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100141 NewLIR1(kA64Br1x, branch_reg.GetReg());
142
143 // branch_over target here
Matteo Franchin43ec8732014-03-31 15:00:14 +0100144 LIR* target = NewLIR0(kPseudoTargetLabel);
145 branch_over->target = target;
146}
147
148/*
Matteo Franchin43ec8732014-03-31 15:00:14 +0100149 * Handle unlocked -> thin locked transition inline or else call out to quick entrypoint. For more
150 * details see monitor.cc.
151 */
152void Arm64Mir2Lir::GenMonitorEnter(int opt_flags, RegLocation rl_src) {
Zheng Xuc8304302014-05-15 17:21:01 +0100153 // x0/w0 = object
154 // w1 = thin lock thread id
155 // x2 = address of lock word
156 // w3 = lock word / store failure
157 // TUNING: How much performance we get when we inline this?
158 // Since we've already flush all register.
Matteo Franchin43ec8732014-03-31 15:00:14 +0100159 FlushAllRegs();
Andreas Gampeccc60262014-07-04 18:02:38 -0700160 LoadValueDirectFixed(rl_src, rs_x0); // = TargetReg(kArg0, kRef)
Matteo Franchin43ec8732014-03-31 15:00:14 +0100161 LockCallTemps(); // Prepare for explicit register usage
Zheng Xuc8304302014-05-15 17:21:01 +0100162 LIR* null_check_branch = nullptr;
163 if ((opt_flags & MIR_IGNORE_NULL_CHECK) && !(cu_->disable_opt & (1 << kNullCheckElimination))) {
164 null_check_branch = nullptr; // No null check.
Matteo Franchin43ec8732014-03-31 15:00:14 +0100165 } else {
Zheng Xuc8304302014-05-15 17:21:01 +0100166 // If the null-check fails its handled by the slow-path to reduce exception related meta-data.
Dave Allison69dfe512014-07-11 17:11:58 +0000167 if (!cu_->compiler_driver->GetCompilerOptions().GetImplicitNullChecks()) {
Zheng Xuc8304302014-05-15 17:21:01 +0100168 null_check_branch = OpCmpImmBranch(kCondEq, rs_x0, 0, NULL);
169 }
Matteo Franchin43ec8732014-03-31 15:00:14 +0100170 }
Zheng Xubaa7c882014-06-30 14:26:50 +0800171 Load32Disp(rs_xSELF, Thread::ThinLockIdOffset<8>().Int32Value(), rs_w1);
Zheng Xuc8304302014-05-15 17:21:01 +0100172 OpRegRegImm(kOpAdd, rs_x2, rs_x0, mirror::Object::MonitorOffset().Int32Value());
173 NewLIR2(kA64Ldxr2rX, rw3, rx2);
174 MarkPossibleNullPointerException(opt_flags);
Hiroshi Yamauchie15ea082015-02-09 17:11:42 -0800175 // Zero out the read barrier bits.
176 OpRegRegImm(kOpAnd, rs_w2, rs_w3, LockWord::kReadBarrierStateMaskShiftedToggled);
177 LIR* not_unlocked_branch = OpCmpImmBranch(kCondNe, rs_w2, 0, NULL);
178 // w3 is zero except for the rb bits here. Copy the read barrier bits into w1.
179 OpRegRegReg(kOpOr, rs_w1, rs_w1, rs_w3);
180 OpRegRegImm(kOpAdd, rs_x2, rs_x0, mirror::Object::MonitorOffset().Int32Value());
Zheng Xuc8304302014-05-15 17:21:01 +0100181 NewLIR3(kA64Stxr3wrX, rw3, rw1, rx2);
buzbee5d13f122014-08-19 16:47:06 -0700182 LIR* lock_success_branch = OpCmpImmBranch(kCondEq, rs_w3, 0, NULL);
Zheng Xuc8304302014-05-15 17:21:01 +0100183
184 LIR* slow_path_target = NewLIR0(kPseudoTargetLabel);
185 not_unlocked_branch->target = slow_path_target;
186 if (null_check_branch != nullptr) {
187 null_check_branch->target = slow_path_target;
188 }
189 // TODO: move to a slow path.
190 // Go expensive route - artLockObjectFromCode(obj);
Zheng Xubaa7c882014-06-30 14:26:50 +0800191 LoadWordDisp(rs_xSELF, QUICK_ENTRYPOINT_OFFSET(8, pLockObject).Int32Value(), rs_xLR);
Zheng Xuc8304302014-05-15 17:21:01 +0100192 ClobberCallerSave();
Zheng Xubaa7c882014-06-30 14:26:50 +0800193 LIR* call_inst = OpReg(kOpBlx, rs_xLR);
Zheng Xuc8304302014-05-15 17:21:01 +0100194 MarkSafepointPC(call_inst);
195
196 LIR* success_target = NewLIR0(kPseudoTargetLabel);
197 lock_success_branch->target = success_target;
Hans Boehm48f5c472014-06-27 14:50:10 -0700198 GenMemBarrier(kLoadAny);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100199}
200
201/*
202 * Handle thin locked -> unlocked transition inline or else call out to quick entrypoint. For more
Zheng Xuc8304302014-05-15 17:21:01 +0100203 * details see monitor.cc. Note the code below doesn't use ldxr/stxr as the code holds the lock
Matteo Franchin43ec8732014-03-31 15:00:14 +0100204 * and can only give away ownership if its suspended.
205 */
206void Arm64Mir2Lir::GenMonitorExit(int opt_flags, RegLocation rl_src) {
Zheng Xuc8304302014-05-15 17:21:01 +0100207 // x0/w0 = object
208 // w1 = thin lock thread id
209 // w2 = lock word
210 // TUNING: How much performance we get when we inline this?
211 // Since we've already flush all register.
Matteo Franchin43ec8732014-03-31 15:00:14 +0100212 FlushAllRegs();
Andreas Gampe4b537a82014-06-30 22:24:53 -0700213 LoadValueDirectFixed(rl_src, rs_x0); // Get obj
Matteo Franchin43ec8732014-03-31 15:00:14 +0100214 LockCallTemps(); // Prepare for explicit register usage
215 LIR* null_check_branch = nullptr;
Zheng Xuc8304302014-05-15 17:21:01 +0100216 if ((opt_flags & MIR_IGNORE_NULL_CHECK) && !(cu_->disable_opt & (1 << kNullCheckElimination))) {
217 null_check_branch = nullptr; // No null check.
Matteo Franchin43ec8732014-03-31 15:00:14 +0100218 } else {
Zheng Xuc8304302014-05-15 17:21:01 +0100219 // If the null-check fails its handled by the slow-path to reduce exception related meta-data.
Dave Allison69dfe512014-07-11 17:11:58 +0000220 if (!cu_->compiler_driver->GetCompilerOptions().GetImplicitNullChecks()) {
Zheng Xuc8304302014-05-15 17:21:01 +0100221 null_check_branch = OpCmpImmBranch(kCondEq, rs_x0, 0, NULL);
222 }
Matteo Franchin43ec8732014-03-31 15:00:14 +0100223 }
Zheng Xubaa7c882014-06-30 14:26:50 +0800224 Load32Disp(rs_xSELF, Thread::ThinLockIdOffset<8>().Int32Value(), rs_w1);
Hiroshi Yamauchie15ea082015-02-09 17:11:42 -0800225 if (!kUseReadBarrier) {
226 Load32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_w2);
227 } else {
228 OpRegRegImm(kOpAdd, rs_x3, rs_x0, mirror::Object::MonitorOffset().Int32Value());
229 NewLIR2(kA64Ldxr2rX, rw2, rx3);
230 }
Zheng Xuc8304302014-05-15 17:21:01 +0100231 MarkPossibleNullPointerException(opt_flags);
Hiroshi Yamauchie15ea082015-02-09 17:11:42 -0800232 // Zero out the read barrier bits.
233 OpRegRegImm(kOpAnd, rs_w3, rs_w2, LockWord::kReadBarrierStateMaskShiftedToggled);
234 // Zero out except the read barrier bits.
235 OpRegRegImm(kOpAnd, rs_w2, rs_w2, LockWord::kReadBarrierStateMaskShifted);
236 LIR* slow_unlock_branch = OpCmpBranch(kCondNe, rs_w3, rs_w1, NULL);
Hans Boehm48f5c472014-06-27 14:50:10 -0700237 GenMemBarrier(kAnyStore);
Hiroshi Yamauchie15ea082015-02-09 17:11:42 -0800238 LIR* unlock_success_branch;
239 if (!kUseReadBarrier) {
240 Store32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_w2);
241 unlock_success_branch = OpUnconditionalBranch(NULL);
242 } else {
243 OpRegRegImm(kOpAdd, rs_x3, rs_x0, mirror::Object::MonitorOffset().Int32Value());
244 NewLIR3(kA64Stxr3wrX, rw1, rw2, rx3);
245 unlock_success_branch = OpCmpImmBranch(kCondEq, rs_w1, 0, NULL);
246 }
Zheng Xuc8304302014-05-15 17:21:01 +0100247 LIR* slow_path_target = NewLIR0(kPseudoTargetLabel);
248 slow_unlock_branch->target = slow_path_target;
249 if (null_check_branch != nullptr) {
250 null_check_branch->target = slow_path_target;
251 }
252 // TODO: move to a slow path.
253 // Go expensive route - artUnlockObjectFromCode(obj);
Zheng Xubaa7c882014-06-30 14:26:50 +0800254 LoadWordDisp(rs_xSELF, QUICK_ENTRYPOINT_OFFSET(8, pUnlockObject).Int32Value(), rs_xLR);
Zheng Xuc8304302014-05-15 17:21:01 +0100255 ClobberCallerSave();
Zheng Xubaa7c882014-06-30 14:26:50 +0800256 LIR* call_inst = OpReg(kOpBlx, rs_xLR);
Zheng Xuc8304302014-05-15 17:21:01 +0100257 MarkSafepointPC(call_inst);
258
259 LIR* success_target = NewLIR0(kPseudoTargetLabel);
260 unlock_success_branch->target = success_target;
Matteo Franchin43ec8732014-03-31 15:00:14 +0100261}
262
263void Arm64Mir2Lir::GenMoveException(RegLocation rl_dest) {
Andreas Gampe2f244e92014-05-08 03:35:25 -0700264 int ex_offset = Thread::ExceptionOffset<8>().Int32Value();
buzbeea0cd2d72014-06-01 09:33:49 -0700265 RegLocation rl_result = EvalLoc(rl_dest, kRefReg, true);
Zheng Xubaa7c882014-06-30 14:26:50 +0800266 LoadRefDisp(rs_xSELF, ex_offset, rl_result.reg, kNotVolatile);
267 StoreRefDisp(rs_xSELF, ex_offset, rs_xzr, kNotVolatile);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100268 StoreValue(rl_dest, rl_result);
269}
270
Vladimir Markobf535be2014-11-19 18:52:35 +0000271void Arm64Mir2Lir::UnconditionallyMarkGCCard(RegStorage tgt_addr_reg) {
Matteo Franchinfd2e2912014-06-06 10:09:56 +0100272 RegStorage reg_card_base = AllocTempWide();
Andreas Gampe4b537a82014-06-30 22:24:53 -0700273 RegStorage reg_card_no = AllocTempWide(); // Needs to be wide as addr is ref=64b
Zheng Xubaa7c882014-06-30 14:26:50 +0800274 LoadWordDisp(rs_xSELF, Thread::CardTableOffset<8>().Int32Value(), reg_card_base);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100275 OpRegRegImm(kOpLsr, reg_card_no, tgt_addr_reg, gc::accounting::CardTable::kCardShift);
Matteo Franchinfd2e2912014-06-06 10:09:56 +0100276 // TODO(Arm64): generate "strb wB, [xB, wC, uxtw]" rather than "strb wB, [xB, xC]"?
Andreas Gampe4b537a82014-06-30 22:24:53 -0700277 StoreBaseIndexed(reg_card_base, reg_card_no, As32BitReg(reg_card_base),
Matteo Franchinfd2e2912014-06-06 10:09:56 +0100278 0, kUnsignedByte);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100279 FreeTemp(reg_card_base);
280 FreeTemp(reg_card_no);
281}
282
283void Arm64Mir2Lir::GenEntrySequence(RegLocation* ArgLocs, RegLocation rl_method) {
Matteo Franchin43ec8732014-03-31 15:00:14 +0100284 /*
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100285 * On entry, x0 to x7 are live. Let the register allocation
Matteo Franchin43ec8732014-03-31 15:00:14 +0100286 * mechanism know so it doesn't try to use any of them when
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100287 * expanding the frame or flushing.
288 * Reserve x8 & x9 for temporaries.
Matteo Franchin43ec8732014-03-31 15:00:14 +0100289 */
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100290 LockTemp(rs_x0);
291 LockTemp(rs_x1);
292 LockTemp(rs_x2);
293 LockTemp(rs_x3);
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100294 LockTemp(rs_x4);
295 LockTemp(rs_x5);
296 LockTemp(rs_x6);
297 LockTemp(rs_x7);
Zheng Xub551fdc2014-07-25 11:49:42 +0800298 LockTemp(rs_xIP0);
299 LockTemp(rs_xIP1);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100300
Stuart Monteithd5c78f42014-06-11 16:44:46 +0100301 /* TUNING:
302 * Use AllocTemp() and reuse LR if possible to give us the freedom on adjusting the number
303 * of temp registers.
304 */
305
Matteo Franchin43ec8732014-03-31 15:00:14 +0100306 /*
307 * We can safely skip the stack overflow check if we're
308 * a leaf *and* our frame size < fudge factor.
309 */
Matteo Franchin24314522014-11-12 18:06:14 +0000310 bool skip_overflow_check = mir_graph_->MethodIsLeaf() &&
311 !FrameNeedsStackCheck(frame_size_, kArm64);
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100312
Matteo Franchin43ec8732014-03-31 15:00:14 +0100313 NewLIR0(kPseudoMethodEntry);
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100314
Dave Allison648d7112014-07-25 16:15:27 -0700315 const size_t kStackOverflowReservedUsableBytes = GetStackOverflowReservedBytes(kArm64);
316 const bool large_frame = static_cast<size_t>(frame_size_) > kStackOverflowReservedUsableBytes;
317 bool generate_explicit_stack_overflow_check = large_frame ||
318 !cu_->compiler_driver->GetCompilerOptions().GetImplicitStackOverflowChecks();
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100319 const int spill_count = num_core_spills_ + num_fp_spills_;
320 const int spill_size = (spill_count * kArm64PointerSize + 15) & ~0xf; // SP 16 byte alignment.
321 const int frame_size_without_spills = frame_size_ - spill_size;
322
Matteo Franchin43ec8732014-03-31 15:00:14 +0100323 if (!skip_overflow_check) {
Dave Allison648d7112014-07-25 16:15:27 -0700324 if (generate_explicit_stack_overflow_check) {
Andreas Gampef29ecd62014-07-29 00:35:00 -0700325 // Load stack limit
326 LoadWordDisp(rs_xSELF, Thread::StackEndOffset<8>().Int32Value(), rs_xIP1);
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100327 } else {
328 // Implicit stack overflow check.
329 // Generate a load from [sp, #-framesize]. If this is in the stack
330 // redzone we will get a segmentation fault.
Stuart Monteithd5c78f42014-06-11 16:44:46 +0100331
Andreas Gampef29ecd62014-07-29 00:35:00 -0700332 // TODO: If the frame size is small enough, is it possible to make this a pre-indexed load,
333 // so that we can avoid the following "sub sp" when spilling?
Stuart Monteithd5c78f42014-06-11 16:44:46 +0100334 OpRegRegImm(kOpSub, rs_x8, rs_sp, GetStackOverflowReservedBytes(kArm64));
Matteo Franchin24314522014-11-12 18:06:14 +0000335 Load32Disp(rs_x8, 0, rs_wzr);
Stuart Monteithd5c78f42014-06-11 16:44:46 +0100336 MarkPossibleStackOverflowException();
Matteo Franchin43ec8732014-03-31 15:00:14 +0100337 }
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100338 }
339
Andreas Gampef29ecd62014-07-29 00:35:00 -0700340 int spilled_already = 0;
341 if (spill_size > 0) {
342 spilled_already = SpillRegs(rs_sp, core_spill_mask_, fp_spill_mask_, frame_size_);
343 DCHECK(spill_size == spilled_already || frame_size_ == spilled_already);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100344 }
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100345
Andreas Gampef29ecd62014-07-29 00:35:00 -0700346 if (spilled_already != frame_size_) {
347 OpRegImm(kOpSub, rs_sp, frame_size_without_spills);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100348 }
349
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100350 if (!skip_overflow_check) {
Dave Allison648d7112014-07-25 16:15:27 -0700351 if (generate_explicit_stack_overflow_check) {
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100352 class StackOverflowSlowPath: public LIRSlowPath {
353 public:
Vladimir Marko0b40ecf2015-03-20 12:08:03 +0000354 StackOverflowSlowPath(Mir2Lir* m2l, LIR* branch, size_t sp_displace)
355 : LIRSlowPath(m2l, branch),
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100356 sp_displace_(sp_displace) {
357 }
358 void Compile() OVERRIDE {
359 m2l_->ResetRegPool();
360 m2l_->ResetDefTracking();
361 GenerateTargetLabel(kPseudoThrowTarget);
362 // Unwinds stack.
Zheng Xubaa7c882014-06-30 14:26:50 +0800363 m2l_->OpRegImm(kOpAdd, rs_sp, sp_displace_);
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100364 m2l_->ClobberCallerSave();
365 ThreadOffset<8> func_offset = QUICK_ENTRYPOINT_OFFSET(8, pThrowStackOverflow);
Zheng Xub551fdc2014-07-25 11:49:42 +0800366 m2l_->LockTemp(rs_xIP0);
367 m2l_->LoadWordDisp(rs_xSELF, func_offset.Int32Value(), rs_xIP0);
368 m2l_->NewLIR1(kA64Br1x, rs_xIP0.GetReg());
369 m2l_->FreeTemp(rs_xIP0);
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100370 }
371
372 private:
373 const size_t sp_displace_;
374 };
375
Andreas Gampef29ecd62014-07-29 00:35:00 -0700376 LIR* branch = OpCmpBranch(kCondUlt, rs_sp, rs_xIP1, nullptr);
377 AddSlowPath(new(arena_)StackOverflowSlowPath(this, branch, frame_size_));
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100378 }
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100379 }
380
Matteo Franchin43ec8732014-03-31 15:00:14 +0100381 FlushIns(ArgLocs, rl_method);
382
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100383 FreeTemp(rs_x0);
384 FreeTemp(rs_x1);
385 FreeTemp(rs_x2);
386 FreeTemp(rs_x3);
Stuart Monteithf8ec48e2014-06-06 17:05:08 +0100387 FreeTemp(rs_x4);
388 FreeTemp(rs_x5);
389 FreeTemp(rs_x6);
390 FreeTemp(rs_x7);
Zheng Xub551fdc2014-07-25 11:49:42 +0800391 FreeTemp(rs_xIP0);
392 FreeTemp(rs_xIP1);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100393}
394
395void Arm64Mir2Lir::GenExitSequence() {
Matteo Franchin43ec8732014-03-31 15:00:14 +0100396 /*
397 * In the exit path, r0/r1 are live - make sure they aren't
398 * allocated by the register utilities as temps.
399 */
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100400 LockTemp(rs_x0);
401 LockTemp(rs_x1);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100402
403 NewLIR0(kPseudoMethodExit);
Matteo Franchinbc6d1972014-05-13 12:33:28 +0100404
Andreas Gampef29ecd62014-07-29 00:35:00 -0700405 UnspillRegs(rs_sp, core_spill_mask_, fp_spill_mask_, frame_size_);
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100406
buzbeeb5860fb2014-06-21 15:31:01 -0700407 // Finally return.
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100408 NewLIR0(kA64Ret);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100409}
410
411void Arm64Mir2Lir::GenSpecialExitSequence() {
Matteo Franchine45fb9e2014-05-06 10:10:30 +0100412 NewLIR0(kA64Ret);
Matteo Franchin43ec8732014-03-31 15:00:14 +0100413}
414
Vladimir Marko6ce3eba2015-02-16 13:05:59 +0000415void Arm64Mir2Lir::GenSpecialEntryForSuspend() {
416 // Keep 16-byte stack alignment - push x0, i.e. ArtMethod*, lr.
417 core_spill_mask_ = (1u << rs_xLR.GetRegNum());
418 num_core_spills_ = 1u;
419 fp_spill_mask_ = 0u;
420 num_fp_spills_ = 0u;
421 frame_size_ = 16u;
422 core_vmap_table_.clear();
423 fp_vmap_table_.clear();
424 NewLIR4(WIDE(kA64StpPre4rrXD), rs_x0.GetReg(), rs_xLR.GetReg(), rs_sp.GetReg(), -frame_size_ / 8);
425}
426
427void Arm64Mir2Lir::GenSpecialExitForSuspend() {
428 // Pop the frame. (ArtMethod* no longer needed but restore it anyway.)
429 NewLIR4(WIDE(kA64LdpPost4rrXD), rs_x0.GetReg(), rs_xLR.GetReg(), rs_sp.GetReg(), frame_size_ / 8);
430}
431
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100432static bool Arm64UseRelativeCall(CompilationUnit* cu, const MethodReference& target_method) {
Jeff Haoa0acc2d2015-01-27 11:22:04 -0800433 // Emit relative calls anywhere in the image or within a dex file otherwise.
434 return cu->compiler_driver->IsImage() || cu->dex_file == target_method.dex_file;
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100435}
436
437/*
438 * Bit of a hack here - in the absence of a real scheduling pass,
439 * emit the next instruction in static & direct invoke sequences.
440 */
441static int Arm64NextSDCallInsn(CompilationUnit* cu, CallInfo* info,
442 int state, const MethodReference& target_method,
Ian Rogers6a3c1fc2014-10-31 00:33:20 -0700443 uint32_t unused_idx,
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100444 uintptr_t direct_code, uintptr_t direct_method,
445 InvokeType type) {
Ian Rogers6a3c1fc2014-10-31 00:33:20 -0700446 UNUSED(info, unused_idx);
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100447 Mir2Lir* cg = static_cast<Mir2Lir*>(cu->cg.get());
448 if (direct_code != 0 && direct_method != 0) {
449 switch (state) {
450 case 0: // Get the current Method* [sets kArg0]
451 if (direct_code != static_cast<uintptr_t>(-1)) {
Mathieu Chartier921d6eb2015-03-13 16:32:44 -0700452 cg->LoadConstantWide(cg->TargetPtrReg(kInvokeTgt), direct_code);
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100453 } else if (Arm64UseRelativeCall(cu, target_method)) {
454 // Defer to linker patch.
455 } else {
456 cg->LoadCodeAddress(target_method, type, kInvokeTgt);
457 }
458 if (direct_method != static_cast<uintptr_t>(-1)) {
Mathieu Chartier921d6eb2015-03-13 16:32:44 -0700459 cg->LoadConstantWide(cg->TargetReg(kArg0, kRef), direct_method);
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100460 } else {
461 cg->LoadMethodAddress(target_method, type, kArg0);
462 }
463 break;
464 default:
465 return -1;
466 }
467 } else {
468 RegStorage arg0_ref = cg->TargetReg(kArg0, kRef);
469 switch (state) {
470 case 0: // Get the current Method* [sets kArg0]
471 // TUNING: we can save a reg copy if Method* has been promoted.
472 cg->LoadCurrMethodDirect(arg0_ref);
473 break;
474 case 1: // Get method->dex_cache_resolved_methods_
475 cg->LoadRefDisp(arg0_ref,
476 mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value(),
477 arg0_ref,
478 kNotVolatile);
479 // Set up direct code if known.
480 if (direct_code != 0) {
481 if (direct_code != static_cast<uintptr_t>(-1)) {
Mathieu Chartier921d6eb2015-03-13 16:32:44 -0700482 cg->LoadConstantWide(cg->TargetPtrReg(kInvokeTgt), direct_code);
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100483 } else if (Arm64UseRelativeCall(cu, target_method)) {
484 // Defer to linker patch.
485 } else {
486 CHECK_LT(target_method.dex_method_index, target_method.dex_file->NumMethodIds());
487 cg->LoadCodeAddress(target_method, type, kInvokeTgt);
488 }
489 }
490 break;
491 case 2: // Grab target method*
492 CHECK_EQ(cu->dex_file, target_method.dex_file);
493 cg->LoadRefDisp(arg0_ref,
494 mirror::ObjectArray<mirror::Object>::OffsetOfElement(
495 target_method.dex_method_index).Int32Value(),
496 arg0_ref,
497 kNotVolatile);
498 break;
499 case 3: // Grab the code from the method*
500 if (direct_code == 0) {
501 // kInvokeTgt := arg0_ref->entrypoint
502 cg->LoadWordDisp(arg0_ref,
Mathieu Chartier2d721012014-11-10 11:08:06 -0800503 mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
504 kArm64PointerSize).Int32Value(), cg->TargetPtrReg(kInvokeTgt));
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100505 }
506 break;
507 default:
508 return -1;
509 }
510 }
511 return state + 1;
512}
513
514NextCallInsn Arm64Mir2Lir::GetNextSDCallInsn() {
515 return Arm64NextSDCallInsn;
516}
517
518LIR* Arm64Mir2Lir::CallWithLinkerFixup(const MethodReference& target_method, InvokeType type) {
519 // For ARM64, just generate a relative BL instruction that will be filled in at 'link time'.
520 // If the target turns out to be too far, the linker will generate a thunk for dispatch.
521 int target_method_idx = target_method.dex_method_index;
522 const DexFile* target_dex_file = target_method.dex_file;
523
524 // Generate the call instruction and save index, dex_file, and type.
525 // NOTE: Method deduplication takes linker patches into account, so we can just pass 0
526 // as a placeholder for the offset.
527 LIR* call = RawLIR(current_dalvik_offset_, kA64Bl1t, 0,
Vladimir Markof6737f72015-03-23 17:05:14 +0000528 target_method_idx, WrapPointer(target_dex_file), type);
Vladimir Marko7c2ad5a2014-09-24 12:42:55 +0100529 AppendLIR(call);
530 call_method_insns_.push_back(call);
531 return call;
532}
533
534LIR* Arm64Mir2Lir::GenCallInsn(const MirMethodLoweringInfo& method_info) {
535 LIR* call_insn;
536 if (method_info.FastPath() && Arm64UseRelativeCall(cu_, method_info.GetTargetMethod()) &&
537 (method_info.GetSharpType() == kDirect || method_info.GetSharpType() == kStatic) &&
538 method_info.DirectCode() == static_cast<uintptr_t>(-1)) {
539 call_insn = CallWithLinkerFixup(method_info.GetTargetMethod(), method_info.GetSharpType());
540 } else {
541 call_insn = OpReg(kOpBlx, TargetPtrReg(kInvokeTgt));
542 }
543 return call_insn;
544}
545
Matteo Franchin43ec8732014-03-31 15:00:14 +0100546} // namespace art