blob: aef46c8c5da2d7d3147e092b43bddb4ea95b0c02 [file] [log] [blame]
Alexandre Rames5319def2014-10-23 10:03:10 +01001/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_arm64.h"
18
Serban Constantinescu579885a2015-02-22 20:51:33 +000019#include "arch/arm64/instruction_set_features_arm64.h"
Mathieu Chartiere401d142015-04-22 13:56:20 -070020#include "art_method.h"
Zheng Xuc6667102015-05-15 16:08:45 +080021#include "code_generator_utils.h"
Vladimir Marko58155012015-08-19 12:49:41 +000022#include "compiled_method.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010023#include "entrypoints/quick/quick_entrypoints.h"
Andreas Gampe1cc7dba2014-12-17 18:43:01 -080024#include "entrypoints/quick/quick_entrypoints_enum.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010025#include "gc/accounting/card_table.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080026#include "intrinsics.h"
27#include "intrinsics_arm64.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010028#include "mirror/array-inl.h"
Mathieu Chartiere401d142015-04-22 13:56:20 -070029#include "mirror/class-inl.h"
Calin Juravlecd6dffe2015-01-08 17:35:35 +000030#include "offsets.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010031#include "thread.h"
32#include "utils/arm64/assembler_arm64.h"
33#include "utils/assembler.h"
34#include "utils/stack_checks.h"
35
Scott Wakeling97c72b72016-06-24 16:19:36 +010036using namespace vixl::aarch64; // NOLINT(build/namespaces)
Alexandre Rames5319def2014-10-23 10:03:10 +010037
38#ifdef __
39#error "ARM64 Codegen VIXL macro-assembler macro already defined."
40#endif
41
Alexandre Rames5319def2014-10-23 10:03:10 +010042namespace art {
43
Roland Levillain22ccc3a2015-11-24 13:10:05 +000044template<class MirrorType>
45class GcRoot;
46
Alexandre Rames5319def2014-10-23 10:03:10 +010047namespace arm64 {
48
Alexandre Ramesbe919d92016-08-23 18:33:36 +010049using helpers::ARM64EncodableConstantOrRegister;
50using helpers::ArtVixlRegCodeCoherentForRegSet;
Andreas Gampe878d58c2015-01-15 23:24:00 -080051using helpers::CPURegisterFrom;
52using helpers::DRegisterFrom;
53using helpers::FPRegisterFrom;
54using helpers::HeapOperand;
55using helpers::HeapOperandFrom;
56using helpers::InputCPURegisterAt;
Alexandre Ramesbe919d92016-08-23 18:33:36 +010057using helpers::InputCPURegisterOrZeroRegAt;
Andreas Gampe878d58c2015-01-15 23:24:00 -080058using helpers::InputFPRegisterAt;
Andreas Gampe878d58c2015-01-15 23:24:00 -080059using helpers::InputOperandAt;
Alexandre Ramesbe919d92016-08-23 18:33:36 +010060using helpers::InputRegisterAt;
Andreas Gampe878d58c2015-01-15 23:24:00 -080061using helpers::Int64ConstantFrom;
Alexandre Ramesbe919d92016-08-23 18:33:36 +010062using helpers::IsConstantZeroBitPattern;
Andreas Gampe878d58c2015-01-15 23:24:00 -080063using helpers::LocationFrom;
64using helpers::OperandFromMemOperand;
65using helpers::OutputCPURegister;
66using helpers::OutputFPRegister;
67using helpers::OutputRegister;
68using helpers::RegisterFrom;
69using helpers::StackOperandFrom;
70using helpers::VIXLRegCodeFromART;
71using helpers::WRegisterFrom;
72using helpers::XRegisterFrom;
73
Alexandre Rames5319def2014-10-23 10:03:10 +010074static constexpr int kCurrentMethodStackOffset = 0;
Vladimir Markof3e0ee22015-12-17 15:23:13 +000075// The compare/jump sequence will generate about (1.5 * num_entries + 3) instructions. While jump
Zheng Xu3927c8b2015-11-18 17:46:25 +080076// table version generates 7 instructions and num_entries literals. Compare/jump sequence will
77// generates less code/data with a small num_entries.
Vladimir Markof3e0ee22015-12-17 15:23:13 +000078static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7;
Alexandre Rames5319def2014-10-23 10:03:10 +010079
Alexandre Rames5319def2014-10-23 10:03:10 +010080inline Condition ARM64Condition(IfCondition cond) {
81 switch (cond) {
82 case kCondEQ: return eq;
83 case kCondNE: return ne;
84 case kCondLT: return lt;
85 case kCondLE: return le;
86 case kCondGT: return gt;
87 case kCondGE: return ge;
Aart Bike9f37602015-10-09 11:15:55 -070088 case kCondB: return lo;
89 case kCondBE: return ls;
90 case kCondA: return hi;
91 case kCondAE: return hs;
Alexandre Rames5319def2014-10-23 10:03:10 +010092 }
Roland Levillain7f63c522015-07-13 15:54:55 +000093 LOG(FATAL) << "Unreachable";
94 UNREACHABLE();
Alexandre Rames5319def2014-10-23 10:03:10 +010095}
96
Vladimir Markod6e069b2016-01-18 11:11:01 +000097inline Condition ARM64FPCondition(IfCondition cond, bool gt_bias) {
98 // The ARM64 condition codes can express all the necessary branches, see the
99 // "Meaning (floating-point)" column in the table C1-1 in the ARMv8 reference manual.
100 // There is no dex instruction or HIR that would need the missing conditions
101 // "equal or unordered" or "not equal".
102 switch (cond) {
103 case kCondEQ: return eq;
104 case kCondNE: return ne /* unordered */;
105 case kCondLT: return gt_bias ? cc : lt /* unordered */;
106 case kCondLE: return gt_bias ? ls : le /* unordered */;
107 case kCondGT: return gt_bias ? hi /* unordered */ : gt;
108 case kCondGE: return gt_bias ? cs /* unordered */ : ge;
109 default:
110 LOG(FATAL) << "UNREACHABLE";
111 UNREACHABLE();
112 }
113}
114
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000115Location ARM64ReturnLocation(Primitive::Type return_type) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000116 // Note that in practice, `LocationFrom(x0)` and `LocationFrom(w0)` create the
117 // same Location object, and so do `LocationFrom(d0)` and `LocationFrom(s0)`,
118 // but we use the exact registers for clarity.
119 if (return_type == Primitive::kPrimFloat) {
120 return LocationFrom(s0);
121 } else if (return_type == Primitive::kPrimDouble) {
122 return LocationFrom(d0);
123 } else if (return_type == Primitive::kPrimLong) {
124 return LocationFrom(x0);
Nicolas Geoffray925e5622015-06-03 12:23:32 +0100125 } else if (return_type == Primitive::kPrimVoid) {
126 return Location::NoLocation();
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000127 } else {
128 return LocationFrom(w0);
129 }
130}
131
Alexandre Rames5319def2014-10-23 10:03:10 +0100132Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type return_type) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000133 return ARM64ReturnLocation(return_type);
Alexandre Rames5319def2014-10-23 10:03:10 +0100134}
135
Roland Levillain7cbd27f2016-08-11 23:53:33 +0100136// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy.
137#define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()-> // NOLINT
Andreas Gampe542451c2016-07-26 09:02:02 -0700138#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64PointerSize, x).Int32Value()
Alexandre Rames5319def2014-10-23 10:03:10 +0100139
Zheng Xuda403092015-04-24 17:35:39 +0800140// Calculate memory accessing operand for save/restore live registers.
141static void SaveRestoreLiveRegistersHelper(CodeGenerator* codegen,
Vladimir Marko804b03f2016-09-14 16:26:36 +0100142 LocationSummary* locations,
Zheng Xuda403092015-04-24 17:35:39 +0800143 int64_t spill_offset,
144 bool is_save) {
Vladimir Marko804b03f2016-09-14 16:26:36 +0100145 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
146 const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
147 DCHECK(ArtVixlRegCodeCoherentForRegSet(core_spills,
Zheng Xuda403092015-04-24 17:35:39 +0800148 codegen->GetNumberOfCoreRegisters(),
Vladimir Marko804b03f2016-09-14 16:26:36 +0100149 fp_spills,
Zheng Xuda403092015-04-24 17:35:39 +0800150 codegen->GetNumberOfFloatingPointRegisters()));
151
Vladimir Marko804b03f2016-09-14 16:26:36 +0100152 CPURegList core_list = CPURegList(CPURegister::kRegister, kXRegSize, core_spills);
153 CPURegList fp_list = CPURegList(CPURegister::kFPRegister, kDRegSize, fp_spills);
Zheng Xuda403092015-04-24 17:35:39 +0800154
155 MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler();
156 UseScratchRegisterScope temps(masm);
157
158 Register base = masm->StackPointer();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100159 int64_t core_spill_size = core_list.GetTotalSizeInBytes();
160 int64_t fp_spill_size = fp_list.GetTotalSizeInBytes();
Zheng Xuda403092015-04-24 17:35:39 +0800161 int64_t reg_size = kXRegSizeInBytes;
162 int64_t max_ls_pair_offset = spill_offset + core_spill_size + fp_spill_size - 2 * reg_size;
163 uint32_t ls_access_size = WhichPowerOf2(reg_size);
Scott Wakeling97c72b72016-06-24 16:19:36 +0100164 if (((core_list.GetCount() > 1) || (fp_list.GetCount() > 1)) &&
Zheng Xuda403092015-04-24 17:35:39 +0800165 !masm->IsImmLSPair(max_ls_pair_offset, ls_access_size)) {
166 // If the offset does not fit in the instruction's immediate field, use an alternate register
167 // to compute the base address(float point registers spill base address).
168 Register new_base = temps.AcquireSameSizeAs(base);
169 __ Add(new_base, base, Operand(spill_offset + core_spill_size));
170 base = new_base;
171 spill_offset = -core_spill_size;
172 int64_t new_max_ls_pair_offset = fp_spill_size - 2 * reg_size;
173 DCHECK(masm->IsImmLSPair(spill_offset, ls_access_size));
174 DCHECK(masm->IsImmLSPair(new_max_ls_pair_offset, ls_access_size));
175 }
176
177 if (is_save) {
178 __ StoreCPURegList(core_list, MemOperand(base, spill_offset));
179 __ StoreCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
180 } else {
181 __ LoadCPURegList(core_list, MemOperand(base, spill_offset));
182 __ LoadCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
183 }
184}
185
186void SlowPathCodeARM64::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
Zheng Xuda403092015-04-24 17:35:39 +0800187 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
Vladimir Marko804b03f2016-09-14 16:26:36 +0100188 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
189 for (uint32_t i : LowToHighBits(core_spills)) {
190 // If the register holds an object, update the stack mask.
191 if (locations->RegisterContainsObject(i)) {
192 locations->SetStackBit(stack_offset / kVRegSize);
Zheng Xuda403092015-04-24 17:35:39 +0800193 }
Vladimir Marko804b03f2016-09-14 16:26:36 +0100194 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
195 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
196 saved_core_stack_offsets_[i] = stack_offset;
197 stack_offset += kXRegSizeInBytes;
Zheng Xuda403092015-04-24 17:35:39 +0800198 }
199
Vladimir Marko804b03f2016-09-14 16:26:36 +0100200 const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
201 for (uint32_t i : LowToHighBits(fp_spills)) {
202 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
203 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
204 saved_fpu_stack_offsets_[i] = stack_offset;
205 stack_offset += kDRegSizeInBytes;
Zheng Xuda403092015-04-24 17:35:39 +0800206 }
207
Vladimir Marko804b03f2016-09-14 16:26:36 +0100208 SaveRestoreLiveRegistersHelper(codegen,
209 locations,
Zheng Xuda403092015-04-24 17:35:39 +0800210 codegen->GetFirstRegisterSlotInSlowPath(), true /* is_save */);
211}
212
213void SlowPathCodeARM64::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
Vladimir Marko804b03f2016-09-14 16:26:36 +0100214 SaveRestoreLiveRegistersHelper(codegen,
215 locations,
Zheng Xuda403092015-04-24 17:35:39 +0800216 codegen->GetFirstRegisterSlotInSlowPath(), false /* is_save */);
217}
218
Alexandre Rames5319def2014-10-23 10:03:10 +0100219class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 {
220 public:
David Srbecky9cd6d372016-02-09 15:24:47 +0000221 explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction) : SlowPathCodeARM64(instruction) {}
Alexandre Rames5319def2014-10-23 10:03:10 +0100222
Alexandre Rames67555f72014-11-18 10:55:16 +0000223 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100224 LocationSummary* locations = instruction_->GetLocations();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000225 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100226
Alexandre Rames5319def2014-10-23 10:03:10 +0100227 __ Bind(GetEntryLabel());
David Brazdil77a48ae2015-09-15 12:34:04 +0000228 if (instruction_->CanThrowIntoCatchBlock()) {
229 // Live registers will be restored in the catch block if caught.
230 SaveLiveRegisters(codegen, instruction_->GetLocations());
231 }
Alexandre Rames3e69f162014-12-10 10:36:50 +0000232 // We're moving two locations to locations that could overlap, so we need a parallel
233 // move resolver.
234 InvokeRuntimeCallingConvention calling_convention;
235 codegen->EmitParallelMoves(
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100236 locations->InAt(0), LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimInt,
237 locations->InAt(1), LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt);
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000238 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt()
239 ? kQuickThrowStringBounds
240 : kQuickThrowArrayBounds;
241 arm64_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this);
Vladimir Marko87f3fcb2016-04-28 15:52:11 +0100242 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>();
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800243 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
Alexandre Rames5319def2014-10-23 10:03:10 +0100244 }
245
Alexandre Rames8158f282015-08-07 10:26:17 +0100246 bool IsFatal() const OVERRIDE { return true; }
247
Alexandre Rames9931f312015-06-19 14:47:01 +0100248 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM64"; }
249
Alexandre Rames5319def2014-10-23 10:03:10 +0100250 private:
Alexandre Rames5319def2014-10-23 10:03:10 +0100251 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64);
252};
253
Alexandre Rames67555f72014-11-18 10:55:16 +0000254class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 {
255 public:
David Srbecky9cd6d372016-02-09 15:24:47 +0000256 explicit DivZeroCheckSlowPathARM64(HDivZeroCheck* instruction) : SlowPathCodeARM64(instruction) {}
Alexandre Rames67555f72014-11-18 10:55:16 +0000257
258 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
259 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
260 __ Bind(GetEntryLabel());
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000261 arm64_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800262 CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
Alexandre Rames67555f72014-11-18 10:55:16 +0000263 }
264
Alexandre Rames8158f282015-08-07 10:26:17 +0100265 bool IsFatal() const OVERRIDE { return true; }
266
Alexandre Rames9931f312015-06-19 14:47:01 +0100267 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM64"; }
268
Alexandre Rames67555f72014-11-18 10:55:16 +0000269 private:
Alexandre Rames67555f72014-11-18 10:55:16 +0000270 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM64);
271};
272
273class LoadClassSlowPathARM64 : public SlowPathCodeARM64 {
274 public:
275 LoadClassSlowPathARM64(HLoadClass* cls,
276 HInstruction* at,
277 uint32_t dex_pc,
278 bool do_clinit)
David Srbecky9cd6d372016-02-09 15:24:47 +0000279 : SlowPathCodeARM64(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
Alexandre Rames67555f72014-11-18 10:55:16 +0000280 DCHECK(at->IsLoadClass() || at->IsClinitCheck());
281 }
282
283 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
284 LocationSummary* locations = at_->GetLocations();
285 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
286
287 __ Bind(GetEntryLabel());
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000288 SaveLiveRegisters(codegen, locations);
Alexandre Rames67555f72014-11-18 10:55:16 +0000289
290 InvokeRuntimeCallingConvention calling_convention;
291 __ Mov(calling_convention.GetRegisterAt(0).W(), cls_->GetTypeIndex());
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000292 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage
293 : kQuickInitializeType;
294 arm64_codegen->InvokeRuntime(entrypoint, at_, dex_pc_, this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800295 if (do_clinit_) {
Vladimir Marko5ea536a2015-04-20 20:11:30 +0100296 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800297 } else {
Vladimir Marko5ea536a2015-04-20 20:11:30 +0100298 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800299 }
Alexandre Rames67555f72014-11-18 10:55:16 +0000300
301 // Move the class to the desired location.
302 Location out = locations->Out();
303 if (out.IsValid()) {
304 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
305 Primitive::Type type = at_->GetType();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000306 arm64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type);
Alexandre Rames67555f72014-11-18 10:55:16 +0000307 }
308
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000309 RestoreLiveRegisters(codegen, locations);
Alexandre Rames67555f72014-11-18 10:55:16 +0000310 __ B(GetExitLabel());
311 }
312
Alexandre Rames9931f312015-06-19 14:47:01 +0100313 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM64"; }
314
Alexandre Rames67555f72014-11-18 10:55:16 +0000315 private:
316 // The class this slow path will load.
317 HLoadClass* const cls_;
318
319 // The instruction where this slow path is happening.
320 // (Might be the load class or an initialization check).
321 HInstruction* const at_;
322
323 // The dex PC of `at_`.
324 const uint32_t dex_pc_;
325
326 // Whether to initialize the class.
327 const bool do_clinit_;
328
329 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64);
330};
331
Vladimir Markoaad75c62016-10-03 08:46:48 +0000332class LoadStringSlowPathARM64 : public SlowPathCodeARM64 {
333 public:
Vladimir Marko94ce9c22016-09-30 14:50:51 +0100334 LoadStringSlowPathARM64(HLoadString* instruction, Register temp, vixl::aarch64::Label* adrp_label)
335 : SlowPathCodeARM64(instruction),
336 temp_(temp),
337 adrp_label_(adrp_label) {}
Vladimir Markoaad75c62016-10-03 08:46:48 +0000338
339 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
340 LocationSummary* locations = instruction_->GetLocations();
341 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
342 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
343
Vladimir Marko94ce9c22016-09-30 14:50:51 +0100344 // temp_ is a scratch register. Make sure it's not used for saving/restoring registers.
345 UseScratchRegisterScope temps(arm64_codegen->GetVIXLAssembler());
346 temps.Exclude(temp_);
347
Vladimir Markoaad75c62016-10-03 08:46:48 +0000348 __ Bind(GetEntryLabel());
349 SaveLiveRegisters(codegen, locations);
350
351 InvokeRuntimeCallingConvention calling_convention;
352 const uint32_t string_index = instruction_->AsLoadString()->GetStringIndex();
353 __ Mov(calling_convention.GetRegisterAt(0).W(), string_index);
354 arm64_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this);
355 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
356 Primitive::Type type = instruction_->GetType();
357 arm64_codegen->MoveLocation(locations->Out(), calling_convention.GetReturnLocation(type), type);
358
359 RestoreLiveRegisters(codegen, locations);
360
361 // Store the resolved String to the BSS entry.
Vladimir Markoaad75c62016-10-03 08:46:48 +0000362 const DexFile& dex_file = instruction_->AsLoadString()->GetDexFile();
Vladimir Marko94ce9c22016-09-30 14:50:51 +0100363 if (!kUseReadBarrier || kUseBakerReadBarrier) {
364 // The string entry page address was preserved in temp_ thanks to kSaveEverything.
365 } else {
366 // For non-Baker read barrier, we need to re-calculate the address of the string entry page.
367 adrp_label_ = arm64_codegen->NewPcRelativeStringPatch(dex_file, string_index);
368 arm64_codegen->EmitAdrpPlaceholder(adrp_label_, temp_);
369 }
Vladimir Markoaad75c62016-10-03 08:46:48 +0000370 vixl::aarch64::Label* strp_label =
Vladimir Marko94ce9c22016-09-30 14:50:51 +0100371 arm64_codegen->NewPcRelativeStringPatch(dex_file, string_index, adrp_label_);
Vladimir Markoaad75c62016-10-03 08:46:48 +0000372 {
373 SingleEmissionCheckScope guard(arm64_codegen->GetVIXLAssembler());
374 __ Bind(strp_label);
375 __ str(RegisterFrom(locations->Out(), Primitive::kPrimNot),
Vladimir Marko94ce9c22016-09-30 14:50:51 +0100376 MemOperand(temp_, /* offset placeholder */ 0));
Vladimir Markoaad75c62016-10-03 08:46:48 +0000377 }
378
379 __ B(GetExitLabel());
380 }
381
382 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM64"; }
383
384 private:
Vladimir Marko94ce9c22016-09-30 14:50:51 +0100385 const Register temp_;
386 vixl::aarch64::Label* adrp_label_;
387
Vladimir Markoaad75c62016-10-03 08:46:48 +0000388 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM64);
389};
390
Alexandre Rames5319def2014-10-23 10:03:10 +0100391class NullCheckSlowPathARM64 : public SlowPathCodeARM64 {
392 public:
David Srbecky9cd6d372016-02-09 15:24:47 +0000393 explicit NullCheckSlowPathARM64(HNullCheck* instr) : SlowPathCodeARM64(instr) {}
Alexandre Rames5319def2014-10-23 10:03:10 +0100394
Alexandre Rames67555f72014-11-18 10:55:16 +0000395 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
396 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Alexandre Rames5319def2014-10-23 10:03:10 +0100397 __ Bind(GetEntryLabel());
David Brazdil77a48ae2015-09-15 12:34:04 +0000398 if (instruction_->CanThrowIntoCatchBlock()) {
399 // Live registers will be restored in the catch block if caught.
400 SaveLiveRegisters(codegen, instruction_->GetLocations());
401 }
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000402 arm64_codegen->InvokeRuntime(kQuickThrowNullPointer,
403 instruction_,
404 instruction_->GetDexPc(),
405 this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800406 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
Alexandre Rames5319def2014-10-23 10:03:10 +0100407 }
408
Alexandre Rames8158f282015-08-07 10:26:17 +0100409 bool IsFatal() const OVERRIDE { return true; }
410
Alexandre Rames9931f312015-06-19 14:47:01 +0100411 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM64"; }
412
Alexandre Rames5319def2014-10-23 10:03:10 +0100413 private:
Alexandre Rames5319def2014-10-23 10:03:10 +0100414 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM64);
415};
416
417class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 {
418 public:
Roland Levillain3887c462015-08-12 18:15:42 +0100419 SuspendCheckSlowPathARM64(HSuspendCheck* instruction, HBasicBlock* successor)
David Srbecky9cd6d372016-02-09 15:24:47 +0000420 : SlowPathCodeARM64(instruction), successor_(successor) {}
Alexandre Rames5319def2014-10-23 10:03:10 +0100421
Alexandre Rames67555f72014-11-18 10:55:16 +0000422 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
423 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Alexandre Rames5319def2014-10-23 10:03:10 +0100424 __ Bind(GetEntryLabel());
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000425 arm64_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800426 CheckEntrypointTypes<kQuickTestSuspend, void, void>();
Alexandre Rames67555f72014-11-18 10:55:16 +0000427 if (successor_ == nullptr) {
428 __ B(GetReturnLabel());
429 } else {
430 __ B(arm64_codegen->GetLabelOf(successor_));
431 }
Alexandre Rames5319def2014-10-23 10:03:10 +0100432 }
433
Scott Wakeling97c72b72016-06-24 16:19:36 +0100434 vixl::aarch64::Label* GetReturnLabel() {
Alexandre Rames5319def2014-10-23 10:03:10 +0100435 DCHECK(successor_ == nullptr);
436 return &return_label_;
437 }
438
Nicolas Geoffraydb216f42015-05-05 17:02:20 +0100439 HBasicBlock* GetSuccessor() const {
440 return successor_;
441 }
442
Alexandre Rames9931f312015-06-19 14:47:01 +0100443 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM64"; }
444
Alexandre Rames5319def2014-10-23 10:03:10 +0100445 private:
Alexandre Rames5319def2014-10-23 10:03:10 +0100446 // If not null, the block to branch to after the suspend check.
447 HBasicBlock* const successor_;
448
449 // If `successor_` is null, the label to branch to after the suspend check.
Scott Wakeling97c72b72016-06-24 16:19:36 +0100450 vixl::aarch64::Label return_label_;
Alexandre Rames5319def2014-10-23 10:03:10 +0100451
452 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64);
453};
454
Alexandre Rames67555f72014-11-18 10:55:16 +0000455class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 {
456 public:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000457 TypeCheckSlowPathARM64(HInstruction* instruction, bool is_fatal)
David Srbecky9cd6d372016-02-09 15:24:47 +0000458 : SlowPathCodeARM64(instruction), is_fatal_(is_fatal) {}
Alexandre Rames67555f72014-11-18 10:55:16 +0000459
460 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
Alexandre Rames3e69f162014-12-10 10:36:50 +0000461 LocationSummary* locations = instruction_->GetLocations();
Mathieu Chartierb99f4d62016-11-07 16:17:26 -0800462 Location arg0, arg1;
463 if (instruction_->IsInstanceOf()) {
464 arg0 = locations->InAt(1);
465 arg1 = locations->Out();
466 } else {
467 arg0 = locations->InAt(0);
468 arg1 = locations->InAt(1);
469 }
470
Alexandre Rames3e69f162014-12-10 10:36:50 +0000471 DCHECK(instruction_->IsCheckCast()
472 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
473 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100474 uint32_t dex_pc = instruction_->GetDexPc();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000475
Alexandre Rames67555f72014-11-18 10:55:16 +0000476 __ Bind(GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000477
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000478 if (!is_fatal_) {
479 SaveLiveRegisters(codegen, locations);
480 }
Alexandre Rames3e69f162014-12-10 10:36:50 +0000481
482 // We're moving two locations to locations that could overlap, so we need a parallel
483 // move resolver.
484 InvokeRuntimeCallingConvention calling_convention;
Mathieu Chartierb99f4d62016-11-07 16:17:26 -0800485 codegen->EmitParallelMoves(arg0,
486 LocationFrom(calling_convention.GetRegisterAt(0)),
487 Primitive::kPrimNot,
488 arg1,
489 LocationFrom(calling_convention.GetRegisterAt(1)),
490 Primitive::kPrimNot);
Alexandre Rames3e69f162014-12-10 10:36:50 +0000491 if (instruction_->IsInstanceOf()) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000492 arm64_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, instruction_, dex_pc, this);
Mathieu Chartierb99f4d62016-11-07 16:17:26 -0800493 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Class*, mirror::Class*>();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000494 Primitive::Type ret_type = instruction_->GetType();
495 Location ret_loc = calling_convention.GetReturnLocation(ret_type);
496 arm64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type);
497 } else {
498 DCHECK(instruction_->IsCheckCast());
Mathieu Chartierb99f4d62016-11-07 16:17:26 -0800499 arm64_codegen->InvokeRuntime(kQuickCheckInstanceOf, instruction_, dex_pc, this);
500 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000501 }
502
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000503 if (!is_fatal_) {
504 RestoreLiveRegisters(codegen, locations);
505 __ B(GetExitLabel());
506 }
Alexandre Rames67555f72014-11-18 10:55:16 +0000507 }
508
Alexandre Rames9931f312015-06-19 14:47:01 +0100509 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM64"; }
Roland Levillainf41f9562016-09-14 19:26:48 +0100510 bool IsFatal() const OVERRIDE { return is_fatal_; }
Alexandre Rames9931f312015-06-19 14:47:01 +0100511
Alexandre Rames67555f72014-11-18 10:55:16 +0000512 private:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000513 const bool is_fatal_;
Alexandre Rames3e69f162014-12-10 10:36:50 +0000514
Alexandre Rames67555f72014-11-18 10:55:16 +0000515 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM64);
516};
517
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700518class DeoptimizationSlowPathARM64 : public SlowPathCodeARM64 {
519 public:
Aart Bik42249c32016-01-07 15:33:50 -0800520 explicit DeoptimizationSlowPathARM64(HDeoptimize* instruction)
David Srbecky9cd6d372016-02-09 15:24:47 +0000521 : SlowPathCodeARM64(instruction) {}
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700522
523 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
Aart Bik42249c32016-01-07 15:33:50 -0800524 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700525 __ Bind(GetEntryLabel());
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000526 arm64_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this);
Roland Levillain888d0672015-11-23 18:53:50 +0000527 CheckEntrypointTypes<kQuickDeoptimize, void, void>();
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700528 }
529
Alexandre Rames9931f312015-06-19 14:47:01 +0100530 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM64"; }
531
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700532 private:
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700533 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM64);
534};
535
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +0100536class ArraySetSlowPathARM64 : public SlowPathCodeARM64 {
537 public:
David Srbecky9cd6d372016-02-09 15:24:47 +0000538 explicit ArraySetSlowPathARM64(HInstruction* instruction) : SlowPathCodeARM64(instruction) {}
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +0100539
540 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
541 LocationSummary* locations = instruction_->GetLocations();
542 __ Bind(GetEntryLabel());
543 SaveLiveRegisters(codegen, locations);
544
545 InvokeRuntimeCallingConvention calling_convention;
546 HParallelMove parallel_move(codegen->GetGraph()->GetArena());
547 parallel_move.AddMove(
548 locations->InAt(0),
549 LocationFrom(calling_convention.GetRegisterAt(0)),
550 Primitive::kPrimNot,
551 nullptr);
552 parallel_move.AddMove(
553 locations->InAt(1),
554 LocationFrom(calling_convention.GetRegisterAt(1)),
555 Primitive::kPrimInt,
556 nullptr);
557 parallel_move.AddMove(
558 locations->InAt(2),
559 LocationFrom(calling_convention.GetRegisterAt(2)),
560 Primitive::kPrimNot,
561 nullptr);
562 codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
563
564 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000565 arm64_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +0100566 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
567 RestoreLiveRegisters(codegen, locations);
568 __ B(GetExitLabel());
569 }
570
571 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM64"; }
572
573 private:
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +0100574 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM64);
575};
576
Zheng Xu3927c8b2015-11-18 17:46:25 +0800577void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) {
578 uint32_t num_entries = switch_instr_->GetNumEntries();
Vladimir Markof3e0ee22015-12-17 15:23:13 +0000579 DCHECK_GE(num_entries, kPackedSwitchCompareJumpThreshold);
Zheng Xu3927c8b2015-11-18 17:46:25 +0800580
581 // We are about to use the assembler to place literals directly. Make sure we have enough
582 // underlying code buffer and we have generated the jump table with right size.
583 CodeBufferCheckScope scope(codegen->GetVIXLAssembler(), num_entries * sizeof(int32_t),
584 CodeBufferCheckScope::kCheck, CodeBufferCheckScope::kExactSize);
585
586 __ Bind(&table_start_);
587 const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors();
588 for (uint32_t i = 0; i < num_entries; i++) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100589 vixl::aarch64::Label* target_label = codegen->GetLabelOf(successors[i]);
Zheng Xu3927c8b2015-11-18 17:46:25 +0800590 DCHECK(target_label->IsBound());
Scott Wakeling97c72b72016-06-24 16:19:36 +0100591 ptrdiff_t jump_offset = target_label->GetLocation() - table_start_.GetLocation();
Zheng Xu3927c8b2015-11-18 17:46:25 +0800592 DCHECK_GT(jump_offset, std::numeric_limits<int32_t>::min());
593 DCHECK_LE(jump_offset, std::numeric_limits<int32_t>::max());
594 Literal<int32_t> literal(jump_offset);
595 __ place(&literal);
596 }
597}
598
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100599// Slow path marking an object reference `ref` during a read
600// barrier. The field `obj.field` in the object `obj` holding this
601// reference does not get updated by this slow path after marking (see
602// ReadBarrierMarkAndUpdateFieldSlowPathARM64 below for that).
603//
604// This means that after the execution of this slow path, `ref` will
605// always be up-to-date, but `obj.field` may not; i.e., after the
606// flip, `ref` will be a to-space reference, but `obj.field` will
607// probably still be a from-space reference (unless it gets updated by
608// another thread, or if another thread installed another object
609// reference (different from `ref`) in `obj.field`).
Roland Levillain44015862016-01-22 11:47:17 +0000610class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 {
611 public:
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100612 ReadBarrierMarkSlowPathARM64(HInstruction* instruction, Location ref)
613 : SlowPathCodeARM64(instruction), ref_(ref) {
Roland Levillain44015862016-01-22 11:47:17 +0000614 DCHECK(kEmitCompilerReadBarrier);
615 }
616
617 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM64"; }
618
619 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
620 LocationSummary* locations = instruction_->GetLocations();
Roland Levillain44015862016-01-22 11:47:17 +0000621 DCHECK(locations->CanCall());
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100622 DCHECK(ref_.IsRegister()) << ref_;
623 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
Roland Levillain44015862016-01-22 11:47:17 +0000624 DCHECK(instruction_->IsInstanceFieldGet() ||
625 instruction_->IsStaticFieldGet() ||
626 instruction_->IsArrayGet() ||
Roland Levillain16d9f942016-08-25 17:27:56 +0100627 instruction_->IsArraySet() ||
Roland Levillain44015862016-01-22 11:47:17 +0000628 instruction_->IsLoadClass() ||
629 instruction_->IsLoadString() ||
630 instruction_->IsInstanceOf() ||
Roland Levillain3d312422016-06-23 13:53:42 +0100631 instruction_->IsCheckCast() ||
Roland Levillain0b671c02016-08-19 12:02:34 +0100632 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
633 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
Roland Levillain44015862016-01-22 11:47:17 +0000634 << "Unexpected instruction in read barrier marking slow path: "
635 << instruction_->DebugName();
Roland Levillain19c54192016-11-04 13:44:09 +0000636 // The read barrier instrumentation of object ArrayGet
637 // instructions does not support the HIntermediateAddress
638 // instruction.
639 DCHECK(!(instruction_->IsArrayGet() &&
640 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
Roland Levillain44015862016-01-22 11:47:17 +0000641
642 __ Bind(GetEntryLabel());
Roland Levillain4359e612016-07-20 11:32:19 +0100643 // No need to save live registers; it's taken care of by the
644 // entrypoint. Also, there is no need to update the stack mask,
645 // as this runtime call will not trigger a garbage collection.
Roland Levillain44015862016-01-22 11:47:17 +0000646 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100647 DCHECK_NE(ref_.reg(), LR);
648 DCHECK_NE(ref_.reg(), WSP);
649 DCHECK_NE(ref_.reg(), WZR);
Roland Levillain0b671c02016-08-19 12:02:34 +0100650 // IP0 is used internally by the ReadBarrierMarkRegX entry point
651 // as a temporary, it cannot be the entry point's input/output.
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100652 DCHECK_NE(ref_.reg(), IP0);
653 DCHECK(0 <= ref_.reg() && ref_.reg() < kNumberOfWRegisters) << ref_.reg();
Roland Levillain02b75802016-07-13 11:54:35 +0100654 // "Compact" slow path, saving two moves.
655 //
656 // Instead of using the standard runtime calling convention (input
657 // and output in W0):
658 //
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100659 // W0 <- ref
Roland Levillain02b75802016-07-13 11:54:35 +0100660 // W0 <- ReadBarrierMark(W0)
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100661 // ref <- W0
Roland Levillain02b75802016-07-13 11:54:35 +0100662 //
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100663 // we just use rX (the register containing `ref`) as input and output
Roland Levillain02b75802016-07-13 11:54:35 +0100664 // of a dedicated entrypoint:
665 //
666 // rX <- ReadBarrierMarkRegX(rX)
667 //
668 int32_t entry_point_offset =
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100669 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
Roland Levillaindec8f632016-07-22 17:10:06 +0100670 // This runtime call does not require a stack map.
671 arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
Roland Levillain44015862016-01-22 11:47:17 +0000672 __ B(GetExitLabel());
673 }
674
675 private:
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100676 // The location (register) of the marked object reference.
677 const Location ref_;
Roland Levillain44015862016-01-22 11:47:17 +0000678
679 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM64);
680};
681
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100682// Slow path marking an object reference `ref` during a read barrier,
683// and if needed, atomically updating the field `obj.field` in the
684// object `obj` holding this reference after marking (contrary to
685// ReadBarrierMarkSlowPathARM64 above, which never tries to update
686// `obj.field`).
687//
688// This means that after the execution of this slow path, both `ref`
689// and `obj.field` will be up-to-date; i.e., after the flip, both will
690// hold the same to-space reference (unless another thread installed
691// another object reference (different from `ref`) in `obj.field`).
692class ReadBarrierMarkAndUpdateFieldSlowPathARM64 : public SlowPathCodeARM64 {
693 public:
694 ReadBarrierMarkAndUpdateFieldSlowPathARM64(HInstruction* instruction,
695 Location ref,
696 Register obj,
697 Location field_offset,
698 Register temp)
699 : SlowPathCodeARM64(instruction),
700 ref_(ref),
701 obj_(obj),
702 field_offset_(field_offset),
703 temp_(temp) {
704 DCHECK(kEmitCompilerReadBarrier);
705 }
706
707 const char* GetDescription() const OVERRIDE {
708 return "ReadBarrierMarkAndUpdateFieldSlowPathARM64";
709 }
710
711 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
712 LocationSummary* locations = instruction_->GetLocations();
713 Register ref_reg = WRegisterFrom(ref_);
714 DCHECK(locations->CanCall());
715 DCHECK(ref_.IsRegister()) << ref_;
716 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
717 // This slow path is only used by the UnsafeCASObject intrinsic.
718 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
719 << "Unexpected instruction in read barrier marking and field updating slow path: "
720 << instruction_->DebugName();
721 DCHECK(instruction_->GetLocations()->Intrinsified());
722 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
723 DCHECK(field_offset_.IsRegister()) << field_offset_;
724
725 __ Bind(GetEntryLabel());
726
727 // Save the old reference.
728 // Note that we cannot use IP to save the old reference, as IP is
729 // used internally by the ReadBarrierMarkRegX entry point, and we
730 // need the old reference after the call to that entry point.
731 DCHECK_NE(LocationFrom(temp_).reg(), IP0);
732 __ Mov(temp_.W(), ref_reg);
733
734 // No need to save live registers; it's taken care of by the
735 // entrypoint. Also, there is no need to update the stack mask,
736 // as this runtime call will not trigger a garbage collection.
737 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
738 DCHECK_NE(ref_.reg(), LR);
739 DCHECK_NE(ref_.reg(), WSP);
740 DCHECK_NE(ref_.reg(), WZR);
741 // IP0 is used internally by the ReadBarrierMarkRegX entry point
742 // as a temporary, it cannot be the entry point's input/output.
743 DCHECK_NE(ref_.reg(), IP0);
744 DCHECK(0 <= ref_.reg() && ref_.reg() < kNumberOfWRegisters) << ref_.reg();
745 // "Compact" slow path, saving two moves.
746 //
747 // Instead of using the standard runtime calling convention (input
748 // and output in W0):
749 //
750 // W0 <- ref
751 // W0 <- ReadBarrierMark(W0)
752 // ref <- W0
753 //
754 // we just use rX (the register containing `ref`) as input and output
755 // of a dedicated entrypoint:
756 //
757 // rX <- ReadBarrierMarkRegX(rX)
758 //
759 int32_t entry_point_offset =
760 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
761 // This runtime call does not require a stack map.
762 arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
763
764 // If the new reference is different from the old reference,
765 // update the field in the holder (`*(obj_ + field_offset_)`).
766 //
767 // Note that this field could also hold a different object, if
768 // another thread had concurrently changed it. In that case, the
769 // LDXR/CMP/BNE sequence of instructions in the compare-and-set
770 // (CAS) operation below would abort the CAS, leaving the field
771 // as-is.
772 vixl::aarch64::Label done;
773 __ Cmp(temp_.W(), ref_reg);
774 __ B(eq, &done);
775
776 // Update the the holder's field atomically. This may fail if
777 // mutator updates before us, but it's OK. This is achieved
778 // using a strong compare-and-set (CAS) operation with relaxed
779 // memory synchronization ordering, where the expected value is
780 // the old reference and the desired value is the new reference.
781
782 MacroAssembler* masm = arm64_codegen->GetVIXLAssembler();
783 UseScratchRegisterScope temps(masm);
784
785 // Convenience aliases.
786 Register base = obj_.W();
787 Register offset = XRegisterFrom(field_offset_);
788 Register expected = temp_.W();
789 Register value = ref_reg;
790 Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory.
791 Register tmp_value = temps.AcquireW(); // Value in memory.
792
793 __ Add(tmp_ptr, base.X(), Operand(offset));
794
795 if (kPoisonHeapReferences) {
796 arm64_codegen->GetAssembler()->PoisonHeapReference(expected);
797 if (value.Is(expected)) {
798 // Do not poison `value`, as it is the same register as
799 // `expected`, which has just been poisoned.
800 } else {
801 arm64_codegen->GetAssembler()->PoisonHeapReference(value);
802 }
803 }
804
805 // do {
806 // tmp_value = [tmp_ptr] - expected;
807 // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
808
Roland Levillain24a4d112016-10-26 13:10:46 +0100809 vixl::aarch64::Label loop_head, comparison_failed, exit_loop;
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100810 __ Bind(&loop_head);
811 __ Ldxr(tmp_value, MemOperand(tmp_ptr));
812 __ Cmp(tmp_value, expected);
Roland Levillain24a4d112016-10-26 13:10:46 +0100813 __ B(&comparison_failed, ne);
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100814 __ Stxr(tmp_value, value, MemOperand(tmp_ptr));
815 __ Cbnz(tmp_value, &loop_head);
Roland Levillain24a4d112016-10-26 13:10:46 +0100816 __ B(&exit_loop);
817 __ Bind(&comparison_failed);
818 __ Clrex();
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100819 __ Bind(&exit_loop);
820
821 if (kPoisonHeapReferences) {
822 arm64_codegen->GetAssembler()->UnpoisonHeapReference(expected);
823 if (value.Is(expected)) {
824 // Do not unpoison `value`, as it is the same register as
825 // `expected`, which has just been unpoisoned.
826 } else {
827 arm64_codegen->GetAssembler()->UnpoisonHeapReference(value);
828 }
829 }
830
831 __ Bind(&done);
832 __ B(GetExitLabel());
833 }
834
835 private:
836 // The location (register) of the marked object reference.
837 const Location ref_;
838 // The register containing the object holding the marked object reference field.
839 const Register obj_;
840 // The location of the offset of the marked reference field within `obj_`.
841 Location field_offset_;
842
843 const Register temp_;
844
845 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM64);
846};
847
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000848// Slow path generating a read barrier for a heap reference.
849class ReadBarrierForHeapReferenceSlowPathARM64 : public SlowPathCodeARM64 {
850 public:
851 ReadBarrierForHeapReferenceSlowPathARM64(HInstruction* instruction,
852 Location out,
853 Location ref,
854 Location obj,
855 uint32_t offset,
856 Location index)
David Srbecky9cd6d372016-02-09 15:24:47 +0000857 : SlowPathCodeARM64(instruction),
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000858 out_(out),
859 ref_(ref),
860 obj_(obj),
861 offset_(offset),
862 index_(index) {
863 DCHECK(kEmitCompilerReadBarrier);
864 // If `obj` is equal to `out` or `ref`, it means the initial object
865 // has been overwritten by (or after) the heap object reference load
866 // to be instrumented, e.g.:
867 //
868 // __ Ldr(out, HeapOperand(out, class_offset);
Roland Levillain44015862016-01-22 11:47:17 +0000869 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset);
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000870 //
871 // In that case, we have lost the information about the original
872 // object, and the emitted read barrier cannot work properly.
873 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
874 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
875 }
876
877 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
878 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
879 LocationSummary* locations = instruction_->GetLocations();
880 Primitive::Type type = Primitive::kPrimNot;
881 DCHECK(locations->CanCall());
882 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg()));
Roland Levillain3d312422016-06-23 13:53:42 +0100883 DCHECK(instruction_->IsInstanceFieldGet() ||
884 instruction_->IsStaticFieldGet() ||
885 instruction_->IsArrayGet() ||
886 instruction_->IsInstanceOf() ||
887 instruction_->IsCheckCast() ||
Roland Levillaindec8f632016-07-22 17:10:06 +0100888 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified())
Roland Levillain44015862016-01-22 11:47:17 +0000889 << "Unexpected instruction in read barrier for heap reference slow path: "
890 << instruction_->DebugName();
Roland Levillain19c54192016-11-04 13:44:09 +0000891 // The read barrier instrumentation of object ArrayGet
892 // instructions does not support the HIntermediateAddress
893 // instruction.
Roland Levillaincd3d0fb2016-01-15 19:26:48 +0000894 DCHECK(!(instruction_->IsArrayGet() &&
Artem Serov328429f2016-07-06 16:23:04 +0100895 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000896
897 __ Bind(GetEntryLabel());
898
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000899 SaveLiveRegisters(codegen, locations);
900
901 // We may have to change the index's value, but as `index_` is a
902 // constant member (like other "inputs" of this slow path),
903 // introduce a copy of it, `index`.
904 Location index = index_;
905 if (index_.IsValid()) {
Roland Levillain3d312422016-06-23 13:53:42 +0100906 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics.
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000907 if (instruction_->IsArrayGet()) {
908 // Compute the actual memory offset and store it in `index`.
909 Register index_reg = RegisterFrom(index_, Primitive::kPrimInt);
910 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_.reg()));
911 if (codegen->IsCoreCalleeSaveRegister(index_.reg())) {
912 // We are about to change the value of `index_reg` (see the
913 // calls to vixl::MacroAssembler::Lsl and
914 // vixl::MacroAssembler::Mov below), but it has
915 // not been saved by the previous call to
916 // art::SlowPathCode::SaveLiveRegisters, as it is a
917 // callee-save register --
918 // art::SlowPathCode::SaveLiveRegisters does not consider
919 // callee-save registers, as it has been designed with the
920 // assumption that callee-save registers are supposed to be
921 // handled by the called function. So, as a callee-save
922 // register, `index_reg` _would_ eventually be saved onto
923 // the stack, but it would be too late: we would have
924 // changed its value earlier. Therefore, we manually save
925 // it here into another freely available register,
926 // `free_reg`, chosen of course among the caller-save
927 // registers (as a callee-save `free_reg` register would
928 // exhibit the same problem).
929 //
930 // Note we could have requested a temporary register from
931 // the register allocator instead; but we prefer not to, as
932 // this is a slow path, and we know we can find a
933 // caller-save register that is available.
934 Register free_reg = FindAvailableCallerSaveRegister(codegen);
935 __ Mov(free_reg.W(), index_reg);
936 index_reg = free_reg;
937 index = LocationFrom(index_reg);
938 } else {
939 // The initial register stored in `index_` has already been
940 // saved in the call to art::SlowPathCode::SaveLiveRegisters
941 // (as it is not a callee-save register), so we can freely
942 // use it.
943 }
944 // Shifting the index value contained in `index_reg` by the scale
945 // factor (2) cannot overflow in practice, as the runtime is
946 // unable to allocate object arrays with a size larger than
947 // 2^26 - 1 (that is, 2^28 - 4 bytes).
948 __ Lsl(index_reg, index_reg, Primitive::ComponentSizeShift(type));
949 static_assert(
950 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
951 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
952 __ Add(index_reg, index_reg, Operand(offset_));
953 } else {
Roland Levillain3d312422016-06-23 13:53:42 +0100954 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile
955 // intrinsics, `index_` is not shifted by a scale factor of 2
956 // (as in the case of ArrayGet), as it is actually an offset
957 // to an object field within an object.
958 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName();
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000959 DCHECK(instruction_->GetLocations()->Intrinsified());
960 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
961 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
962 << instruction_->AsInvoke()->GetIntrinsic();
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100963 DCHECK_EQ(offset_, 0u);
Roland Levillaina7426c62016-08-03 15:02:10 +0100964 DCHECK(index_.IsRegister());
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000965 }
966 }
967
968 // We're moving two or three locations to locations that could
969 // overlap, so we need a parallel move resolver.
970 InvokeRuntimeCallingConvention calling_convention;
971 HParallelMove parallel_move(codegen->GetGraph()->GetArena());
972 parallel_move.AddMove(ref_,
973 LocationFrom(calling_convention.GetRegisterAt(0)),
974 type,
975 nullptr);
976 parallel_move.AddMove(obj_,
977 LocationFrom(calling_convention.GetRegisterAt(1)),
978 type,
979 nullptr);
980 if (index.IsValid()) {
981 parallel_move.AddMove(index,
982 LocationFrom(calling_convention.GetRegisterAt(2)),
983 Primitive::kPrimInt,
984 nullptr);
985 codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
986 } else {
987 codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
988 arm64_codegen->MoveConstant(LocationFrom(calling_convention.GetRegisterAt(2)), offset_);
989 }
Serban Constantinescu22f81d32016-02-18 16:06:31 +0000990 arm64_codegen->InvokeRuntime(kQuickReadBarrierSlow,
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000991 instruction_,
992 instruction_->GetDexPc(),
993 this);
994 CheckEntrypointTypes<
995 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
996 arm64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type);
997
998 RestoreLiveRegisters(codegen, locations);
999
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001000 __ B(GetExitLabel());
1001 }
1002
1003 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM64"; }
1004
1005 private:
1006 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001007 size_t ref = static_cast<int>(XRegisterFrom(ref_).GetCode());
1008 size_t obj = static_cast<int>(XRegisterFrom(obj_).GetCode());
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001009 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1010 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
1011 return Register(VIXLRegCodeFromART(i), kXRegSize);
1012 }
1013 }
1014 // We shall never fail to find a free caller-save register, as
1015 // there are more than two core caller-save registers on ARM64
1016 // (meaning it is possible to find one which is different from
1017 // `ref` and `obj`).
1018 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
1019 LOG(FATAL) << "Could not find a free register";
1020 UNREACHABLE();
1021 }
1022
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001023 const Location out_;
1024 const Location ref_;
1025 const Location obj_;
1026 const uint32_t offset_;
1027 // An additional location containing an index to an array.
1028 // Only used for HArrayGet and the UnsafeGetObject &
1029 // UnsafeGetObjectVolatile intrinsics.
1030 const Location index_;
1031
1032 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM64);
1033};
1034
1035// Slow path generating a read barrier for a GC root.
1036class ReadBarrierForRootSlowPathARM64 : public SlowPathCodeARM64 {
1037 public:
1038 ReadBarrierForRootSlowPathARM64(HInstruction* instruction, Location out, Location root)
David Srbecky9cd6d372016-02-09 15:24:47 +00001039 : SlowPathCodeARM64(instruction), out_(out), root_(root) {
Roland Levillain44015862016-01-22 11:47:17 +00001040 DCHECK(kEmitCompilerReadBarrier);
1041 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001042
1043 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
1044 LocationSummary* locations = instruction_->GetLocations();
1045 Primitive::Type type = Primitive::kPrimNot;
1046 DCHECK(locations->CanCall());
1047 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg()));
Roland Levillain44015862016-01-22 11:47:17 +00001048 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
1049 << "Unexpected instruction in read barrier for GC root slow path: "
1050 << instruction_->DebugName();
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001051
1052 __ Bind(GetEntryLabel());
1053 SaveLiveRegisters(codegen, locations);
1054
1055 InvokeRuntimeCallingConvention calling_convention;
1056 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
1057 // The argument of the ReadBarrierForRootSlow is not a managed
1058 // reference (`mirror::Object*`), but a `GcRoot<mirror::Object>*`;
1059 // thus we need a 64-bit move here, and we cannot use
1060 //
1061 // arm64_codegen->MoveLocation(
1062 // LocationFrom(calling_convention.GetRegisterAt(0)),
1063 // root_,
1064 // type);
1065 //
1066 // which would emit a 32-bit move, as `type` is a (32-bit wide)
1067 // reference type (`Primitive::kPrimNot`).
1068 __ Mov(calling_convention.GetRegisterAt(0), XRegisterFrom(out_));
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001069 arm64_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow,
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001070 instruction_,
1071 instruction_->GetDexPc(),
1072 this);
1073 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
1074 arm64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type);
1075
1076 RestoreLiveRegisters(codegen, locations);
1077 __ B(GetExitLabel());
1078 }
1079
1080 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM64"; }
1081
1082 private:
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001083 const Location out_;
1084 const Location root_;
1085
1086 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM64);
1087};
1088
Alexandre Rames5319def2014-10-23 10:03:10 +01001089#undef __
1090
Roland Levillain2d27c8e2015-04-28 15:48:45 +01001091Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type type) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001092 Location next_location;
1093 if (type == Primitive::kPrimVoid) {
1094 LOG(FATAL) << "Unreachable type " << type;
1095 }
1096
Alexandre Rames542361f2015-01-29 16:57:31 +00001097 if (Primitive::IsFloatingPointType(type) &&
Roland Levillain2d27c8e2015-04-28 15:48:45 +01001098 (float_index_ < calling_convention.GetNumberOfFpuRegisters())) {
1099 next_location = LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++));
Alexandre Rames542361f2015-01-29 16:57:31 +00001100 } else if (!Primitive::IsFloatingPointType(type) &&
1101 (gp_index_ < calling_convention.GetNumberOfRegisters())) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001102 next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_++));
1103 } else {
1104 size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_);
Alexandre Rames542361f2015-01-29 16:57:31 +00001105 next_location = Primitive::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset)
1106 : Location::StackSlot(stack_offset);
Alexandre Rames5319def2014-10-23 10:03:10 +01001107 }
1108
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001109 // Space on the stack is reserved for all arguments.
Alexandre Rames542361f2015-01-29 16:57:31 +00001110 stack_index_ += Primitive::Is64BitType(type) ? 2 : 1;
Alexandre Rames5319def2014-10-23 10:03:10 +01001111 return next_location;
1112}
1113
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +01001114Location InvokeDexCallingConventionVisitorARM64::GetMethodLocation() const {
Nicolas Geoffray38207af2015-06-01 15:46:22 +01001115 return LocationFrom(kArtMethodRegister);
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +01001116}
1117
Serban Constantinescu579885a2015-02-22 20:51:33 +00001118CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph,
1119 const Arm64InstructionSetFeatures& isa_features,
Serban Constantinescuecc43662015-08-13 13:33:12 +01001120 const CompilerOptions& compiler_options,
1121 OptimizingCompilerStats* stats)
Alexandre Rames5319def2014-10-23 10:03:10 +01001122 : CodeGenerator(graph,
1123 kNumberOfAllocatableRegisters,
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001124 kNumberOfAllocatableFPRegisters,
Calin Juravlecd6dffe2015-01-08 17:35:35 +00001125 kNumberOfAllocatableRegisterPairs,
Scott Wakeling97c72b72016-06-24 16:19:36 +01001126 callee_saved_core_registers.GetList(),
1127 callee_saved_fp_registers.GetList(),
Serban Constantinescuecc43662015-08-13 13:33:12 +01001128 compiler_options,
1129 stats),
Alexandre Ramesc01a6642016-04-15 11:54:06 +01001130 block_labels_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
Zheng Xu3927c8b2015-11-18 17:46:25 +08001131 jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
Alexandre Rames5319def2014-10-23 10:03:10 +01001132 location_builder_(graph, this),
Alexandre Rames3e69f162014-12-10 10:36:50 +00001133 instruction_visitor_(graph, this),
Serban Constantinescu579885a2015-02-22 20:51:33 +00001134 move_resolver_(graph->GetArena(), this),
Vladimir Marko93205e32016-04-13 11:59:46 +01001135 assembler_(graph->GetArena()),
Vladimir Marko58155012015-08-19 12:49:41 +00001136 isa_features_(isa_features),
Vladimir Markocac5a7e2016-02-22 10:39:50 +00001137 uint32_literals_(std::less<uint32_t>(),
1138 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
Vladimir Marko5233f932015-09-29 19:01:15 +01001139 uint64_literals_(std::less<uint64_t>(),
1140 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
1141 method_patches_(MethodReferenceComparator(),
1142 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
1143 call_patches_(MethodReferenceComparator(),
1144 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
1145 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
Vladimir Markocac5a7e2016-02-22 10:39:50 +00001146 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
1147 boot_image_string_patches_(StringReferenceValueComparator(),
1148 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
1149 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01001150 boot_image_type_patches_(TypeReferenceValueComparator(),
1151 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
1152 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
Vladimir Markocac5a7e2016-02-22 10:39:50 +00001153 boot_image_address_patches_(std::less<uint32_t>(),
Nicolas Geoffray07c919f2016-11-09 17:29:03 +00001154 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +00001155 // Save the link register (containing the return address) to mimic Quick.
Serban Constantinescu3d087de2015-01-28 11:57:05 +00001156 AddAllocatedRegister(LocationFrom(lr));
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +00001157}
Alexandre Rames5319def2014-10-23 10:03:10 +01001158
Alexandre Rames67555f72014-11-18 10:55:16 +00001159#define __ GetVIXLAssembler()->
Alexandre Rames5319def2014-10-23 10:03:10 +01001160
Zheng Xu3927c8b2015-11-18 17:46:25 +08001161void CodeGeneratorARM64::EmitJumpTables() {
Alexandre Ramesc01a6642016-04-15 11:54:06 +01001162 for (auto&& jump_table : jump_tables_) {
Zheng Xu3927c8b2015-11-18 17:46:25 +08001163 jump_table->EmitTable(this);
1164 }
1165}
1166
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +00001167void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) {
Zheng Xu3927c8b2015-11-18 17:46:25 +08001168 EmitJumpTables();
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +00001169 // Ensure we emit the literal pool.
1170 __ FinalizeCode();
Vladimir Marko58155012015-08-19 12:49:41 +00001171
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +00001172 CodeGenerator::Finalize(allocator);
1173}
1174
Zheng Xuad4450e2015-04-17 18:48:56 +08001175void ParallelMoveResolverARM64::PrepareForEmitNativeCode() {
1176 // Note: There are 6 kinds of moves:
1177 // 1. constant -> GPR/FPR (non-cycle)
1178 // 2. constant -> stack (non-cycle)
1179 // 3. GPR/FPR -> GPR/FPR
1180 // 4. GPR/FPR -> stack
1181 // 5. stack -> GPR/FPR
1182 // 6. stack -> stack (non-cycle)
1183 // Case 1, 2 and 6 should never be included in a dependency cycle on ARM64. For case 3, 4, and 5
1184 // VIXL uses at most 1 GPR. VIXL has 2 GPR and 1 FPR temps, and there should be no intersecting
1185 // cycles on ARM64, so we always have 1 GPR and 1 FPR available VIXL temps to resolve the
1186 // dependency.
1187 vixl_temps_.Open(GetVIXLAssembler());
1188}
1189
1190void ParallelMoveResolverARM64::FinishEmitNativeCode() {
1191 vixl_temps_.Close();
1192}
1193
1194Location ParallelMoveResolverARM64::AllocateScratchLocationFor(Location::Kind kind) {
1195 DCHECK(kind == Location::kRegister || kind == Location::kFpuRegister ||
1196 kind == Location::kStackSlot || kind == Location::kDoubleStackSlot);
1197 kind = (kind == Location::kFpuRegister) ? Location::kFpuRegister : Location::kRegister;
1198 Location scratch = GetScratchLocation(kind);
1199 if (!scratch.Equals(Location::NoLocation())) {
1200 return scratch;
1201 }
1202 // Allocate from VIXL temp registers.
1203 if (kind == Location::kRegister) {
1204 scratch = LocationFrom(vixl_temps_.AcquireX());
1205 } else {
1206 DCHECK(kind == Location::kFpuRegister);
1207 scratch = LocationFrom(vixl_temps_.AcquireD());
1208 }
1209 AddScratchLocation(scratch);
1210 return scratch;
1211}
1212
1213void ParallelMoveResolverARM64::FreeScratchLocation(Location loc) {
1214 if (loc.IsRegister()) {
1215 vixl_temps_.Release(XRegisterFrom(loc));
1216 } else {
1217 DCHECK(loc.IsFpuRegister());
1218 vixl_temps_.Release(DRegisterFrom(loc));
1219 }
1220 RemoveScratchLocation(loc);
1221}
1222
Alexandre Rames3e69f162014-12-10 10:36:50 +00001223void ParallelMoveResolverARM64::EmitMove(size_t index) {
Vladimir Marko225b6462015-09-28 12:17:40 +01001224 MoveOperands* move = moves_[index];
Calin Juravlee460d1d2015-09-29 04:52:17 +01001225 codegen_->MoveLocation(move->GetDestination(), move->GetSource(), Primitive::kPrimVoid);
Alexandre Rames3e69f162014-12-10 10:36:50 +00001226}
1227
Alexandre Rames5319def2014-10-23 10:03:10 +01001228void CodeGeneratorARM64::GenerateFrameEntry() {
Alexandre Ramesd921d642015-04-16 15:07:16 +01001229 MacroAssembler* masm = GetVIXLAssembler();
1230 BlockPoolsScope block_pools(masm);
Nicolas Geoffray1cf95282014-12-12 19:22:03 +00001231 __ Bind(&frame_entry_label_);
1232
Serban Constantinescu02164b32014-11-13 14:05:07 +00001233 bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod();
1234 if (do_overflow_check) {
Alexandre Ramesd921d642015-04-16 15:07:16 +01001235 UseScratchRegisterScope temps(masm);
Serban Constantinescu02164b32014-11-13 14:05:07 +00001236 Register temp = temps.AcquireX();
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +00001237 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
Serban Constantinescu3d087de2015-01-28 11:57:05 +00001238 __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +00001239 __ Ldr(wzr, MemOperand(temp, 0));
1240 RecordPcInfo(nullptr, 0);
Serban Constantinescu02164b32014-11-13 14:05:07 +00001241 }
Alexandre Rames5319def2014-10-23 10:03:10 +01001242
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +00001243 if (!HasEmptyFrame()) {
1244 int frame_size = GetFrameSize();
1245 // Stack layout:
1246 // sp[frame_size - 8] : lr.
1247 // ... : other preserved core registers.
1248 // ... : other preserved fp registers.
1249 // ... : reserved frame space.
1250 // sp[0] : current method.
Nicolas Geoffray96eeb4e2016-10-12 22:03:31 +01001251
1252 // Save the current method if we need it. Note that we do not
1253 // do this in HCurrentMethod, as the instruction might have been removed
1254 // in the SSA graph.
1255 if (RequiresCurrentMethod()) {
1256 __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex));
Nicolas Geoffray9989b162016-10-13 13:42:30 +01001257 } else {
1258 __ Claim(frame_size);
Nicolas Geoffray96eeb4e2016-10-12 22:03:31 +01001259 }
David Srbeckyc6b4dd82015-04-07 20:32:43 +01001260 GetAssembler()->cfi().AdjustCFAOffset(frame_size);
Zheng Xu69a50302015-04-14 20:04:41 +08001261 GetAssembler()->SpillRegisters(GetFramePreservedCoreRegisters(),
1262 frame_size - GetCoreSpillSize());
1263 GetAssembler()->SpillRegisters(GetFramePreservedFPRegisters(),
1264 frame_size - FrameEntrySpillSize());
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +00001265 }
Alexandre Rames5319def2014-10-23 10:03:10 +01001266}
1267
1268void CodeGeneratorARM64::GenerateFrameExit() {
Alexandre Ramesd921d642015-04-16 15:07:16 +01001269 BlockPoolsScope block_pools(GetVIXLAssembler());
David Srbeckyc34dc932015-04-12 09:27:43 +01001270 GetAssembler()->cfi().RememberState();
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +00001271 if (!HasEmptyFrame()) {
1272 int frame_size = GetFrameSize();
Zheng Xu69a50302015-04-14 20:04:41 +08001273 GetAssembler()->UnspillRegisters(GetFramePreservedFPRegisters(),
1274 frame_size - FrameEntrySpillSize());
1275 GetAssembler()->UnspillRegisters(GetFramePreservedCoreRegisters(),
1276 frame_size - GetCoreSpillSize());
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +00001277 __ Drop(frame_size);
David Srbeckyc6b4dd82015-04-07 20:32:43 +01001278 GetAssembler()->cfi().AdjustCFAOffset(-frame_size);
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +00001279 }
David Srbeckyc34dc932015-04-12 09:27:43 +01001280 __ Ret();
1281 GetAssembler()->cfi().RestoreState();
1282 GetAssembler()->cfi().DefCFAOffset(GetFrameSize());
Alexandre Rames5319def2014-10-23 10:03:10 +01001283}
1284
Scott Wakeling97c72b72016-06-24 16:19:36 +01001285CPURegList CodeGeneratorARM64::GetFramePreservedCoreRegisters() const {
Zheng Xuda403092015-04-24 17:35:39 +08001286 DCHECK(ArtVixlRegCodeCoherentForRegSet(core_spill_mask_, GetNumberOfCoreRegisters(), 0, 0));
Scott Wakeling97c72b72016-06-24 16:19:36 +01001287 return CPURegList(CPURegister::kRegister, kXRegSize,
1288 core_spill_mask_);
Zheng Xuda403092015-04-24 17:35:39 +08001289}
1290
Scott Wakeling97c72b72016-06-24 16:19:36 +01001291CPURegList CodeGeneratorARM64::GetFramePreservedFPRegisters() const {
Zheng Xuda403092015-04-24 17:35:39 +08001292 DCHECK(ArtVixlRegCodeCoherentForRegSet(0, 0, fpu_spill_mask_,
1293 GetNumberOfFloatingPointRegisters()));
Scott Wakeling97c72b72016-06-24 16:19:36 +01001294 return CPURegList(CPURegister::kFPRegister, kDRegSize,
1295 fpu_spill_mask_);
Zheng Xuda403092015-04-24 17:35:39 +08001296}
1297
Alexandre Rames5319def2014-10-23 10:03:10 +01001298void CodeGeneratorARM64::Bind(HBasicBlock* block) {
1299 __ Bind(GetLabelOf(block));
1300}
1301
Calin Juravle175dc732015-08-25 15:42:32 +01001302void CodeGeneratorARM64::MoveConstant(Location location, int32_t value) {
1303 DCHECK(location.IsRegister());
1304 __ Mov(RegisterFrom(location, Primitive::kPrimInt), value);
1305}
1306
Calin Juravlee460d1d2015-09-29 04:52:17 +01001307void CodeGeneratorARM64::AddLocationAsTemp(Location location, LocationSummary* locations) {
1308 if (location.IsRegister()) {
1309 locations->AddTemp(location);
1310 } else {
1311 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1312 }
1313}
1314
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001315void CodeGeneratorARM64::MarkGCCard(Register object, Register value, bool value_can_be_null) {
Alexandre Rames67555f72014-11-18 10:55:16 +00001316 UseScratchRegisterScope temps(GetVIXLAssembler());
Alexandre Rames5319def2014-10-23 10:03:10 +01001317 Register card = temps.AcquireX();
Serban Constantinescu02164b32014-11-13 14:05:07 +00001318 Register temp = temps.AcquireW(); // Index within the CardTable - 32bit.
Scott Wakeling97c72b72016-06-24 16:19:36 +01001319 vixl::aarch64::Label done;
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001320 if (value_can_be_null) {
1321 __ Cbz(value, &done);
1322 }
Andreas Gampe542451c2016-07-26 09:02:02 -07001323 __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64PointerSize>().Int32Value()));
Alexandre Rames5319def2014-10-23 10:03:10 +01001324 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift);
Serban Constantinescu02164b32014-11-13 14:05:07 +00001325 __ Strb(card, MemOperand(card, temp.X()));
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001326 if (value_can_be_null) {
1327 __ Bind(&done);
1328 }
Alexandre Rames5319def2014-10-23 10:03:10 +01001329}
1330
David Brazdil58282f42016-01-14 12:45:10 +00001331void CodeGeneratorARM64::SetupBlockedRegisters() const {
Serban Constantinescu3d087de2015-01-28 11:57:05 +00001332 // Blocked core registers:
1333 // lr : Runtime reserved.
1334 // tr : Runtime reserved.
1335 // xSuspend : Runtime reserved. TODO: Unblock this when the runtime stops using it.
1336 // ip1 : VIXL core temp.
1337 // ip0 : VIXL core temp.
1338 //
1339 // Blocked fp registers:
1340 // d31 : VIXL fp temp.
Alexandre Rames5319def2014-10-23 10:03:10 +01001341 CPURegList reserved_core_registers = vixl_reserved_core_registers;
1342 reserved_core_registers.Combine(runtime_reserved_core_registers);
Alexandre Rames5319def2014-10-23 10:03:10 +01001343 while (!reserved_core_registers.IsEmpty()) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001344 blocked_core_registers_[reserved_core_registers.PopLowestIndex().GetCode()] = true;
Alexandre Rames5319def2014-10-23 10:03:10 +01001345 }
Serban Constantinescu3d087de2015-01-28 11:57:05 +00001346
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001347 CPURegList reserved_fp_registers = vixl_reserved_fp_registers;
Zheng Xua3ec3942015-02-15 18:39:46 +08001348 while (!reserved_fp_registers.IsEmpty()) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001349 blocked_fpu_registers_[reserved_fp_registers.PopLowestIndex().GetCode()] = true;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001350 }
Serban Constantinescu3d087de2015-01-28 11:57:05 +00001351
David Brazdil58282f42016-01-14 12:45:10 +00001352 if (GetGraph()->IsDebuggable()) {
Nicolas Geoffrayecf680d2015-10-05 11:15:37 +01001353 // Stubs do not save callee-save floating point registers. If the graph
1354 // is debuggable, we need to deal with these registers differently. For
1355 // now, just block them.
David Brazdil58282f42016-01-14 12:45:10 +00001356 CPURegList reserved_fp_registers_debuggable = callee_saved_fp_registers;
1357 while (!reserved_fp_registers_debuggable.IsEmpty()) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001358 blocked_fpu_registers_[reserved_fp_registers_debuggable.PopLowestIndex().GetCode()] = true;
Serban Constantinescu3d087de2015-01-28 11:57:05 +00001359 }
1360 }
Alexandre Rames5319def2014-10-23 10:03:10 +01001361}
1362
Alexandre Rames3e69f162014-12-10 10:36:50 +00001363size_t CodeGeneratorARM64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
1364 Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
1365 __ Str(reg, MemOperand(sp, stack_index));
1366 return kArm64WordSize;
1367}
1368
1369size_t CodeGeneratorARM64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
1370 Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
1371 __ Ldr(reg, MemOperand(sp, stack_index));
1372 return kArm64WordSize;
1373}
1374
1375size_t CodeGeneratorARM64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
1376 FPRegister reg = FPRegister(reg_id, kDRegSize);
1377 __ Str(reg, MemOperand(sp, stack_index));
1378 return kArm64WordSize;
1379}
1380
1381size_t CodeGeneratorARM64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
1382 FPRegister reg = FPRegister(reg_id, kDRegSize);
1383 __ Ldr(reg, MemOperand(sp, stack_index));
1384 return kArm64WordSize;
1385}
1386
Alexandre Rames5319def2014-10-23 10:03:10 +01001387void CodeGeneratorARM64::DumpCoreRegister(std::ostream& stream, int reg) const {
David Brazdilc74652862015-05-13 17:50:09 +01001388 stream << XRegister(reg);
Alexandre Rames5319def2014-10-23 10:03:10 +01001389}
1390
1391void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
David Brazdilc74652862015-05-13 17:50:09 +01001392 stream << DRegister(reg);
Alexandre Rames5319def2014-10-23 10:03:10 +01001393}
1394
Alexandre Rames67555f72014-11-18 10:55:16 +00001395void CodeGeneratorARM64::MoveConstant(CPURegister destination, HConstant* constant) {
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00001396 if (constant->IsIntConstant()) {
1397 __ Mov(Register(destination), constant->AsIntConstant()->GetValue());
1398 } else if (constant->IsLongConstant()) {
1399 __ Mov(Register(destination), constant->AsLongConstant()->GetValue());
1400 } else if (constant->IsNullConstant()) {
1401 __ Mov(Register(destination), 0);
Alexandre Rames67555f72014-11-18 10:55:16 +00001402 } else if (constant->IsFloatConstant()) {
1403 __ Fmov(FPRegister(destination), constant->AsFloatConstant()->GetValue());
1404 } else {
1405 DCHECK(constant->IsDoubleConstant());
1406 __ Fmov(FPRegister(destination), constant->AsDoubleConstant()->GetValue());
1407 }
1408}
1409
Alexandre Rames3e69f162014-12-10 10:36:50 +00001410
1411static bool CoherentConstantAndType(Location constant, Primitive::Type type) {
1412 DCHECK(constant.IsConstant());
1413 HConstant* cst = constant.GetConstant();
1414 return (cst->IsIntConstant() && type == Primitive::kPrimInt) ||
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00001415 // Null is mapped to a core W register, which we associate with kPrimInt.
1416 (cst->IsNullConstant() && type == Primitive::kPrimInt) ||
Alexandre Rames3e69f162014-12-10 10:36:50 +00001417 (cst->IsLongConstant() && type == Primitive::kPrimLong) ||
1418 (cst->IsFloatConstant() && type == Primitive::kPrimFloat) ||
1419 (cst->IsDoubleConstant() && type == Primitive::kPrimDouble);
1420}
1421
Calin Juravlee460d1d2015-09-29 04:52:17 +01001422void CodeGeneratorARM64::MoveLocation(Location destination,
1423 Location source,
1424 Primitive::Type dst_type) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001425 if (source.Equals(destination)) {
1426 return;
1427 }
Alexandre Rames3e69f162014-12-10 10:36:50 +00001428
1429 // A valid move can always be inferred from the destination and source
1430 // locations. When moving from and to a register, the argument type can be
1431 // used to generate 32bit instead of 64bit moves. In debug mode we also
1432 // checks the coherency of the locations and the type.
Calin Juravlee460d1d2015-09-29 04:52:17 +01001433 bool unspecified_type = (dst_type == Primitive::kPrimVoid);
Alexandre Rames3e69f162014-12-10 10:36:50 +00001434
1435 if (destination.IsRegister() || destination.IsFpuRegister()) {
1436 if (unspecified_type) {
1437 HConstant* src_cst = source.IsConstant() ? source.GetConstant() : nullptr;
1438 if (source.IsStackSlot() ||
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00001439 (src_cst != nullptr && (src_cst->IsIntConstant()
1440 || src_cst->IsFloatConstant()
1441 || src_cst->IsNullConstant()))) {
Alexandre Rames3e69f162014-12-10 10:36:50 +00001442 // For stack slots and 32bit constants, a 64bit type is appropriate.
Calin Juravlee460d1d2015-09-29 04:52:17 +01001443 dst_type = destination.IsRegister() ? Primitive::kPrimInt : Primitive::kPrimFloat;
Alexandre Rames67555f72014-11-18 10:55:16 +00001444 } else {
Alexandre Rames3e69f162014-12-10 10:36:50 +00001445 // If the source is a double stack slot or a 64bit constant, a 64bit
1446 // type is appropriate. Else the source is a register, and since the
1447 // type has not been specified, we chose a 64bit type to force a 64bit
1448 // move.
Calin Juravlee460d1d2015-09-29 04:52:17 +01001449 dst_type = destination.IsRegister() ? Primitive::kPrimLong : Primitive::kPrimDouble;
Alexandre Rames67555f72014-11-18 10:55:16 +00001450 }
Alexandre Rames3e69f162014-12-10 10:36:50 +00001451 }
Calin Juravlee460d1d2015-09-29 04:52:17 +01001452 DCHECK((destination.IsFpuRegister() && Primitive::IsFloatingPointType(dst_type)) ||
1453 (destination.IsRegister() && !Primitive::IsFloatingPointType(dst_type)));
1454 CPURegister dst = CPURegisterFrom(destination, dst_type);
Alexandre Rames3e69f162014-12-10 10:36:50 +00001455 if (source.IsStackSlot() || source.IsDoubleStackSlot()) {
1456 DCHECK(dst.Is64Bits() == source.IsDoubleStackSlot());
1457 __ Ldr(dst, StackOperandFrom(source));
1458 } else if (source.IsConstant()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001459 DCHECK(CoherentConstantAndType(source, dst_type));
Alexandre Rames3e69f162014-12-10 10:36:50 +00001460 MoveConstant(dst, source.GetConstant());
Calin Juravlee460d1d2015-09-29 04:52:17 +01001461 } else if (source.IsRegister()) {
Alexandre Rames3e69f162014-12-10 10:36:50 +00001462 if (destination.IsRegister()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001463 __ Mov(Register(dst), RegisterFrom(source, dst_type));
Alexandre Rames3e69f162014-12-10 10:36:50 +00001464 } else {
Zheng Xuad4450e2015-04-17 18:48:56 +08001465 DCHECK(destination.IsFpuRegister());
Calin Juravlee460d1d2015-09-29 04:52:17 +01001466 Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1467 ? Primitive::kPrimLong
1468 : Primitive::kPrimInt;
1469 __ Fmov(FPRegisterFrom(destination, dst_type), RegisterFrom(source, source_type));
1470 }
1471 } else {
1472 DCHECK(source.IsFpuRegister());
1473 if (destination.IsRegister()) {
1474 Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1475 ? Primitive::kPrimDouble
1476 : Primitive::kPrimFloat;
1477 __ Fmov(RegisterFrom(destination, dst_type), FPRegisterFrom(source, source_type));
1478 } else {
1479 DCHECK(destination.IsFpuRegister());
1480 __ Fmov(FPRegister(dst), FPRegisterFrom(source, dst_type));
Alexandre Rames3e69f162014-12-10 10:36:50 +00001481 }
1482 }
Alexandre Rames3e69f162014-12-10 10:36:50 +00001483 } else { // The destination is not a register. It must be a stack slot.
1484 DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot());
1485 if (source.IsRegister() || source.IsFpuRegister()) {
1486 if (unspecified_type) {
1487 if (source.IsRegister()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001488 dst_type = destination.IsStackSlot() ? Primitive::kPrimInt : Primitive::kPrimLong;
Alexandre Rames3e69f162014-12-10 10:36:50 +00001489 } else {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001490 dst_type = destination.IsStackSlot() ? Primitive::kPrimFloat : Primitive::kPrimDouble;
Alexandre Rames3e69f162014-12-10 10:36:50 +00001491 }
1492 }
Calin Juravlee460d1d2015-09-29 04:52:17 +01001493 DCHECK((destination.IsDoubleStackSlot() == Primitive::Is64BitType(dst_type)) &&
1494 (source.IsFpuRegister() == Primitive::IsFloatingPointType(dst_type)));
1495 __ Str(CPURegisterFrom(source, dst_type), StackOperandFrom(destination));
Alexandre Rames3e69f162014-12-10 10:36:50 +00001496 } else if (source.IsConstant()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001497 DCHECK(unspecified_type || CoherentConstantAndType(source, dst_type))
1498 << source << " " << dst_type;
Alexandre Rames3e69f162014-12-10 10:36:50 +00001499 UseScratchRegisterScope temps(GetVIXLAssembler());
1500 HConstant* src_cst = source.GetConstant();
1501 CPURegister temp;
Alexandre Ramesb2b753c2016-08-02 13:45:28 +01001502 if (src_cst->IsZeroBitPattern()) {
1503 temp = (src_cst->IsLongConstant() || src_cst->IsDoubleConstant()) ? xzr : wzr;
Alexandre Rames3e69f162014-12-10 10:36:50 +00001504 } else {
Alexandre Ramesb2b753c2016-08-02 13:45:28 +01001505 if (src_cst->IsIntConstant()) {
1506 temp = temps.AcquireW();
1507 } else if (src_cst->IsLongConstant()) {
1508 temp = temps.AcquireX();
1509 } else if (src_cst->IsFloatConstant()) {
1510 temp = temps.AcquireS();
1511 } else {
1512 DCHECK(src_cst->IsDoubleConstant());
1513 temp = temps.AcquireD();
1514 }
1515 MoveConstant(temp, src_cst);
Alexandre Rames3e69f162014-12-10 10:36:50 +00001516 }
Alexandre Rames67555f72014-11-18 10:55:16 +00001517 __ Str(temp, StackOperandFrom(destination));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001518 } else {
Alexandre Rames67555f72014-11-18 10:55:16 +00001519 DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot());
Alexandre Rames3e69f162014-12-10 10:36:50 +00001520 DCHECK(source.IsDoubleStackSlot() == destination.IsDoubleStackSlot());
Alexandre Rames67555f72014-11-18 10:55:16 +00001521 UseScratchRegisterScope temps(GetVIXLAssembler());
Alexandre Rames3e69f162014-12-10 10:36:50 +00001522 // There is generally less pressure on FP registers.
1523 FPRegister temp = destination.IsDoubleStackSlot() ? temps.AcquireD() : temps.AcquireS();
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001524 __ Ldr(temp, StackOperandFrom(source));
1525 __ Str(temp, StackOperandFrom(destination));
1526 }
1527 }
1528}
1529
1530void CodeGeneratorARM64::Load(Primitive::Type type,
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001531 CPURegister dst,
1532 const MemOperand& src) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001533 switch (type) {
1534 case Primitive::kPrimBoolean:
Alexandre Rames67555f72014-11-18 10:55:16 +00001535 __ Ldrb(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001536 break;
1537 case Primitive::kPrimByte:
Alexandre Rames67555f72014-11-18 10:55:16 +00001538 __ Ldrsb(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001539 break;
1540 case Primitive::kPrimShort:
Alexandre Rames67555f72014-11-18 10:55:16 +00001541 __ Ldrsh(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001542 break;
1543 case Primitive::kPrimChar:
Alexandre Rames67555f72014-11-18 10:55:16 +00001544 __ Ldrh(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001545 break;
1546 case Primitive::kPrimInt:
1547 case Primitive::kPrimNot:
1548 case Primitive::kPrimLong:
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001549 case Primitive::kPrimFloat:
1550 case Primitive::kPrimDouble:
Alexandre Rames542361f2015-01-29 16:57:31 +00001551 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
Alexandre Rames67555f72014-11-18 10:55:16 +00001552 __ Ldr(dst, src);
1553 break;
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001554 case Primitive::kPrimVoid:
1555 LOG(FATAL) << "Unreachable type " << type;
1556 }
1557}
1558
Calin Juravle77520bc2015-01-12 18:45:46 +00001559void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction,
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001560 CPURegister dst,
Roland Levillain44015862016-01-22 11:47:17 +00001561 const MemOperand& src,
1562 bool needs_null_check) {
Alexandre Ramesd921d642015-04-16 15:07:16 +01001563 MacroAssembler* masm = GetVIXLAssembler();
1564 BlockPoolsScope block_pools(masm);
1565 UseScratchRegisterScope temps(masm);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001566 Register temp_base = temps.AcquireX();
Calin Juravle77520bc2015-01-12 18:45:46 +00001567 Primitive::Type type = instruction->GetType();
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001568
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001569 DCHECK(!src.IsPreIndex());
1570 DCHECK(!src.IsPostIndex());
1571
1572 // TODO(vixl): Let the MacroAssembler handle MemOperand.
Scott Wakeling97c72b72016-06-24 16:19:36 +01001573 __ Add(temp_base, src.GetBaseRegister(), OperandFromMemOperand(src));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001574 MemOperand base = MemOperand(temp_base);
1575 switch (type) {
1576 case Primitive::kPrimBoolean:
1577 __ Ldarb(Register(dst), base);
Roland Levillain44015862016-01-22 11:47:17 +00001578 if (needs_null_check) {
1579 MaybeRecordImplicitNullCheck(instruction);
1580 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001581 break;
1582 case Primitive::kPrimByte:
1583 __ Ldarb(Register(dst), base);
Roland Levillain44015862016-01-22 11:47:17 +00001584 if (needs_null_check) {
1585 MaybeRecordImplicitNullCheck(instruction);
1586 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001587 __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1588 break;
1589 case Primitive::kPrimChar:
1590 __ Ldarh(Register(dst), base);
Roland Levillain44015862016-01-22 11:47:17 +00001591 if (needs_null_check) {
1592 MaybeRecordImplicitNullCheck(instruction);
1593 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001594 break;
1595 case Primitive::kPrimShort:
1596 __ Ldarh(Register(dst), base);
Roland Levillain44015862016-01-22 11:47:17 +00001597 if (needs_null_check) {
1598 MaybeRecordImplicitNullCheck(instruction);
1599 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001600 __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1601 break;
1602 case Primitive::kPrimInt:
1603 case Primitive::kPrimNot:
1604 case Primitive::kPrimLong:
Alexandre Rames542361f2015-01-29 16:57:31 +00001605 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001606 __ Ldar(Register(dst), base);
Roland Levillain44015862016-01-22 11:47:17 +00001607 if (needs_null_check) {
1608 MaybeRecordImplicitNullCheck(instruction);
1609 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001610 break;
1611 case Primitive::kPrimFloat:
1612 case Primitive::kPrimDouble: {
1613 DCHECK(dst.IsFPRegister());
Alexandre Rames542361f2015-01-29 16:57:31 +00001614 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001615
1616 Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1617 __ Ldar(temp, base);
Roland Levillain44015862016-01-22 11:47:17 +00001618 if (needs_null_check) {
1619 MaybeRecordImplicitNullCheck(instruction);
1620 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001621 __ Fmov(FPRegister(dst), temp);
1622 break;
1623 }
1624 case Primitive::kPrimVoid:
1625 LOG(FATAL) << "Unreachable type " << type;
1626 }
1627}
1628
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001629void CodeGeneratorARM64::Store(Primitive::Type type,
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001630 CPURegister src,
1631 const MemOperand& dst) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001632 switch (type) {
1633 case Primitive::kPrimBoolean:
1634 case Primitive::kPrimByte:
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001635 __ Strb(Register(src), dst);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001636 break;
1637 case Primitive::kPrimChar:
1638 case Primitive::kPrimShort:
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001639 __ Strh(Register(src), dst);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001640 break;
1641 case Primitive::kPrimInt:
1642 case Primitive::kPrimNot:
1643 case Primitive::kPrimLong:
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001644 case Primitive::kPrimFloat:
1645 case Primitive::kPrimDouble:
Alexandre Rames542361f2015-01-29 16:57:31 +00001646 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001647 __ Str(src, dst);
Alexandre Rames67555f72014-11-18 10:55:16 +00001648 break;
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001649 case Primitive::kPrimVoid:
1650 LOG(FATAL) << "Unreachable type " << type;
1651 }
1652}
1653
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001654void CodeGeneratorARM64::StoreRelease(Primitive::Type type,
1655 CPURegister src,
1656 const MemOperand& dst) {
1657 UseScratchRegisterScope temps(GetVIXLAssembler());
1658 Register temp_base = temps.AcquireX();
1659
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001660 DCHECK(!dst.IsPreIndex());
1661 DCHECK(!dst.IsPostIndex());
1662
1663 // TODO(vixl): Let the MacroAssembler handle this.
Andreas Gampe878d58c2015-01-15 23:24:00 -08001664 Operand op = OperandFromMemOperand(dst);
Scott Wakeling97c72b72016-06-24 16:19:36 +01001665 __ Add(temp_base, dst.GetBaseRegister(), op);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001666 MemOperand base = MemOperand(temp_base);
1667 switch (type) {
1668 case Primitive::kPrimBoolean:
1669 case Primitive::kPrimByte:
1670 __ Stlrb(Register(src), base);
1671 break;
1672 case Primitive::kPrimChar:
1673 case Primitive::kPrimShort:
1674 __ Stlrh(Register(src), base);
1675 break;
1676 case Primitive::kPrimInt:
1677 case Primitive::kPrimNot:
1678 case Primitive::kPrimLong:
Alexandre Rames542361f2015-01-29 16:57:31 +00001679 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001680 __ Stlr(Register(src), base);
1681 break;
1682 case Primitive::kPrimFloat:
1683 case Primitive::kPrimDouble: {
Alexandre Rames542361f2015-01-29 16:57:31 +00001684 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
Alexandre Ramesbe919d92016-08-23 18:33:36 +01001685 Register temp_src;
1686 if (src.IsZero()) {
1687 // The zero register is used to avoid synthesizing zero constants.
1688 temp_src = Register(src);
1689 } else {
1690 DCHECK(src.IsFPRegister());
1691 temp_src = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1692 __ Fmov(temp_src, FPRegister(src));
1693 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001694
Alexandre Ramesbe919d92016-08-23 18:33:36 +01001695 __ Stlr(temp_src, base);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001696 break;
1697 }
1698 case Primitive::kPrimVoid:
1699 LOG(FATAL) << "Unreachable type " << type;
1700 }
1701}
1702
Calin Juravle175dc732015-08-25 15:42:32 +01001703void CodeGeneratorARM64::InvokeRuntime(QuickEntrypointEnum entrypoint,
1704 HInstruction* instruction,
1705 uint32_t dex_pc,
1706 SlowPathCode* slow_path) {
Alexandre Rames91a65162016-09-19 13:54:30 +01001707 ValidateInvokeRuntime(entrypoint, instruction, slow_path);
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001708 GenerateInvokeRuntime(GetThreadOffset<kArm64PointerSize>(entrypoint).Int32Value());
Serban Constantinescuda8ffec2016-03-09 12:02:11 +00001709 if (EntrypointRequiresStackMap(entrypoint)) {
1710 RecordPcInfo(instruction, dex_pc, slow_path);
1711 }
Alexandre Rames67555f72014-11-18 10:55:16 +00001712}
1713
Roland Levillaindec8f632016-07-22 17:10:06 +01001714void CodeGeneratorARM64::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
1715 HInstruction* instruction,
1716 SlowPathCode* slow_path) {
1717 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path);
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001718 GenerateInvokeRuntime(entry_point_offset);
1719}
1720
1721void CodeGeneratorARM64::GenerateInvokeRuntime(int32_t entry_point_offset) {
Roland Levillaindec8f632016-07-22 17:10:06 +01001722 BlockPoolsScope block_pools(GetVIXLAssembler());
1723 __ Ldr(lr, MemOperand(tr, entry_point_offset));
1724 __ Blr(lr);
1725}
1726
Alexandre Rames67555f72014-11-18 10:55:16 +00001727void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
Scott Wakeling97c72b72016-06-24 16:19:36 +01001728 Register class_reg) {
Alexandre Rames67555f72014-11-18 10:55:16 +00001729 UseScratchRegisterScope temps(GetVIXLAssembler());
1730 Register temp = temps.AcquireW();
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001731 size_t status_offset = mirror::Class::StatusOffset().SizeValue();
1732
Serban Constantinescu02164b32014-11-13 14:05:07 +00001733 // Even if the initialized flag is set, we need to ensure consistent memory ordering.
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001734 // TODO(vixl): Let the MacroAssembler handle MemOperand.
1735 __ Add(temp, class_reg, status_offset);
1736 __ Ldar(temp, HeapOperand(temp));
1737 __ Cmp(temp, mirror::Class::kStatusInitialized);
1738 __ B(lt, slow_path->GetEntryLabel());
Alexandre Rames67555f72014-11-18 10:55:16 +00001739 __ Bind(slow_path->GetExitLabel());
1740}
Alexandre Rames5319def2014-10-23 10:03:10 +01001741
Roland Levillain44015862016-01-22 11:47:17 +00001742void CodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) {
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001743 BarrierType type = BarrierAll;
1744
1745 switch (kind) {
1746 case MemBarrierKind::kAnyAny:
1747 case MemBarrierKind::kAnyStore: {
1748 type = BarrierAll;
1749 break;
1750 }
1751 case MemBarrierKind::kLoadAny: {
1752 type = BarrierReads;
1753 break;
1754 }
1755 case MemBarrierKind::kStoreStore: {
1756 type = BarrierWrites;
1757 break;
1758 }
1759 default:
1760 LOG(FATAL) << "Unexpected memory barrier " << kind;
1761 }
1762 __ Dmb(InnerShareable, type);
1763}
1764
Serban Constantinescu02164b32014-11-13 14:05:07 +00001765void InstructionCodeGeneratorARM64::GenerateSuspendCheck(HSuspendCheck* instruction,
1766 HBasicBlock* successor) {
1767 SuspendCheckSlowPathARM64* slow_path =
Nicolas Geoffraydb216f42015-05-05 17:02:20 +01001768 down_cast<SuspendCheckSlowPathARM64*>(instruction->GetSlowPath());
1769 if (slow_path == nullptr) {
1770 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM64(instruction, successor);
1771 instruction->SetSlowPath(slow_path);
1772 codegen_->AddSlowPath(slow_path);
1773 if (successor != nullptr) {
1774 DCHECK(successor->IsLoopHeader());
1775 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
1776 }
1777 } else {
1778 DCHECK_EQ(slow_path->GetSuccessor(), successor);
1779 }
1780
Serban Constantinescu02164b32014-11-13 14:05:07 +00001781 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
1782 Register temp = temps.AcquireW();
1783
Andreas Gampe542451c2016-07-26 09:02:02 -07001784 __ Ldrh(temp, MemOperand(tr, Thread::ThreadFlagsOffset<kArm64PointerSize>().SizeValue()));
Serban Constantinescu02164b32014-11-13 14:05:07 +00001785 if (successor == nullptr) {
1786 __ Cbnz(temp, slow_path->GetEntryLabel());
1787 __ Bind(slow_path->GetReturnLabel());
1788 } else {
1789 __ Cbz(temp, codegen_->GetLabelOf(successor));
1790 __ B(slow_path->GetEntryLabel());
1791 // slow_path will return to GetLabelOf(successor).
1792 }
1793}
1794
Alexandre Rames5319def2014-10-23 10:03:10 +01001795InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph,
1796 CodeGeneratorARM64* codegen)
Aart Bik42249c32016-01-07 15:33:50 -08001797 : InstructionCodeGenerator(graph, codegen),
Alexandre Rames5319def2014-10-23 10:03:10 +01001798 assembler_(codegen->GetAssembler()),
1799 codegen_(codegen) {}
1800
1801#define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M) \
Alexandre Rames3e69f162014-12-10 10:36:50 +00001802 /* No unimplemented IR. */
Alexandre Rames5319def2014-10-23 10:03:10 +01001803
1804#define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode
1805
1806enum UnimplementedInstructionBreakCode {
Alexandre Rames67555f72014-11-18 10:55:16 +00001807 // Using a base helps identify when we hit such breakpoints.
1808 UnimplementedInstructionBreakCodeBaseCode = 0x900,
Alexandre Rames5319def2014-10-23 10:03:10 +01001809#define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name),
1810 FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION)
1811#undef ENUM_UNIMPLEMENTED_INSTRUCTION
1812};
1813
1814#define DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS(name) \
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01001815 void InstructionCodeGeneratorARM64::Visit##name(H##name* instr ATTRIBUTE_UNUSED) { \
Alexandre Rames5319def2014-10-23 10:03:10 +01001816 __ Brk(UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name)); \
1817 } \
1818 void LocationsBuilderARM64::Visit##name(H##name* instr) { \
1819 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); \
1820 locations->SetOut(Location::Any()); \
1821 }
1822 FOR_EACH_UNIMPLEMENTED_INSTRUCTION(DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS)
1823#undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS
1824
1825#undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE
Alexandre Rames67555f72014-11-18 10:55:16 +00001826#undef FOR_EACH_UNIMPLEMENTED_INSTRUCTION
Alexandre Rames5319def2014-10-23 10:03:10 +01001827
Alexandre Rames67555f72014-11-18 10:55:16 +00001828void LocationsBuilderARM64::HandleBinaryOp(HBinaryOperation* instr) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001829 DCHECK_EQ(instr->InputCount(), 2U);
1830 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1831 Primitive::Type type = instr->GetResultType();
1832 switch (type) {
1833 case Primitive::kPrimInt:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001834 case Primitive::kPrimLong:
Alexandre Rames5319def2014-10-23 10:03:10 +01001835 locations->SetInAt(0, Location::RequiresRegister());
Serban Constantinescu2d35d9d2015-02-22 22:08:01 +00001836 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instr->InputAt(1), instr));
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00001837 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01001838 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001839
1840 case Primitive::kPrimFloat:
1841 case Primitive::kPrimDouble:
1842 locations->SetInAt(0, Location::RequiresFpuRegister());
1843 locations->SetInAt(1, Location::RequiresFpuRegister());
Alexandre Rames67555f72014-11-18 10:55:16 +00001844 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01001845 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001846
Alexandre Rames5319def2014-10-23 10:03:10 +01001847 default:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001848 LOG(FATAL) << "Unexpected " << instr->DebugName() << " type " << type;
Alexandre Rames5319def2014-10-23 10:03:10 +01001849 }
1850}
1851
Alexandre Rames09a99962015-04-15 11:47:56 +01001852void LocationsBuilderARM64::HandleFieldGet(HInstruction* instruction) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001853 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
1854
1855 bool object_field_get_with_read_barrier =
1856 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
Alexandre Rames09a99962015-04-15 11:47:56 +01001857 LocationSummary* locations =
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001858 new (GetGraph()->GetArena()) LocationSummary(instruction,
1859 object_field_get_with_read_barrier ?
1860 LocationSummary::kCallOnSlowPath :
1861 LocationSummary::kNoCall);
Vladimir Marko70e97462016-08-09 11:04:26 +01001862 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01001863 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
Vladimir Marko70e97462016-08-09 11:04:26 +01001864 }
Alexandre Rames09a99962015-04-15 11:47:56 +01001865 locations->SetInAt(0, Location::RequiresRegister());
1866 if (Primitive::IsFloatingPointType(instruction->GetType())) {
1867 locations->SetOut(Location::RequiresFpuRegister());
1868 } else {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00001869 // The output overlaps for an object field get when read barriers
1870 // are enabled: we do not want the load to overwrite the object's
1871 // location, as we need it to emit the read barrier.
1872 locations->SetOut(
1873 Location::RequiresRegister(),
1874 object_field_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
Alexandre Rames09a99962015-04-15 11:47:56 +01001875 }
1876}
1877
1878void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction,
1879 const FieldInfo& field_info) {
1880 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
Roland Levillain44015862016-01-22 11:47:17 +00001881 LocationSummary* locations = instruction->GetLocations();
1882 Location base_loc = locations->InAt(0);
1883 Location out = locations->Out();
1884 uint32_t offset = field_info.GetFieldOffset().Uint32Value();
Roland Levillain4d027112015-07-01 15:41:14 +01001885 Primitive::Type field_type = field_info.GetFieldType();
Alexandre Ramesd921d642015-04-16 15:07:16 +01001886 BlockPoolsScope block_pools(GetVIXLAssembler());
Alexandre Rames09a99962015-04-15 11:47:56 +01001887 MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset());
Alexandre Rames09a99962015-04-15 11:47:56 +01001888
Roland Levillain44015862016-01-22 11:47:17 +00001889 if (field_type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
1890 // Object FieldGet with Baker's read barrier case.
1891 MacroAssembler* masm = GetVIXLAssembler();
1892 UseScratchRegisterScope temps(masm);
1893 // /* HeapReference<Object> */ out = *(base + offset)
1894 Register base = RegisterFrom(base_loc, Primitive::kPrimNot);
1895 Register temp = temps.AcquireW();
1896 // Note that potential implicit null checks are handled in this
1897 // CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier call.
1898 codegen_->GenerateFieldLoadWithBakerReadBarrier(
1899 instruction,
1900 out,
1901 base,
1902 offset,
1903 temp,
1904 /* needs_null_check */ true,
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001905 field_info.IsVolatile());
Roland Levillain44015862016-01-22 11:47:17 +00001906 } else {
1907 // General case.
1908 if (field_info.IsVolatile()) {
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001909 // Note that a potential implicit null check is handled in this
1910 // CodeGeneratorARM64::LoadAcquire call.
1911 // NB: LoadAcquire will record the pc info if needed.
1912 codegen_->LoadAcquire(
1913 instruction, OutputCPURegister(instruction), field, /* needs_null_check */ true);
Alexandre Rames09a99962015-04-15 11:47:56 +01001914 } else {
Roland Levillain4d027112015-07-01 15:41:14 +01001915 codegen_->Load(field_type, OutputCPURegister(instruction), field);
Alexandre Rames09a99962015-04-15 11:47:56 +01001916 codegen_->MaybeRecordImplicitNullCheck(instruction);
Alexandre Rames09a99962015-04-15 11:47:56 +01001917 }
Roland Levillain44015862016-01-22 11:47:17 +00001918 if (field_type == Primitive::kPrimNot) {
1919 // If read barriers are enabled, emit read barriers other than
1920 // Baker's using a slow path (and also unpoison the loaded
1921 // reference, if heap poisoning is enabled).
1922 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset);
1923 }
Roland Levillain4d027112015-07-01 15:41:14 +01001924 }
Alexandre Rames09a99962015-04-15 11:47:56 +01001925}
1926
1927void LocationsBuilderARM64::HandleFieldSet(HInstruction* instruction) {
1928 LocationSummary* locations =
1929 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1930 locations->SetInAt(0, Location::RequiresRegister());
Alexandre Ramesbe919d92016-08-23 18:33:36 +01001931 if (IsConstantZeroBitPattern(instruction->InputAt(1))) {
1932 locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant()));
1933 } else if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
Alexandre Rames09a99962015-04-15 11:47:56 +01001934 locations->SetInAt(1, Location::RequiresFpuRegister());
1935 } else {
1936 locations->SetInAt(1, Location::RequiresRegister());
1937 }
1938}
1939
1940void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction,
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001941 const FieldInfo& field_info,
1942 bool value_can_be_null) {
Alexandre Rames09a99962015-04-15 11:47:56 +01001943 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
Alexandre Ramesd921d642015-04-16 15:07:16 +01001944 BlockPoolsScope block_pools(GetVIXLAssembler());
Alexandre Rames09a99962015-04-15 11:47:56 +01001945
1946 Register obj = InputRegisterAt(instruction, 0);
Alexandre Ramesbe919d92016-08-23 18:33:36 +01001947 CPURegister value = InputCPURegisterOrZeroRegAt(instruction, 1);
Roland Levillain4d027112015-07-01 15:41:14 +01001948 CPURegister source = value;
Alexandre Rames09a99962015-04-15 11:47:56 +01001949 Offset offset = field_info.GetFieldOffset();
1950 Primitive::Type field_type = field_info.GetFieldType();
Alexandre Rames09a99962015-04-15 11:47:56 +01001951
Roland Levillain4d027112015-07-01 15:41:14 +01001952 {
1953 // We use a block to end the scratch scope before the write barrier, thus
1954 // freeing the temporary registers so they can be used in `MarkGCCard`.
1955 UseScratchRegisterScope temps(GetVIXLAssembler());
1956
1957 if (kPoisonHeapReferences && field_type == Primitive::kPrimNot) {
1958 DCHECK(value.IsW());
1959 Register temp = temps.AcquireW();
1960 __ Mov(temp, value.W());
1961 GetAssembler()->PoisonHeapReference(temp.W());
1962 source = temp;
Alexandre Rames09a99962015-04-15 11:47:56 +01001963 }
Roland Levillain4d027112015-07-01 15:41:14 +01001964
1965 if (field_info.IsVolatile()) {
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001966 codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset));
1967 codegen_->MaybeRecordImplicitNullCheck(instruction);
Roland Levillain4d027112015-07-01 15:41:14 +01001968 } else {
1969 codegen_->Store(field_type, source, HeapOperand(obj, offset));
1970 codegen_->MaybeRecordImplicitNullCheck(instruction);
1971 }
Alexandre Rames09a99962015-04-15 11:47:56 +01001972 }
1973
1974 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001975 codegen_->MarkGCCard(obj, Register(value), value_can_be_null);
Alexandre Rames09a99962015-04-15 11:47:56 +01001976 }
1977}
1978
Alexandre Rames67555f72014-11-18 10:55:16 +00001979void InstructionCodeGeneratorARM64::HandleBinaryOp(HBinaryOperation* instr) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001980 Primitive::Type type = instr->GetType();
Alexandre Rames5319def2014-10-23 10:03:10 +01001981
1982 switch (type) {
1983 case Primitive::kPrimInt:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001984 case Primitive::kPrimLong: {
1985 Register dst = OutputRegister(instr);
1986 Register lhs = InputRegisterAt(instr, 0);
1987 Operand rhs = InputOperandAt(instr, 1);
Alexandre Rames5319def2014-10-23 10:03:10 +01001988 if (instr->IsAdd()) {
1989 __ Add(dst, lhs, rhs);
Alexandre Rames67555f72014-11-18 10:55:16 +00001990 } else if (instr->IsAnd()) {
1991 __ And(dst, lhs, rhs);
1992 } else if (instr->IsOr()) {
1993 __ Orr(dst, lhs, rhs);
1994 } else if (instr->IsSub()) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001995 __ Sub(dst, lhs, rhs);
Scott Wakeling40a04bf2015-12-11 09:50:36 +00001996 } else if (instr->IsRor()) {
1997 if (rhs.IsImmediate()) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001998 uint32_t shift = rhs.GetImmediate() & (lhs.GetSizeInBits() - 1);
Scott Wakeling40a04bf2015-12-11 09:50:36 +00001999 __ Ror(dst, lhs, shift);
2000 } else {
2001 // Ensure shift distance is in the same size register as the result. If
2002 // we are rotating a long and the shift comes in a w register originally,
2003 // we don't need to sxtw for use as an x since the shift distances are
2004 // all & reg_bits - 1.
2005 __ Ror(dst, lhs, RegisterFrom(instr->GetLocations()->InAt(1), type));
2006 }
Alexandre Rames67555f72014-11-18 10:55:16 +00002007 } else {
2008 DCHECK(instr->IsXor());
2009 __ Eor(dst, lhs, rhs);
Alexandre Rames5319def2014-10-23 10:03:10 +01002010 }
2011 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002012 }
2013 case Primitive::kPrimFloat:
2014 case Primitive::kPrimDouble: {
2015 FPRegister dst = OutputFPRegister(instr);
2016 FPRegister lhs = InputFPRegisterAt(instr, 0);
2017 FPRegister rhs = InputFPRegisterAt(instr, 1);
2018 if (instr->IsAdd()) {
2019 __ Fadd(dst, lhs, rhs);
Alexandre Rames67555f72014-11-18 10:55:16 +00002020 } else if (instr->IsSub()) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002021 __ Fsub(dst, lhs, rhs);
Alexandre Rames67555f72014-11-18 10:55:16 +00002022 } else {
2023 LOG(FATAL) << "Unexpected floating-point binary operation";
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002024 }
Alexandre Rames5319def2014-10-23 10:03:10 +01002025 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002026 }
Alexandre Rames5319def2014-10-23 10:03:10 +01002027 default:
Alexandre Rames67555f72014-11-18 10:55:16 +00002028 LOG(FATAL) << "Unexpected binary operation type " << type;
Alexandre Rames5319def2014-10-23 10:03:10 +01002029 }
2030}
2031
Serban Constantinescu02164b32014-11-13 14:05:07 +00002032void LocationsBuilderARM64::HandleShift(HBinaryOperation* instr) {
2033 DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
2034
2035 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
2036 Primitive::Type type = instr->GetResultType();
2037 switch (type) {
2038 case Primitive::kPrimInt:
2039 case Primitive::kPrimLong: {
2040 locations->SetInAt(0, Location::RequiresRegister());
2041 locations->SetInAt(1, Location::RegisterOrConstant(instr->InputAt(1)));
2042 locations->SetOut(Location::RequiresRegister());
2043 break;
2044 }
2045 default:
2046 LOG(FATAL) << "Unexpected shift type " << type;
2047 }
2048}
2049
2050void InstructionCodeGeneratorARM64::HandleShift(HBinaryOperation* instr) {
2051 DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
2052
2053 Primitive::Type type = instr->GetType();
2054 switch (type) {
2055 case Primitive::kPrimInt:
2056 case Primitive::kPrimLong: {
2057 Register dst = OutputRegister(instr);
2058 Register lhs = InputRegisterAt(instr, 0);
2059 Operand rhs = InputOperandAt(instr, 1);
2060 if (rhs.IsImmediate()) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01002061 uint32_t shift_value = rhs.GetImmediate() &
Roland Levillain5b5b9312016-03-22 14:57:31 +00002062 (type == Primitive::kPrimInt ? kMaxIntShiftDistance : kMaxLongShiftDistance);
Serban Constantinescu02164b32014-11-13 14:05:07 +00002063 if (instr->IsShl()) {
2064 __ Lsl(dst, lhs, shift_value);
2065 } else if (instr->IsShr()) {
2066 __ Asr(dst, lhs, shift_value);
2067 } else {
2068 __ Lsr(dst, lhs, shift_value);
2069 }
2070 } else {
Scott Wakeling97c72b72016-06-24 16:19:36 +01002071 Register rhs_reg = dst.IsX() ? rhs.GetRegister().X() : rhs.GetRegister().W();
Serban Constantinescu02164b32014-11-13 14:05:07 +00002072
2073 if (instr->IsShl()) {
2074 __ Lsl(dst, lhs, rhs_reg);
2075 } else if (instr->IsShr()) {
2076 __ Asr(dst, lhs, rhs_reg);
2077 } else {
2078 __ Lsr(dst, lhs, rhs_reg);
2079 }
2080 }
2081 break;
2082 }
2083 default:
2084 LOG(FATAL) << "Unexpected shift operation type " << type;
2085 }
2086}
2087
Alexandre Rames5319def2014-10-23 10:03:10 +01002088void LocationsBuilderARM64::VisitAdd(HAdd* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00002089 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01002090}
2091
2092void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00002093 HandleBinaryOp(instruction);
2094}
2095
2096void LocationsBuilderARM64::VisitAnd(HAnd* instruction) {
2097 HandleBinaryOp(instruction);
2098}
2099
2100void InstructionCodeGeneratorARM64::VisitAnd(HAnd* instruction) {
2101 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01002102}
2103
Artem Serov7fc63502016-02-09 17:15:29 +00002104void LocationsBuilderARM64::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instr) {
Kevin Brodsky9ff0d202016-01-11 13:43:31 +00002105 DCHECK(Primitive::IsIntegralType(instr->GetType())) << instr->GetType();
2106 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
2107 locations->SetInAt(0, Location::RequiresRegister());
2108 // There is no immediate variant of negated bitwise instructions in AArch64.
2109 locations->SetInAt(1, Location::RequiresRegister());
2110 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2111}
2112
Artem Serov7fc63502016-02-09 17:15:29 +00002113void InstructionCodeGeneratorARM64::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instr) {
Kevin Brodsky9ff0d202016-01-11 13:43:31 +00002114 Register dst = OutputRegister(instr);
2115 Register lhs = InputRegisterAt(instr, 0);
2116 Register rhs = InputRegisterAt(instr, 1);
2117
2118 switch (instr->GetOpKind()) {
2119 case HInstruction::kAnd:
2120 __ Bic(dst, lhs, rhs);
2121 break;
2122 case HInstruction::kOr:
2123 __ Orn(dst, lhs, rhs);
2124 break;
2125 case HInstruction::kXor:
2126 __ Eon(dst, lhs, rhs);
2127 break;
2128 default:
2129 LOG(FATAL) << "Unreachable";
2130 }
2131}
2132
Alexandre Rames8626b742015-11-25 16:28:08 +00002133void LocationsBuilderARM64::VisitArm64DataProcWithShifterOp(
2134 HArm64DataProcWithShifterOp* instruction) {
2135 DCHECK(instruction->GetType() == Primitive::kPrimInt ||
2136 instruction->GetType() == Primitive::kPrimLong);
2137 LocationSummary* locations =
2138 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2139 if (instruction->GetInstrKind() == HInstruction::kNeg) {
2140 locations->SetInAt(0, Location::ConstantLocation(instruction->InputAt(0)->AsConstant()));
2141 } else {
2142 locations->SetInAt(0, Location::RequiresRegister());
2143 }
2144 locations->SetInAt(1, Location::RequiresRegister());
2145 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2146}
2147
2148void InstructionCodeGeneratorARM64::VisitArm64DataProcWithShifterOp(
2149 HArm64DataProcWithShifterOp* instruction) {
2150 Primitive::Type type = instruction->GetType();
2151 HInstruction::InstructionKind kind = instruction->GetInstrKind();
2152 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
2153 Register out = OutputRegister(instruction);
2154 Register left;
2155 if (kind != HInstruction::kNeg) {
2156 left = InputRegisterAt(instruction, 0);
2157 }
2158 // If this `HArm64DataProcWithShifterOp` was created by merging a type conversion as the
2159 // shifter operand operation, the IR generating `right_reg` (input to the type
2160 // conversion) can have a different type from the current instruction's type,
2161 // so we manually indicate the type.
2162 Register right_reg = RegisterFrom(instruction->GetLocations()->InAt(1), type);
Roland Levillain5b5b9312016-03-22 14:57:31 +00002163 int64_t shift_amount = instruction->GetShiftAmount() &
2164 (type == Primitive::kPrimInt ? kMaxIntShiftDistance : kMaxLongShiftDistance);
Alexandre Rames8626b742015-11-25 16:28:08 +00002165
2166 Operand right_operand(0);
2167
2168 HArm64DataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind();
2169 if (HArm64DataProcWithShifterOp::IsExtensionOp(op_kind)) {
2170 right_operand = Operand(right_reg, helpers::ExtendFromOpKind(op_kind));
2171 } else {
2172 right_operand = Operand(right_reg, helpers::ShiftFromOpKind(op_kind), shift_amount);
2173 }
2174
2175 // Logical binary operations do not support extension operations in the
2176 // operand. Note that VIXL would still manage if it was passed by generating
2177 // the extension as a separate instruction.
2178 // `HNeg` also does not support extension. See comments in `ShifterOperandSupportsExtension()`.
2179 DCHECK(!right_operand.IsExtendedRegister() ||
2180 (kind != HInstruction::kAnd && kind != HInstruction::kOr && kind != HInstruction::kXor &&
2181 kind != HInstruction::kNeg));
2182 switch (kind) {
2183 case HInstruction::kAdd:
2184 __ Add(out, left, right_operand);
2185 break;
2186 case HInstruction::kAnd:
2187 __ And(out, left, right_operand);
2188 break;
2189 case HInstruction::kNeg:
Roland Levillain1a653882016-03-18 18:05:57 +00002190 DCHECK(instruction->InputAt(0)->AsConstant()->IsArithmeticZero());
Alexandre Rames8626b742015-11-25 16:28:08 +00002191 __ Neg(out, right_operand);
2192 break;
2193 case HInstruction::kOr:
2194 __ Orr(out, left, right_operand);
2195 break;
2196 case HInstruction::kSub:
2197 __ Sub(out, left, right_operand);
2198 break;
2199 case HInstruction::kXor:
2200 __ Eor(out, left, right_operand);
2201 break;
2202 default:
2203 LOG(FATAL) << "Unexpected operation kind: " << kind;
2204 UNREACHABLE();
2205 }
2206}
2207
Artem Serov328429f2016-07-06 16:23:04 +01002208void LocationsBuilderARM64::VisitIntermediateAddress(HIntermediateAddress* instruction) {
Alexandre Ramese6dbf482015-10-19 10:10:41 +01002209 LocationSummary* locations =
2210 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2211 locations->SetInAt(0, Location::RequiresRegister());
2212 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->GetOffset(), instruction));
2213 locations->SetOut(Location::RequiresRegister());
2214}
2215
Roland Levillain19c54192016-11-04 13:44:09 +00002216void InstructionCodeGeneratorARM64::VisitIntermediateAddress(HIntermediateAddress* instruction) {
Alexandre Ramese6dbf482015-10-19 10:10:41 +01002217 __ Add(OutputRegister(instruction),
2218 InputRegisterAt(instruction, 0),
2219 Operand(InputOperandAt(instruction, 1)));
2220}
2221
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002222void LocationsBuilderARM64::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) {
Alexandre Rames418318f2015-11-20 15:55:47 +00002223 LocationSummary* locations =
2224 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall);
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002225 HInstruction* accumulator = instr->InputAt(HMultiplyAccumulate::kInputAccumulatorIndex);
2226 if (instr->GetOpKind() == HInstruction::kSub &&
2227 accumulator->IsConstant() &&
Roland Levillain1a653882016-03-18 18:05:57 +00002228 accumulator->AsConstant()->IsArithmeticZero()) {
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002229 // Don't allocate register for Mneg instruction.
2230 } else {
2231 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex,
2232 Location::RequiresRegister());
2233 }
2234 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister());
2235 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister());
Alexandre Rames418318f2015-11-20 15:55:47 +00002236 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2237}
2238
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002239void InstructionCodeGeneratorARM64::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) {
Alexandre Rames418318f2015-11-20 15:55:47 +00002240 Register res = OutputRegister(instr);
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002241 Register mul_left = InputRegisterAt(instr, HMultiplyAccumulate::kInputMulLeftIndex);
2242 Register mul_right = InputRegisterAt(instr, HMultiplyAccumulate::kInputMulRightIndex);
Alexandre Rames418318f2015-11-20 15:55:47 +00002243
2244 // Avoid emitting code that could trigger Cortex A53's erratum 835769.
2245 // This fixup should be carried out for all multiply-accumulate instructions:
2246 // madd, msub, smaddl, smsubl, umaddl and umsubl.
2247 if (instr->GetType() == Primitive::kPrimLong &&
2248 codegen_->GetInstructionSetFeatures().NeedFixCortexA53_835769()) {
2249 MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen_)->GetVIXLAssembler();
Scott Wakeling97c72b72016-06-24 16:19:36 +01002250 vixl::aarch64::Instruction* prev =
2251 masm->GetCursorAddress<vixl::aarch64::Instruction*>() - kInstructionSize;
Alexandre Rames418318f2015-11-20 15:55:47 +00002252 if (prev->IsLoadOrStore()) {
2253 // Make sure we emit only exactly one nop.
Scott Wakeling97c72b72016-06-24 16:19:36 +01002254 vixl::aarch64::CodeBufferCheckScope scope(masm,
2255 kInstructionSize,
2256 vixl::aarch64::CodeBufferCheckScope::kCheck,
2257 vixl::aarch64::CodeBufferCheckScope::kExactSize);
Alexandre Rames418318f2015-11-20 15:55:47 +00002258 __ nop();
2259 }
2260 }
2261
2262 if (instr->GetOpKind() == HInstruction::kAdd) {
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002263 Register accumulator = InputRegisterAt(instr, HMultiplyAccumulate::kInputAccumulatorIndex);
Alexandre Rames418318f2015-11-20 15:55:47 +00002264 __ Madd(res, mul_left, mul_right, accumulator);
2265 } else {
2266 DCHECK(instr->GetOpKind() == HInstruction::kSub);
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002267 HInstruction* accum_instr = instr->InputAt(HMultiplyAccumulate::kInputAccumulatorIndex);
Roland Levillain1a653882016-03-18 18:05:57 +00002268 if (accum_instr->IsConstant() && accum_instr->AsConstant()->IsArithmeticZero()) {
Artem Udovichenko4a0dad62016-01-26 12:28:31 +03002269 __ Mneg(res, mul_left, mul_right);
2270 } else {
2271 Register accumulator = InputRegisterAt(instr, HMultiplyAccumulate::kInputAccumulatorIndex);
2272 __ Msub(res, mul_left, mul_right, accumulator);
2273 }
Alexandre Rames418318f2015-11-20 15:55:47 +00002274 }
2275}
2276
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002277void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002278 bool object_array_get_with_read_barrier =
2279 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002280 LocationSummary* locations =
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002281 new (GetGraph()->GetArena()) LocationSummary(instruction,
2282 object_array_get_with_read_barrier ?
2283 LocationSummary::kCallOnSlowPath :
2284 LocationSummary::kNoCall);
Vladimir Marko70e97462016-08-09 11:04:26 +01002285 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01002286 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
Vladimir Marko70e97462016-08-09 11:04:26 +01002287 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002288 locations->SetInAt(0, Location::RequiresRegister());
2289 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
Alexandre Rames88c13cd2015-04-14 17:35:39 +01002290 if (Primitive::IsFloatingPointType(instruction->GetType())) {
2291 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2292 } else {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002293 // The output overlaps in the case of an object array get with
2294 // read barriers enabled: we do not want the move to overwrite the
2295 // array's location, as we need it to emit the read barrier.
2296 locations->SetOut(
2297 Location::RequiresRegister(),
2298 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
Alexandre Rames88c13cd2015-04-14 17:35:39 +01002299 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002300}
2301
2302void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002303 Primitive::Type type = instruction->GetType();
2304 Register obj = InputRegisterAt(instruction, 0);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002305 LocationSummary* locations = instruction->GetLocations();
2306 Location index = locations->InAt(1);
Roland Levillain44015862016-01-22 11:47:17 +00002307 Location out = locations->Out();
Vladimir Marko87f3fcb2016-04-28 15:52:11 +01002308 uint32_t offset = CodeGenerator::GetArrayDataOffset(instruction);
jessicahandojo05765752016-09-09 19:01:32 -07002309 const bool maybe_compressed_char_at = mirror::kUseStringCompression &&
2310 instruction->IsStringCharAt();
Alexandre Ramesd921d642015-04-16 15:07:16 +01002311 MacroAssembler* masm = GetVIXLAssembler();
2312 UseScratchRegisterScope temps(masm);
Alexandre Ramese6dbf482015-10-19 10:10:41 +01002313 // Block pools between `Load` and `MaybeRecordImplicitNullCheck`.
Alexandre Ramesd921d642015-04-16 15:07:16 +01002314 BlockPoolsScope block_pools(masm);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002315
Roland Levillain19c54192016-11-04 13:44:09 +00002316 // The read barrier instrumentation of object ArrayGet instructions
2317 // does not support the HIntermediateAddress instruction.
2318 DCHECK(!((type == Primitive::kPrimNot) &&
2319 instruction->GetArray()->IsIntermediateAddress() &&
2320 kEmitCompilerReadBarrier));
2321
Roland Levillain44015862016-01-22 11:47:17 +00002322 if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2323 // Object ArrayGet with Baker's read barrier case.
2324 Register temp = temps.AcquireW();
Roland Levillain44015862016-01-22 11:47:17 +00002325 // Note that a potential implicit null check is handled in the
2326 // CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier call.
2327 codegen_->GenerateArrayLoadWithBakerReadBarrier(
2328 instruction, out, obj.W(), offset, index, temp, /* needs_null_check */ true);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002329 } else {
Roland Levillain44015862016-01-22 11:47:17 +00002330 // General case.
2331 MemOperand source = HeapOperand(obj);
jessicahandojo05765752016-09-09 19:01:32 -07002332 Register length;
2333 if (maybe_compressed_char_at) {
2334 uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
2335 length = temps.AcquireW();
2336 __ Ldr(length, HeapOperand(obj, count_offset));
2337 codegen_->MaybeRecordImplicitNullCheck(instruction);
2338 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002339 if (index.IsConstant()) {
jessicahandojo05765752016-09-09 19:01:32 -07002340 if (maybe_compressed_char_at) {
2341 vixl::aarch64::Label uncompressed_load, done;
2342 __ Tbz(length.W(), kWRegSize - 1, &uncompressed_load);
2343 __ Ldrb(Register(OutputCPURegister(instruction)),
2344 HeapOperand(obj, offset + Int64ConstantFrom(index)));
2345 __ B(&done);
2346 __ Bind(&uncompressed_load);
2347 __ Ldrh(Register(OutputCPURegister(instruction)),
2348 HeapOperand(obj, offset + (Int64ConstantFrom(index) << 1)));
2349 __ Bind(&done);
2350 } else {
2351 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(type);
2352 source = HeapOperand(obj, offset);
2353 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002354 } else {
Roland Levillain44015862016-01-22 11:47:17 +00002355 Register temp = temps.AcquireSameSizeAs(obj);
Artem Serov328429f2016-07-06 16:23:04 +01002356 if (instruction->GetArray()->IsIntermediateAddress()) {
Roland Levillain44015862016-01-22 11:47:17 +00002357 // We do not need to compute the intermediate address from the array: the
2358 // input instruction has done it already. See the comment in
Artem Serov328429f2016-07-06 16:23:04 +01002359 // `TryExtractArrayAccessAddress()`.
Roland Levillain44015862016-01-22 11:47:17 +00002360 if (kIsDebugBuild) {
Artem Serov328429f2016-07-06 16:23:04 +01002361 HIntermediateAddress* tmp = instruction->GetArray()->AsIntermediateAddress();
Roland Levillain44015862016-01-22 11:47:17 +00002362 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), offset);
2363 }
2364 temp = obj;
2365 } else {
2366 __ Add(temp, obj, offset);
2367 }
jessicahandojo05765752016-09-09 19:01:32 -07002368 if (maybe_compressed_char_at) {
2369 vixl::aarch64::Label uncompressed_load, done;
2370 __ Tbz(length.W(), kWRegSize - 1, &uncompressed_load);
2371 __ Ldrb(Register(OutputCPURegister(instruction)),
2372 HeapOperand(temp, XRegisterFrom(index), LSL, 0));
2373 __ B(&done);
2374 __ Bind(&uncompressed_load);
2375 __ Ldrh(Register(OutputCPURegister(instruction)),
2376 HeapOperand(temp, XRegisterFrom(index), LSL, 1));
2377 __ Bind(&done);
2378 } else {
2379 source = HeapOperand(temp, XRegisterFrom(index), LSL, Primitive::ComponentSizeShift(type));
2380 }
Roland Levillain44015862016-01-22 11:47:17 +00002381 }
jessicahandojo05765752016-09-09 19:01:32 -07002382 if (!maybe_compressed_char_at) {
2383 codegen_->Load(type, OutputCPURegister(instruction), source);
2384 codegen_->MaybeRecordImplicitNullCheck(instruction);
2385 }
Roland Levillain44015862016-01-22 11:47:17 +00002386
2387 if (type == Primitive::kPrimNot) {
2388 static_assert(
2389 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
2390 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
2391 Location obj_loc = locations->InAt(0);
2392 if (index.IsConstant()) {
2393 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, obj_loc, offset);
2394 } else {
2395 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, obj_loc, offset, index);
2396 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002397 }
Roland Levillain4d027112015-07-01 15:41:14 +01002398 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002399}
2400
Alexandre Rames5319def2014-10-23 10:03:10 +01002401void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) {
2402 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
2403 locations->SetInAt(0, Location::RequiresRegister());
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00002404 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01002405}
2406
2407void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) {
Vladimir Markodce016e2016-04-28 13:10:02 +01002408 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction);
jessicahandojo05765752016-09-09 19:01:32 -07002409 vixl::aarch64::Register out = OutputRegister(instruction);
Alexandre Ramesd921d642015-04-16 15:07:16 +01002410 BlockPoolsScope block_pools(GetVIXLAssembler());
jessicahandojo05765752016-09-09 19:01:32 -07002411 __ Ldr(out, HeapOperand(InputRegisterAt(instruction, 0), offset));
Calin Juravle77520bc2015-01-12 18:45:46 +00002412 codegen_->MaybeRecordImplicitNullCheck(instruction);
jessicahandojo05765752016-09-09 19:01:32 -07002413 // Mask out compression flag from String's array length.
2414 if (mirror::kUseStringCompression && instruction->IsStringLength()) {
2415 __ And(out.W(), out.W(), Operand(static_cast<int32_t>(INT32_MAX)));
2416 }
Alexandre Rames5319def2014-10-23 10:03:10 +01002417}
2418
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002419void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002420 Primitive::Type value_type = instruction->GetComponentType();
2421
2422 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002423 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
2424 instruction,
Vladimir Marko8d49fd72016-08-25 15:20:47 +01002425 may_need_runtime_call_for_type_check ?
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002426 LocationSummary::kCallOnSlowPath :
2427 LocationSummary::kNoCall);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002428 locations->SetInAt(0, Location::RequiresRegister());
2429 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
Alexandre Ramesbe919d92016-08-23 18:33:36 +01002430 if (IsConstantZeroBitPattern(instruction->InputAt(2))) {
2431 locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant()));
2432 } else if (Primitive::IsFloatingPointType(value_type)) {
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002433 locations->SetInAt(2, Location::RequiresFpuRegister());
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002434 } else {
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002435 locations->SetInAt(2, Location::RequiresRegister());
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002436 }
2437}
2438
2439void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) {
2440 Primitive::Type value_type = instruction->GetComponentType();
Alexandre Rames97833a02015-04-16 15:07:12 +01002441 LocationSummary* locations = instruction->GetLocations();
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002442 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002443 bool needs_write_barrier =
2444 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
Alexandre Rames97833a02015-04-16 15:07:12 +01002445
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002446 Register array = InputRegisterAt(instruction, 0);
Alexandre Ramesbe919d92016-08-23 18:33:36 +01002447 CPURegister value = InputCPURegisterOrZeroRegAt(instruction, 2);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002448 CPURegister source = value;
2449 Location index = locations->InAt(1);
2450 size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value();
2451 MemOperand destination = HeapOperand(array);
2452 MacroAssembler* masm = GetVIXLAssembler();
2453 BlockPoolsScope block_pools(masm);
2454
2455 if (!needs_write_barrier) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002456 DCHECK(!may_need_runtime_call_for_type_check);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002457 if (index.IsConstant()) {
2458 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
2459 destination = HeapOperand(array, offset);
2460 } else {
2461 UseScratchRegisterScope temps(masm);
2462 Register temp = temps.AcquireSameSizeAs(array);
Artem Serov328429f2016-07-06 16:23:04 +01002463 if (instruction->GetArray()->IsIntermediateAddress()) {
Alexandre Ramese6dbf482015-10-19 10:10:41 +01002464 // We do not need to compute the intermediate address from the array: the
2465 // input instruction has done it already. See the comment in
Artem Serov328429f2016-07-06 16:23:04 +01002466 // `TryExtractArrayAccessAddress()`.
Alexandre Ramese6dbf482015-10-19 10:10:41 +01002467 if (kIsDebugBuild) {
Artem Serov328429f2016-07-06 16:23:04 +01002468 HIntermediateAddress* tmp = instruction->GetArray()->AsIntermediateAddress();
Alexandre Ramese6dbf482015-10-19 10:10:41 +01002469 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset);
2470 }
2471 temp = array;
2472 } else {
2473 __ Add(temp, array, offset);
2474 }
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002475 destination = HeapOperand(temp,
2476 XRegisterFrom(index),
2477 LSL,
2478 Primitive::ComponentSizeShift(value_type));
2479 }
2480 codegen_->Store(value_type, value, destination);
2481 codegen_->MaybeRecordImplicitNullCheck(instruction);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002482 } else {
Artem Serov328429f2016-07-06 16:23:04 +01002483 DCHECK(!instruction->GetArray()->IsIntermediateAddress());
Scott Wakeling97c72b72016-06-24 16:19:36 +01002484 vixl::aarch64::Label done;
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002485 SlowPathCodeARM64* slow_path = nullptr;
Alexandre Rames97833a02015-04-16 15:07:12 +01002486 {
2487 // We use a block to end the scratch scope before the write barrier, thus
2488 // freeing the temporary registers so they can be used in `MarkGCCard`.
2489 UseScratchRegisterScope temps(masm);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002490 Register temp = temps.AcquireSameSizeAs(array);
Alexandre Rames97833a02015-04-16 15:07:12 +01002491 if (index.IsConstant()) {
2492 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002493 destination = HeapOperand(array, offset);
Alexandre Rames97833a02015-04-16 15:07:12 +01002494 } else {
Alexandre Rames82000b02015-07-07 11:34:16 +01002495 destination = HeapOperand(temp,
2496 XRegisterFrom(index),
2497 LSL,
2498 Primitive::ComponentSizeShift(value_type));
Alexandre Rames97833a02015-04-16 15:07:12 +01002499 }
2500
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002501 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2502 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2503 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2504
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002505 if (may_need_runtime_call_for_type_check) {
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002506 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM64(instruction);
2507 codegen_->AddSlowPath(slow_path);
2508 if (instruction->GetValueCanBeNull()) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01002509 vixl::aarch64::Label non_zero;
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002510 __ Cbnz(Register(value), &non_zero);
2511 if (!index.IsConstant()) {
2512 __ Add(temp, array, offset);
2513 }
2514 __ Str(wzr, destination);
2515 codegen_->MaybeRecordImplicitNullCheck(instruction);
2516 __ B(&done);
2517 __ Bind(&non_zero);
2518 }
2519
Roland Levillain9d6e1f82016-09-05 15:57:33 +01002520 // Note that when Baker read barriers are enabled, the type
2521 // checks are performed without read barriers. This is fine,
2522 // even in the case where a class object is in the from-space
2523 // after the flip, as a comparison involving such a type would
2524 // not produce a false positive; it may of course produce a
2525 // false negative, in which case we would take the ArraySet
2526 // slow path.
Roland Levillain16d9f942016-08-25 17:27:56 +01002527
Roland Levillain9d6e1f82016-09-05 15:57:33 +01002528 Register temp2 = temps.AcquireSameSizeAs(array);
2529 // /* HeapReference<Class> */ temp = array->klass_
2530 __ Ldr(temp, HeapOperand(array, class_offset));
2531 codegen_->MaybeRecordImplicitNullCheck(instruction);
2532 GetAssembler()->MaybeUnpoisonHeapReference(temp);
Roland Levillain16d9f942016-08-25 17:27:56 +01002533
Roland Levillain9d6e1f82016-09-05 15:57:33 +01002534 // /* HeapReference<Class> */ temp = temp->component_type_
2535 __ Ldr(temp, HeapOperand(temp, component_offset));
2536 // /* HeapReference<Class> */ temp2 = value->klass_
2537 __ Ldr(temp2, HeapOperand(Register(value), class_offset));
2538 // If heap poisoning is enabled, no need to unpoison `temp`
2539 // nor `temp2`, as we are comparing two poisoned references.
2540 __ Cmp(temp, temp2);
2541 temps.Release(temp2);
Roland Levillain16d9f942016-08-25 17:27:56 +01002542
Roland Levillain9d6e1f82016-09-05 15:57:33 +01002543 if (instruction->StaticTypeOfArrayIsObjectArray()) {
2544 vixl::aarch64::Label do_put;
2545 __ B(eq, &do_put);
2546 // If heap poisoning is enabled, the `temp` reference has
2547 // not been unpoisoned yet; unpoison it now.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002548 GetAssembler()->MaybeUnpoisonHeapReference(temp);
2549
Roland Levillain9d6e1f82016-09-05 15:57:33 +01002550 // /* HeapReference<Class> */ temp = temp->super_class_
2551 __ Ldr(temp, HeapOperand(temp, super_offset));
2552 // If heap poisoning is enabled, no need to unpoison
2553 // `temp`, as we are comparing against null below.
2554 __ Cbnz(temp, slow_path->GetEntryLabel());
2555 __ Bind(&do_put);
2556 } else {
2557 __ B(ne, slow_path->GetEntryLabel());
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002558 }
2559 }
2560
2561 if (kPoisonHeapReferences) {
Nicolas Geoffraya8a0fe22015-10-01 15:50:27 +01002562 Register temp2 = temps.AcquireSameSizeAs(array);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002563 DCHECK(value.IsW());
Nicolas Geoffraya8a0fe22015-10-01 15:50:27 +01002564 __ Mov(temp2, value.W());
2565 GetAssembler()->PoisonHeapReference(temp2);
2566 source = temp2;
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002567 }
2568
2569 if (!index.IsConstant()) {
2570 __ Add(temp, array, offset);
2571 }
Nicolas Geoffray61b1dbe2015-10-01 10:27:52 +01002572 __ Str(source, destination);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002573
Roland Levillain22ccc3a2015-11-24 13:10:05 +00002574 if (!may_need_runtime_call_for_type_check) {
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002575 codegen_->MaybeRecordImplicitNullCheck(instruction);
2576 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002577 }
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01002578
2579 codegen_->MarkGCCard(array, value.W(), instruction->GetValueCanBeNull());
2580
2581 if (done.IsLinked()) {
2582 __ Bind(&done);
2583 }
2584
2585 if (slow_path != nullptr) {
2586 __ Bind(slow_path->GetExitLabel());
Alexandre Rames97833a02015-04-16 15:07:12 +01002587 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002588 }
2589}
2590
Alexandre Rames67555f72014-11-18 10:55:16 +00002591void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01002592 RegisterSet caller_saves = RegisterSet::Empty();
2593 InvokeRuntimeCallingConvention calling_convention;
2594 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode()));
2595 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1).GetCode()));
2596 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves);
Alexandre Rames67555f72014-11-18 10:55:16 +00002597 locations->SetInAt(0, Location::RequiresRegister());
Serban Constantinescu760d8ef2015-03-28 18:09:56 +00002598 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
Alexandre Rames67555f72014-11-18 10:55:16 +00002599}
2600
2601void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
Serban Constantinescu5a6cc492015-08-13 15:20:25 +01002602 BoundsCheckSlowPathARM64* slow_path =
2603 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64(instruction);
Alexandre Rames67555f72014-11-18 10:55:16 +00002604 codegen_->AddSlowPath(slow_path);
Alexandre Rames67555f72014-11-18 10:55:16 +00002605 __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1));
2606 __ B(slow_path->GetEntryLabel(), hs);
2607}
2608
Alexandre Rames67555f72014-11-18 10:55:16 +00002609void LocationsBuilderARM64::VisitClinitCheck(HClinitCheck* check) {
2610 LocationSummary* locations =
2611 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
2612 locations->SetInAt(0, Location::RequiresRegister());
2613 if (check->HasUses()) {
2614 locations->SetOut(Location::SameAsFirstInput());
2615 }
2616}
2617
2618void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) {
2619 // We assume the class is not null.
2620 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
2621 check->GetLoadClass(), check, check->GetDexPc(), true);
2622 codegen_->AddSlowPath(slow_path);
2623 GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0));
2624}
2625
Roland Levillain1a653882016-03-18 18:05:57 +00002626static bool IsFloatingPointZeroConstant(HInstruction* inst) {
2627 return (inst->IsFloatConstant() && (inst->AsFloatConstant()->IsArithmeticZero()))
2628 || (inst->IsDoubleConstant() && (inst->AsDoubleConstant()->IsArithmeticZero()));
2629}
2630
2631void InstructionCodeGeneratorARM64::GenerateFcmp(HInstruction* instruction) {
2632 FPRegister lhs_reg = InputFPRegisterAt(instruction, 0);
2633 Location rhs_loc = instruction->GetLocations()->InAt(1);
2634 if (rhs_loc.IsConstant()) {
2635 // 0.0 is the only immediate that can be encoded directly in
2636 // an FCMP instruction.
2637 //
2638 // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
2639 // specify that in a floating-point comparison, positive zero
2640 // and negative zero are considered equal, so we can use the
2641 // literal 0.0 for both cases here.
2642 //
2643 // Note however that some methods (Float.equal, Float.compare,
2644 // Float.compareTo, Double.equal, Double.compare,
2645 // Double.compareTo, Math.max, Math.min, StrictMath.max,
2646 // StrictMath.min) consider 0.0 to be (strictly) greater than
2647 // -0.0. So if we ever translate calls to these methods into a
2648 // HCompare instruction, we must handle the -0.0 case with
2649 // care here.
2650 DCHECK(IsFloatingPointZeroConstant(rhs_loc.GetConstant()));
2651 __ Fcmp(lhs_reg, 0.0);
2652 } else {
2653 __ Fcmp(lhs_reg, InputFPRegisterAt(instruction, 1));
2654 }
Roland Levillain7f63c522015-07-13 15:54:55 +00002655}
2656
Serban Constantinescu02164b32014-11-13 14:05:07 +00002657void LocationsBuilderARM64::VisitCompare(HCompare* compare) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002658 LocationSummary* locations =
Serban Constantinescu02164b32014-11-13 14:05:07 +00002659 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
2660 Primitive::Type in_type = compare->InputAt(0)->GetType();
Alexandre Rames5319def2014-10-23 10:03:10 +01002661 switch (in_type) {
Roland Levillaina5c4a402016-03-15 15:02:50 +00002662 case Primitive::kPrimBoolean:
2663 case Primitive::kPrimByte:
2664 case Primitive::kPrimShort:
2665 case Primitive::kPrimChar:
Aart Bika19616e2016-02-01 18:57:58 -08002666 case Primitive::kPrimInt:
Alexandre Rames5319def2014-10-23 10:03:10 +01002667 case Primitive::kPrimLong: {
Serban Constantinescu02164b32014-11-13 14:05:07 +00002668 locations->SetInAt(0, Location::RequiresRegister());
Serban Constantinescu2d35d9d2015-02-22 22:08:01 +00002669 locations->SetInAt(1, ARM64EncodableConstantOrRegister(compare->InputAt(1), compare));
Serban Constantinescu02164b32014-11-13 14:05:07 +00002670 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2671 break;
2672 }
2673 case Primitive::kPrimFloat:
2674 case Primitive::kPrimDouble: {
2675 locations->SetInAt(0, Location::RequiresFpuRegister());
Roland Levillain7f63c522015-07-13 15:54:55 +00002676 locations->SetInAt(1,
2677 IsFloatingPointZeroConstant(compare->InputAt(1))
2678 ? Location::ConstantLocation(compare->InputAt(1)->AsConstant())
2679 : Location::RequiresFpuRegister());
Serban Constantinescu02164b32014-11-13 14:05:07 +00002680 locations->SetOut(Location::RequiresRegister());
2681 break;
2682 }
2683 default:
2684 LOG(FATAL) << "Unexpected type for compare operation " << in_type;
2685 }
2686}
2687
2688void InstructionCodeGeneratorARM64::VisitCompare(HCompare* compare) {
2689 Primitive::Type in_type = compare->InputAt(0)->GetType();
2690
2691 // 0 if: left == right
2692 // 1 if: left > right
2693 // -1 if: left < right
2694 switch (in_type) {
Roland Levillaina5c4a402016-03-15 15:02:50 +00002695 case Primitive::kPrimBoolean:
2696 case Primitive::kPrimByte:
2697 case Primitive::kPrimShort:
2698 case Primitive::kPrimChar:
Aart Bika19616e2016-02-01 18:57:58 -08002699 case Primitive::kPrimInt:
Serban Constantinescu02164b32014-11-13 14:05:07 +00002700 case Primitive::kPrimLong: {
2701 Register result = OutputRegister(compare);
2702 Register left = InputRegisterAt(compare, 0);
2703 Operand right = InputOperandAt(compare, 1);
Serban Constantinescu02164b32014-11-13 14:05:07 +00002704 __ Cmp(left, right);
Aart Bika19616e2016-02-01 18:57:58 -08002705 __ Cset(result, ne); // result == +1 if NE or 0 otherwise
2706 __ Cneg(result, result, lt); // result == -1 if LT or unchanged otherwise
Serban Constantinescu02164b32014-11-13 14:05:07 +00002707 break;
2708 }
2709 case Primitive::kPrimFloat:
2710 case Primitive::kPrimDouble: {
2711 Register result = OutputRegister(compare);
Roland Levillain1a653882016-03-18 18:05:57 +00002712 GenerateFcmp(compare);
Vladimir Markod6e069b2016-01-18 11:11:01 +00002713 __ Cset(result, ne);
2714 __ Cneg(result, result, ARM64FPCondition(kCondLT, compare->IsGtBias()));
Alexandre Rames5319def2014-10-23 10:03:10 +01002715 break;
2716 }
2717 default:
2718 LOG(FATAL) << "Unimplemented compare type " << in_type;
2719 }
2720}
2721
Vladimir Marko5f7b58e2015-11-23 19:49:34 +00002722void LocationsBuilderARM64::HandleCondition(HCondition* instruction) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002723 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
Roland Levillain7f63c522015-07-13 15:54:55 +00002724
2725 if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
2726 locations->SetInAt(0, Location::RequiresFpuRegister());
2727 locations->SetInAt(1,
2728 IsFloatingPointZeroConstant(instruction->InputAt(1))
2729 ? Location::ConstantLocation(instruction->InputAt(1)->AsConstant())
2730 : Location::RequiresFpuRegister());
2731 } else {
2732 // Integer cases.
2733 locations->SetInAt(0, Location::RequiresRegister());
2734 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
2735 }
2736
David Brazdilb3e773e2016-01-26 11:28:37 +00002737 if (!instruction->IsEmittedAtUseSite()) {
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00002738 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01002739 }
2740}
2741
Vladimir Marko5f7b58e2015-11-23 19:49:34 +00002742void InstructionCodeGeneratorARM64::HandleCondition(HCondition* instruction) {
David Brazdilb3e773e2016-01-26 11:28:37 +00002743 if (instruction->IsEmittedAtUseSite()) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002744 return;
2745 }
2746
2747 LocationSummary* locations = instruction->GetLocations();
Alexandre Rames5319def2014-10-23 10:03:10 +01002748 Register res = RegisterFrom(locations->Out(), instruction->GetType());
Roland Levillain7f63c522015-07-13 15:54:55 +00002749 IfCondition if_cond = instruction->GetCondition();
Alexandre Rames5319def2014-10-23 10:03:10 +01002750
Roland Levillain7f63c522015-07-13 15:54:55 +00002751 if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
Roland Levillain1a653882016-03-18 18:05:57 +00002752 GenerateFcmp(instruction);
Vladimir Markod6e069b2016-01-18 11:11:01 +00002753 __ Cset(res, ARM64FPCondition(if_cond, instruction->IsGtBias()));
Roland Levillain7f63c522015-07-13 15:54:55 +00002754 } else {
2755 // Integer cases.
2756 Register lhs = InputRegisterAt(instruction, 0);
2757 Operand rhs = InputOperandAt(instruction, 1);
2758 __ Cmp(lhs, rhs);
Vladimir Markod6e069b2016-01-18 11:11:01 +00002759 __ Cset(res, ARM64Condition(if_cond));
Roland Levillain7f63c522015-07-13 15:54:55 +00002760 }
Alexandre Rames5319def2014-10-23 10:03:10 +01002761}
2762
2763#define FOR_EACH_CONDITION_INSTRUCTION(M) \
2764 M(Equal) \
2765 M(NotEqual) \
2766 M(LessThan) \
2767 M(LessThanOrEqual) \
2768 M(GreaterThan) \
Aart Bike9f37602015-10-09 11:15:55 -07002769 M(GreaterThanOrEqual) \
2770 M(Below) \
2771 M(BelowOrEqual) \
2772 M(Above) \
2773 M(AboveOrEqual)
Alexandre Rames5319def2014-10-23 10:03:10 +01002774#define DEFINE_CONDITION_VISITORS(Name) \
Vladimir Marko5f7b58e2015-11-23 19:49:34 +00002775void LocationsBuilderARM64::Visit##Name(H##Name* comp) { HandleCondition(comp); } \
2776void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { HandleCondition(comp); }
Alexandre Rames5319def2014-10-23 10:03:10 +01002777FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS)
Alexandre Rames67555f72014-11-18 10:55:16 +00002778#undef DEFINE_CONDITION_VISITORS
Alexandre Rames5319def2014-10-23 10:03:10 +01002779#undef FOR_EACH_CONDITION_INSTRUCTION
2780
Zheng Xuc6667102015-05-15 16:08:45 +08002781void InstructionCodeGeneratorARM64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2782 DCHECK(instruction->IsDiv() || instruction->IsRem());
2783
2784 LocationSummary* locations = instruction->GetLocations();
2785 Location second = locations->InAt(1);
2786 DCHECK(second.IsConstant());
2787
2788 Register out = OutputRegister(instruction);
2789 Register dividend = InputRegisterAt(instruction, 0);
2790 int64_t imm = Int64FromConstant(second.GetConstant());
2791 DCHECK(imm == 1 || imm == -1);
2792
2793 if (instruction->IsRem()) {
2794 __ Mov(out, 0);
2795 } else {
2796 if (imm == 1) {
2797 __ Mov(out, dividend);
2798 } else {
2799 __ Neg(out, dividend);
2800 }
2801 }
2802}
2803
2804void InstructionCodeGeneratorARM64::DivRemByPowerOfTwo(HBinaryOperation* instruction) {
2805 DCHECK(instruction->IsDiv() || instruction->IsRem());
2806
2807 LocationSummary* locations = instruction->GetLocations();
2808 Location second = locations->InAt(1);
2809 DCHECK(second.IsConstant());
2810
2811 Register out = OutputRegister(instruction);
2812 Register dividend = InputRegisterAt(instruction, 0);
2813 int64_t imm = Int64FromConstant(second.GetConstant());
Nicolas Geoffray68f62892016-01-04 08:39:49 +00002814 uint64_t abs_imm = static_cast<uint64_t>(AbsOrMin(imm));
Zheng Xuc6667102015-05-15 16:08:45 +08002815 int ctz_imm = CTZ(abs_imm);
2816
2817 UseScratchRegisterScope temps(GetVIXLAssembler());
2818 Register temp = temps.AcquireSameSizeAs(out);
2819
2820 if (instruction->IsDiv()) {
2821 __ Add(temp, dividend, abs_imm - 1);
2822 __ Cmp(dividend, 0);
2823 __ Csel(out, temp, dividend, lt);
2824 if (imm > 0) {
2825 __ Asr(out, out, ctz_imm);
2826 } else {
2827 __ Neg(out, Operand(out, ASR, ctz_imm));
2828 }
2829 } else {
2830 int bits = instruction->GetResultType() == Primitive::kPrimInt ? 32 : 64;
2831 __ Asr(temp, dividend, bits - 1);
2832 __ Lsr(temp, temp, bits - ctz_imm);
2833 __ Add(out, dividend, temp);
2834 __ And(out, out, abs_imm - 1);
2835 __ Sub(out, out, temp);
2836 }
2837}
2838
2839void InstructionCodeGeneratorARM64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2840 DCHECK(instruction->IsDiv() || instruction->IsRem());
2841
2842 LocationSummary* locations = instruction->GetLocations();
2843 Location second = locations->InAt(1);
2844 DCHECK(second.IsConstant());
2845
2846 Register out = OutputRegister(instruction);
2847 Register dividend = InputRegisterAt(instruction, 0);
2848 int64_t imm = Int64FromConstant(second.GetConstant());
2849
2850 Primitive::Type type = instruction->GetResultType();
2851 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
2852
2853 int64_t magic;
2854 int shift;
2855 CalculateMagicAndShiftForDivRem(imm, type == Primitive::kPrimLong /* is_long */, &magic, &shift);
2856
2857 UseScratchRegisterScope temps(GetVIXLAssembler());
2858 Register temp = temps.AcquireSameSizeAs(out);
2859
2860 // temp = get_high(dividend * magic)
2861 __ Mov(temp, magic);
2862 if (type == Primitive::kPrimLong) {
2863 __ Smulh(temp, dividend, temp);
2864 } else {
2865 __ Smull(temp.X(), dividend, temp);
2866 __ Lsr(temp.X(), temp.X(), 32);
2867 }
2868
2869 if (imm > 0 && magic < 0) {
2870 __ Add(temp, temp, dividend);
2871 } else if (imm < 0 && magic > 0) {
2872 __ Sub(temp, temp, dividend);
2873 }
2874
2875 if (shift != 0) {
2876 __ Asr(temp, temp, shift);
2877 }
2878
2879 if (instruction->IsDiv()) {
2880 __ Sub(out, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2881 } else {
2882 __ Sub(temp, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2883 // TODO: Strength reduction for msub.
2884 Register temp_imm = temps.AcquireSameSizeAs(out);
2885 __ Mov(temp_imm, imm);
2886 __ Msub(out, temp, temp_imm, dividend);
2887 }
2888}
2889
2890void InstructionCodeGeneratorARM64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2891 DCHECK(instruction->IsDiv() || instruction->IsRem());
2892 Primitive::Type type = instruction->GetResultType();
2893 DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2894
2895 LocationSummary* locations = instruction->GetLocations();
2896 Register out = OutputRegister(instruction);
2897 Location second = locations->InAt(1);
2898
2899 if (second.IsConstant()) {
2900 int64_t imm = Int64FromConstant(second.GetConstant());
2901
2902 if (imm == 0) {
2903 // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2904 } else if (imm == 1 || imm == -1) {
2905 DivRemOneOrMinusOne(instruction);
Nicolas Geoffray68f62892016-01-04 08:39:49 +00002906 } else if (IsPowerOfTwo(AbsOrMin(imm))) {
Zheng Xuc6667102015-05-15 16:08:45 +08002907 DivRemByPowerOfTwo(instruction);
2908 } else {
2909 DCHECK(imm <= -2 || imm >= 2);
2910 GenerateDivRemWithAnyConstant(instruction);
2911 }
2912 } else {
2913 Register dividend = InputRegisterAt(instruction, 0);
2914 Register divisor = InputRegisterAt(instruction, 1);
2915 if (instruction->IsDiv()) {
2916 __ Sdiv(out, dividend, divisor);
2917 } else {
2918 UseScratchRegisterScope temps(GetVIXLAssembler());
2919 Register temp = temps.AcquireSameSizeAs(out);
2920 __ Sdiv(temp, dividend, divisor);
2921 __ Msub(out, temp, divisor, dividend);
2922 }
2923 }
2924}
2925
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002926void LocationsBuilderARM64::VisitDiv(HDiv* div) {
2927 LocationSummary* locations =
2928 new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2929 switch (div->GetResultType()) {
2930 case Primitive::kPrimInt:
2931 case Primitive::kPrimLong:
2932 locations->SetInAt(0, Location::RequiresRegister());
Zheng Xuc6667102015-05-15 16:08:45 +08002933 locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002934 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2935 break;
2936
2937 case Primitive::kPrimFloat:
2938 case Primitive::kPrimDouble:
2939 locations->SetInAt(0, Location::RequiresFpuRegister());
2940 locations->SetInAt(1, Location::RequiresFpuRegister());
2941 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2942 break;
2943
2944 default:
2945 LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2946 }
2947}
2948
2949void InstructionCodeGeneratorARM64::VisitDiv(HDiv* div) {
2950 Primitive::Type type = div->GetResultType();
2951 switch (type) {
2952 case Primitive::kPrimInt:
2953 case Primitive::kPrimLong:
Zheng Xuc6667102015-05-15 16:08:45 +08002954 GenerateDivRemIntegral(div);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002955 break;
2956
2957 case Primitive::kPrimFloat:
2958 case Primitive::kPrimDouble:
2959 __ Fdiv(OutputFPRegister(div), InputFPRegisterAt(div, 0), InputFPRegisterAt(div, 1));
2960 break;
2961
2962 default:
2963 LOG(FATAL) << "Unexpected div type " << type;
2964 }
2965}
2966
Alexandre Rames67555f72014-11-18 10:55:16 +00002967void LocationsBuilderARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01002968 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction);
Alexandre Rames67555f72014-11-18 10:55:16 +00002969 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
Alexandre Rames67555f72014-11-18 10:55:16 +00002970}
2971
2972void InstructionCodeGeneratorARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2973 SlowPathCodeARM64* slow_path =
2974 new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM64(instruction);
2975 codegen_->AddSlowPath(slow_path);
2976 Location value = instruction->GetLocations()->InAt(0);
2977
Alexandre Rames3e69f162014-12-10 10:36:50 +00002978 Primitive::Type type = instruction->GetType();
2979
Nicolas Geoffraye5671612016-03-16 11:03:54 +00002980 if (!Primitive::IsIntegralType(type)) {
2981 LOG(FATAL) << "Unexpected type " << type << " for DivZeroCheck.";
Alexandre Rames3e69f162014-12-10 10:36:50 +00002982 return;
2983 }
2984
Alexandre Rames67555f72014-11-18 10:55:16 +00002985 if (value.IsConstant()) {
2986 int64_t divisor = Int64ConstantFrom(value);
2987 if (divisor == 0) {
2988 __ B(slow_path->GetEntryLabel());
2989 } else {
Alexandre Rames3e69f162014-12-10 10:36:50 +00002990 // A division by a non-null constant is valid. We don't need to perform
2991 // any check, so simply fall through.
Alexandre Rames67555f72014-11-18 10:55:16 +00002992 }
2993 } else {
2994 __ Cbz(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel());
2995 }
2996}
2997
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002998void LocationsBuilderARM64::VisitDoubleConstant(HDoubleConstant* constant) {
2999 LocationSummary* locations =
3000 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
3001 locations->SetOut(Location::ConstantLocation(constant));
3002}
3003
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003004void InstructionCodeGeneratorARM64::VisitDoubleConstant(
3005 HDoubleConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003006 // Will be generated at use site.
3007}
3008
Alexandre Rames5319def2014-10-23 10:03:10 +01003009void LocationsBuilderARM64::VisitExit(HExit* exit) {
3010 exit->SetLocations(nullptr);
3011}
3012
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003013void InstructionCodeGeneratorARM64::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003014}
3015
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003016void LocationsBuilderARM64::VisitFloatConstant(HFloatConstant* constant) {
3017 LocationSummary* locations =
3018 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
3019 locations->SetOut(Location::ConstantLocation(constant));
3020}
3021
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003022void InstructionCodeGeneratorARM64::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003023 // Will be generated at use site.
3024}
3025
David Brazdilfc6a86a2015-06-26 10:33:45 +00003026void InstructionCodeGeneratorARM64::HandleGoto(HInstruction* got, HBasicBlock* successor) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00003027 DCHECK(!successor->IsExitBlock());
3028 HBasicBlock* block = got->GetBlock();
3029 HInstruction* previous = got->GetPrevious();
3030 HLoopInformation* info = block->GetLoopInformation();
3031
David Brazdil46e2a392015-03-16 17:31:52 +00003032 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00003033 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
3034 GenerateSuspendCheck(info->GetSuspendCheck(), successor);
3035 return;
3036 }
3037 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
3038 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
3039 }
3040 if (!codegen_->GoesToNextBlock(block, successor)) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003041 __ B(codegen_->GetLabelOf(successor));
3042 }
3043}
3044
David Brazdilfc6a86a2015-06-26 10:33:45 +00003045void LocationsBuilderARM64::VisitGoto(HGoto* got) {
3046 got->SetLocations(nullptr);
3047}
3048
3049void InstructionCodeGeneratorARM64::VisitGoto(HGoto* got) {
3050 HandleGoto(got, got->GetSuccessor());
3051}
3052
3053void LocationsBuilderARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
3054 try_boundary->SetLocations(nullptr);
3055}
3056
3057void InstructionCodeGeneratorARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
3058 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
3059 if (!successor->IsExitBlock()) {
3060 HandleGoto(try_boundary, successor);
3061 }
3062}
3063
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07003064void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruction,
David Brazdil0debae72015-11-12 18:37:00 +00003065 size_t condition_input_index,
Scott Wakeling97c72b72016-06-24 16:19:36 +01003066 vixl::aarch64::Label* true_target,
3067 vixl::aarch64::Label* false_target) {
David Brazdil0debae72015-11-12 18:37:00 +00003068 // FP branching requires both targets to be explicit. If either of the targets
3069 // is nullptr (fallthrough) use and bind `fallthrough_target` instead.
Scott Wakeling97c72b72016-06-24 16:19:36 +01003070 vixl::aarch64::Label fallthrough_target;
David Brazdil0debae72015-11-12 18:37:00 +00003071 HInstruction* cond = instruction->InputAt(condition_input_index);
Alexandre Rames5319def2014-10-23 10:03:10 +01003072
David Brazdil0debae72015-11-12 18:37:00 +00003073 if (true_target == nullptr && false_target == nullptr) {
3074 // Nothing to do. The code always falls through.
3075 return;
3076 } else if (cond->IsIntConstant()) {
Roland Levillain1a653882016-03-18 18:05:57 +00003077 // Constant condition, statically compared against "true" (integer value 1).
3078 if (cond->AsIntConstant()->IsTrue()) {
David Brazdil0debae72015-11-12 18:37:00 +00003079 if (true_target != nullptr) {
3080 __ B(true_target);
Serban Constantinescu02164b32014-11-13 14:05:07 +00003081 }
Serban Constantinescu02164b32014-11-13 14:05:07 +00003082 } else {
Roland Levillain1a653882016-03-18 18:05:57 +00003083 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue();
David Brazdil0debae72015-11-12 18:37:00 +00003084 if (false_target != nullptr) {
3085 __ B(false_target);
3086 }
Serban Constantinescu02164b32014-11-13 14:05:07 +00003087 }
David Brazdil0debae72015-11-12 18:37:00 +00003088 return;
3089 }
3090
3091 // The following code generates these patterns:
3092 // (1) true_target == nullptr && false_target != nullptr
3093 // - opposite condition true => branch to false_target
3094 // (2) true_target != nullptr && false_target == nullptr
3095 // - condition true => branch to true_target
3096 // (3) true_target != nullptr && false_target != nullptr
3097 // - condition true => branch to true_target
3098 // - branch to false_target
3099 if (IsBooleanValueOrMaterializedCondition(cond)) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003100 // The condition instruction has been materialized, compare the output to 0.
David Brazdil0debae72015-11-12 18:37:00 +00003101 Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
Alexandre Rames5319def2014-10-23 10:03:10 +01003102 DCHECK(cond_val.IsRegister());
David Brazdil0debae72015-11-12 18:37:00 +00003103 if (true_target == nullptr) {
3104 __ Cbz(InputRegisterAt(instruction, condition_input_index), false_target);
3105 } else {
3106 __ Cbnz(InputRegisterAt(instruction, condition_input_index), true_target);
3107 }
Alexandre Rames5319def2014-10-23 10:03:10 +01003108 } else {
3109 // The condition instruction has not been materialized, use its inputs as
3110 // the comparison and its condition as the branch condition.
David Brazdil0debae72015-11-12 18:37:00 +00003111 HCondition* condition = cond->AsCondition();
Roland Levillain7f63c522015-07-13 15:54:55 +00003112
David Brazdil0debae72015-11-12 18:37:00 +00003113 Primitive::Type type = condition->InputAt(0)->GetType();
Roland Levillain7f63c522015-07-13 15:54:55 +00003114 if (Primitive::IsFloatingPointType(type)) {
Roland Levillain1a653882016-03-18 18:05:57 +00003115 GenerateFcmp(condition);
David Brazdil0debae72015-11-12 18:37:00 +00003116 if (true_target == nullptr) {
Vladimir Markod6e069b2016-01-18 11:11:01 +00003117 IfCondition opposite_condition = condition->GetOppositeCondition();
3118 __ B(ARM64FPCondition(opposite_condition, condition->IsGtBias()), false_target);
David Brazdil0debae72015-11-12 18:37:00 +00003119 } else {
Vladimir Markod6e069b2016-01-18 11:11:01 +00003120 __ B(ARM64FPCondition(condition->GetCondition(), condition->IsGtBias()), true_target);
David Brazdil0debae72015-11-12 18:37:00 +00003121 }
Alexandre Rames5319def2014-10-23 10:03:10 +01003122 } else {
Roland Levillain7f63c522015-07-13 15:54:55 +00003123 // Integer cases.
3124 Register lhs = InputRegisterAt(condition, 0);
3125 Operand rhs = InputOperandAt(condition, 1);
David Brazdil0debae72015-11-12 18:37:00 +00003126
3127 Condition arm64_cond;
Scott Wakeling97c72b72016-06-24 16:19:36 +01003128 vixl::aarch64::Label* non_fallthrough_target;
David Brazdil0debae72015-11-12 18:37:00 +00003129 if (true_target == nullptr) {
3130 arm64_cond = ARM64Condition(condition->GetOppositeCondition());
3131 non_fallthrough_target = false_target;
3132 } else {
3133 arm64_cond = ARM64Condition(condition->GetCondition());
3134 non_fallthrough_target = true_target;
3135 }
3136
Aart Bik086d27e2016-01-20 17:02:00 -08003137 if ((arm64_cond == eq || arm64_cond == ne || arm64_cond == lt || arm64_cond == ge) &&
Scott Wakeling97c72b72016-06-24 16:19:36 +01003138 rhs.IsImmediate() && (rhs.GetImmediate() == 0)) {
Roland Levillain7f63c522015-07-13 15:54:55 +00003139 switch (arm64_cond) {
3140 case eq:
David Brazdil0debae72015-11-12 18:37:00 +00003141 __ Cbz(lhs, non_fallthrough_target);
Roland Levillain7f63c522015-07-13 15:54:55 +00003142 break;
3143 case ne:
David Brazdil0debae72015-11-12 18:37:00 +00003144 __ Cbnz(lhs, non_fallthrough_target);
Roland Levillain7f63c522015-07-13 15:54:55 +00003145 break;
3146 case lt:
3147 // Test the sign bit and branch accordingly.
David Brazdil0debae72015-11-12 18:37:00 +00003148 __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target);
Roland Levillain7f63c522015-07-13 15:54:55 +00003149 break;
3150 case ge:
3151 // Test the sign bit and branch accordingly.
David Brazdil0debae72015-11-12 18:37:00 +00003152 __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target);
Roland Levillain7f63c522015-07-13 15:54:55 +00003153 break;
3154 default:
3155 // Without the `static_cast` the compiler throws an error for
3156 // `-Werror=sign-promo`.
3157 LOG(FATAL) << "Unexpected condition: " << static_cast<int>(arm64_cond);
3158 }
3159 } else {
3160 __ Cmp(lhs, rhs);
David Brazdil0debae72015-11-12 18:37:00 +00003161 __ B(arm64_cond, non_fallthrough_target);
Roland Levillain7f63c522015-07-13 15:54:55 +00003162 }
Alexandre Rames5319def2014-10-23 10:03:10 +01003163 }
3164 }
David Brazdil0debae72015-11-12 18:37:00 +00003165
3166 // If neither branch falls through (case 3), the conditional branch to `true_target`
3167 // was already emitted (case 2) and we need to emit a jump to `false_target`.
3168 if (true_target != nullptr && false_target != nullptr) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003169 __ B(false_target);
3170 }
David Brazdil0debae72015-11-12 18:37:00 +00003171
3172 if (fallthrough_target.IsLinked()) {
3173 __ Bind(&fallthrough_target);
3174 }
Alexandre Rames5319def2014-10-23 10:03:10 +01003175}
3176
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07003177void LocationsBuilderARM64::VisitIf(HIf* if_instr) {
3178 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
David Brazdil0debae72015-11-12 18:37:00 +00003179 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07003180 locations->SetInAt(0, Location::RequiresRegister());
3181 }
3182}
3183
3184void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) {
David Brazdil0debae72015-11-12 18:37:00 +00003185 HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
3186 HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
Scott Wakeling97c72b72016-06-24 16:19:36 +01003187 vixl::aarch64::Label* true_target = codegen_->GetLabelOf(true_successor);
3188 if (codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor)) {
3189 true_target = nullptr;
3190 }
3191 vixl::aarch64::Label* false_target = codegen_->GetLabelOf(false_successor);
3192 if (codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor)) {
3193 false_target = nullptr;
3194 }
David Brazdil0debae72015-11-12 18:37:00 +00003195 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07003196}
3197
3198void LocationsBuilderARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
3199 LocationSummary* locations = new (GetGraph()->GetArena())
3200 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
Vladimir Marko804b03f2016-09-14 16:26:36 +01003201 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
David Brazdil0debae72015-11-12 18:37:00 +00003202 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07003203 locations->SetInAt(0, Location::RequiresRegister());
3204 }
3205}
3206
3207void InstructionCodeGeneratorARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
Aart Bik42249c32016-01-07 15:33:50 -08003208 SlowPathCodeARM64* slow_path =
3209 deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM64>(deoptimize);
David Brazdil0debae72015-11-12 18:37:00 +00003210 GenerateTestAndBranch(deoptimize,
3211 /* condition_input_index */ 0,
3212 slow_path->GetEntryLabel(),
3213 /* false_target */ nullptr);
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07003214}
3215
David Brazdilc0b601b2016-02-08 14:20:45 +00003216static inline bool IsConditionOnFloatingPointValues(HInstruction* condition) {
3217 return condition->IsCondition() &&
3218 Primitive::IsFloatingPointType(condition->InputAt(0)->GetType());
3219}
3220
Alexandre Rames880f1192016-06-13 16:04:50 +01003221static inline Condition GetConditionForSelect(HCondition* condition) {
3222 IfCondition cond = condition->AsCondition()->GetCondition();
David Brazdilc0b601b2016-02-08 14:20:45 +00003223 return IsConditionOnFloatingPointValues(condition) ? ARM64FPCondition(cond, condition->IsGtBias())
3224 : ARM64Condition(cond);
3225}
3226
David Brazdil74eb1b22015-12-14 11:44:01 +00003227void LocationsBuilderARM64::VisitSelect(HSelect* select) {
3228 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
Alexandre Rames880f1192016-06-13 16:04:50 +01003229 if (Primitive::IsFloatingPointType(select->GetType())) {
3230 locations->SetInAt(0, Location::RequiresFpuRegister());
3231 locations->SetInAt(1, Location::RequiresFpuRegister());
3232 locations->SetOut(Location::RequiresFpuRegister());
3233 } else {
3234 HConstant* cst_true_value = select->GetTrueValue()->AsConstant();
3235 HConstant* cst_false_value = select->GetFalseValue()->AsConstant();
3236 bool is_true_value_constant = cst_true_value != nullptr;
3237 bool is_false_value_constant = cst_false_value != nullptr;
3238 // Ask VIXL whether we should synthesize constants in registers.
3239 // We give an arbitrary register to VIXL when dealing with non-constant inputs.
3240 Operand true_op = is_true_value_constant ?
3241 Operand(Int64FromConstant(cst_true_value)) : Operand(x1);
3242 Operand false_op = is_false_value_constant ?
3243 Operand(Int64FromConstant(cst_false_value)) : Operand(x2);
3244 bool true_value_in_register = false;
3245 bool false_value_in_register = false;
3246 MacroAssembler::GetCselSynthesisInformation(
3247 x0, true_op, false_op, &true_value_in_register, &false_value_in_register);
3248 true_value_in_register |= !is_true_value_constant;
3249 false_value_in_register |= !is_false_value_constant;
3250
3251 locations->SetInAt(1, true_value_in_register ? Location::RequiresRegister()
3252 : Location::ConstantLocation(cst_true_value));
3253 locations->SetInAt(0, false_value_in_register ? Location::RequiresRegister()
3254 : Location::ConstantLocation(cst_false_value));
3255 locations->SetOut(Location::RequiresRegister());
David Brazdil74eb1b22015-12-14 11:44:01 +00003256 }
Alexandre Rames880f1192016-06-13 16:04:50 +01003257
David Brazdil74eb1b22015-12-14 11:44:01 +00003258 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
3259 locations->SetInAt(2, Location::RequiresRegister());
3260 }
David Brazdil74eb1b22015-12-14 11:44:01 +00003261}
3262
3263void InstructionCodeGeneratorARM64::VisitSelect(HSelect* select) {
David Brazdilc0b601b2016-02-08 14:20:45 +00003264 HInstruction* cond = select->GetCondition();
David Brazdilc0b601b2016-02-08 14:20:45 +00003265 Condition csel_cond;
3266
3267 if (IsBooleanValueOrMaterializedCondition(cond)) {
3268 if (cond->IsCondition() && cond->GetNext() == select) {
Alexandre Rames880f1192016-06-13 16:04:50 +01003269 // Use the condition flags set by the previous instruction.
3270 csel_cond = GetConditionForSelect(cond->AsCondition());
David Brazdilc0b601b2016-02-08 14:20:45 +00003271 } else {
3272 __ Cmp(InputRegisterAt(select, 2), 0);
Alexandre Rames880f1192016-06-13 16:04:50 +01003273 csel_cond = ne;
David Brazdilc0b601b2016-02-08 14:20:45 +00003274 }
3275 } else if (IsConditionOnFloatingPointValues(cond)) {
Roland Levillain1a653882016-03-18 18:05:57 +00003276 GenerateFcmp(cond);
Alexandre Rames880f1192016-06-13 16:04:50 +01003277 csel_cond = GetConditionForSelect(cond->AsCondition());
David Brazdilc0b601b2016-02-08 14:20:45 +00003278 } else {
3279 __ Cmp(InputRegisterAt(cond, 0), InputOperandAt(cond, 1));
Alexandre Rames880f1192016-06-13 16:04:50 +01003280 csel_cond = GetConditionForSelect(cond->AsCondition());
David Brazdilc0b601b2016-02-08 14:20:45 +00003281 }
3282
Alexandre Rames880f1192016-06-13 16:04:50 +01003283 if (Primitive::IsFloatingPointType(select->GetType())) {
3284 __ Fcsel(OutputFPRegister(select),
3285 InputFPRegisterAt(select, 1),
3286 InputFPRegisterAt(select, 0),
3287 csel_cond);
3288 } else {
3289 __ Csel(OutputRegister(select),
3290 InputOperandAt(select, 1),
3291 InputOperandAt(select, 0),
3292 csel_cond);
David Brazdilc0b601b2016-02-08 14:20:45 +00003293 }
David Brazdil74eb1b22015-12-14 11:44:01 +00003294}
3295
David Srbecky0cf44932015-12-09 14:09:59 +00003296void LocationsBuilderARM64::VisitNativeDebugInfo(HNativeDebugInfo* info) {
3297 new (GetGraph()->GetArena()) LocationSummary(info);
3298}
3299
David Srbeckyd28f4a02016-03-14 17:14:24 +00003300void InstructionCodeGeneratorARM64::VisitNativeDebugInfo(HNativeDebugInfo*) {
3301 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile.
David Srbeckyc7098ff2016-02-09 14:30:11 +00003302}
3303
3304void CodeGeneratorARM64::GenerateNop() {
3305 __ Nop();
David Srbecky0cf44932015-12-09 14:09:59 +00003306}
3307
Alexandre Rames5319def2014-10-23 10:03:10 +01003308void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01003309 HandleFieldGet(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01003310}
3311
3312void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01003313 HandleFieldGet(instruction, instruction->GetFieldInfo());
Alexandre Rames5319def2014-10-23 10:03:10 +01003314}
3315
3316void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01003317 HandleFieldSet(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01003318}
3319
3320void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01003321 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
Alexandre Rames5319def2014-10-23 10:03:10 +01003322}
3323
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003324// Temp is used for read barrier.
3325static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) {
3326 if (kEmitCompilerReadBarrier &&
Roland Levillain44015862016-01-22 11:47:17 +00003327 (kUseBakerReadBarrier ||
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003328 type_check_kind == TypeCheckKind::kAbstractClassCheck ||
3329 type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
3330 type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
3331 return 1;
3332 }
3333 return 0;
3334}
3335
3336// InteraceCheck has 3 temps, one for holding the number of interfaces, one for the current
3337// interface pointer, one for loading the current interface.
3338// The other checks have one temp for loading the object's class.
3339static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) {
3340 if (type_check_kind == TypeCheckKind::kInterfaceCheck) {
3341 return 3;
3342 }
3343 return 1 + NumberOfInstanceOfTemps(type_check_kind);
Roland Levillain44015862016-01-22 11:47:17 +00003344}
3345
Alexandre Rames67555f72014-11-18 10:55:16 +00003346void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003347 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003348 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
Vladimir Marko70e97462016-08-09 11:04:26 +01003349 bool baker_read_barrier_slow_path = false;
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003350 switch (type_check_kind) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003351 case TypeCheckKind::kExactCheck:
3352 case TypeCheckKind::kAbstractClassCheck:
3353 case TypeCheckKind::kClassHierarchyCheck:
3354 case TypeCheckKind::kArrayObjectCheck:
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003355 call_kind =
3356 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
Vladimir Marko70e97462016-08-09 11:04:26 +01003357 baker_read_barrier_slow_path = kUseBakerReadBarrier;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003358 break;
3359 case TypeCheckKind::kArrayCheck:
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003360 case TypeCheckKind::kUnresolvedCheck:
3361 case TypeCheckKind::kInterfaceCheck:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003362 call_kind = LocationSummary::kCallOnSlowPath;
3363 break;
3364 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003365
Alexandre Rames67555f72014-11-18 10:55:16 +00003366 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
Vladimir Marko70e97462016-08-09 11:04:26 +01003367 if (baker_read_barrier_slow_path) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01003368 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
Vladimir Marko70e97462016-08-09 11:04:26 +01003369 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003370 locations->SetInAt(0, Location::RequiresRegister());
3371 locations->SetInAt(1, Location::RequiresRegister());
3372 // The "out" register is used as a temporary, so it overlaps with the inputs.
3373 // Note that TypeCheckSlowPathARM64 uses this register too.
3374 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003375 // Add temps if necessary for read barriers.
3376 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind));
Alexandre Rames67555f72014-11-18 10:55:16 +00003377}
3378
3379void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) {
Roland Levillain44015862016-01-22 11:47:17 +00003380 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
Alexandre Rames67555f72014-11-18 10:55:16 +00003381 LocationSummary* locations = instruction->GetLocations();
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003382 Location obj_loc = locations->InAt(0);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003383 Register obj = InputRegisterAt(instruction, 0);
3384 Register cls = InputRegisterAt(instruction, 1);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003385 Location out_loc = locations->Out();
Alexandre Rames67555f72014-11-18 10:55:16 +00003386 Register out = OutputRegister(instruction);
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003387 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind);
3388 DCHECK_LE(num_temps, 1u);
3389 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation();
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003390 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3391 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
3392 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
3393 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
Alexandre Rames67555f72014-11-18 10:55:16 +00003394
Scott Wakeling97c72b72016-06-24 16:19:36 +01003395 vixl::aarch64::Label done, zero;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003396 SlowPathCodeARM64* slow_path = nullptr;
Alexandre Rames67555f72014-11-18 10:55:16 +00003397
3398 // Return 0 if `obj` is null.
Guillaume "Vermeille" Sanchezaf888352015-04-20 14:41:30 +01003399 // Avoid null check if we know `obj` is not null.
3400 if (instruction->MustDoNullCheck()) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003401 __ Cbz(obj, &zero);
3402 }
3403
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003404 // /* HeapReference<Class> */ out = obj->klass_
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003405 GenerateReferenceLoadTwoRegisters(instruction,
3406 out_loc,
3407 obj_loc,
3408 class_offset,
3409 maybe_temp_loc,
3410 kEmitCompilerReadBarrier);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003411
Roland Levillain44015862016-01-22 11:47:17 +00003412 switch (type_check_kind) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003413 case TypeCheckKind::kExactCheck: {
3414 __ Cmp(out, cls);
3415 __ Cset(out, eq);
3416 if (zero.IsLinked()) {
3417 __ B(&done);
3418 }
3419 break;
3420 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003421
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003422 case TypeCheckKind::kAbstractClassCheck: {
3423 // If the class is abstract, we eagerly fetch the super class of the
3424 // object to avoid doing a comparison we know will fail.
Scott Wakeling97c72b72016-06-24 16:19:36 +01003425 vixl::aarch64::Label loop, success;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003426 __ Bind(&loop);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003427 // /* HeapReference<Class> */ out = out->super_class_
Roland Levillain44015862016-01-22 11:47:17 +00003428 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003429 // If `out` is null, we use it for the result, and jump to `done`.
3430 __ Cbz(out, &done);
3431 __ Cmp(out, cls);
3432 __ B(ne, &loop);
3433 __ Mov(out, 1);
3434 if (zero.IsLinked()) {
3435 __ B(&done);
3436 }
3437 break;
3438 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003439
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003440 case TypeCheckKind::kClassHierarchyCheck: {
3441 // Walk over the class hierarchy to find a match.
Scott Wakeling97c72b72016-06-24 16:19:36 +01003442 vixl::aarch64::Label loop, success;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003443 __ Bind(&loop);
3444 __ Cmp(out, cls);
3445 __ B(eq, &success);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003446 // /* HeapReference<Class> */ out = out->super_class_
Roland Levillain44015862016-01-22 11:47:17 +00003447 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003448 __ Cbnz(out, &loop);
3449 // If `out` is null, we use it for the result, and jump to `done`.
3450 __ B(&done);
3451 __ Bind(&success);
3452 __ Mov(out, 1);
3453 if (zero.IsLinked()) {
3454 __ B(&done);
3455 }
3456 break;
3457 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003458
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003459 case TypeCheckKind::kArrayObjectCheck: {
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01003460 // Do an exact check.
Scott Wakeling97c72b72016-06-24 16:19:36 +01003461 vixl::aarch64::Label exact_check;
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01003462 __ Cmp(out, cls);
3463 __ B(eq, &exact_check);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003464 // Otherwise, we need to check that the object's class is a non-primitive array.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003465 // /* HeapReference<Class> */ out = out->component_type_
Roland Levillain44015862016-01-22 11:47:17 +00003466 GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, maybe_temp_loc);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003467 // If `out` is null, we use it for the result, and jump to `done`.
3468 __ Cbz(out, &done);
3469 __ Ldrh(out, HeapOperand(out, primitive_offset));
3470 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
3471 __ Cbnz(out, &zero);
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01003472 __ Bind(&exact_check);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003473 __ Mov(out, 1);
3474 __ B(&done);
3475 break;
3476 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003477
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003478 case TypeCheckKind::kArrayCheck: {
3479 __ Cmp(out, cls);
3480 DCHECK(locations->OnlyCallsOnSlowPath());
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003481 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction,
3482 /* is_fatal */ false);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003483 codegen_->AddSlowPath(slow_path);
3484 __ B(ne, slow_path->GetEntryLabel());
3485 __ Mov(out, 1);
3486 if (zero.IsLinked()) {
3487 __ B(&done);
3488 }
3489 break;
3490 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003491
Calin Juravle98893e12015-10-02 21:05:03 +01003492 case TypeCheckKind::kUnresolvedCheck:
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003493 case TypeCheckKind::kInterfaceCheck: {
3494 // Note that we indeed only call on slow path, but we always go
3495 // into the slow path for the unresolved and interface check
3496 // cases.
3497 //
3498 // We cannot directly call the InstanceofNonTrivial runtime
3499 // entry point without resorting to a type checking slow path
3500 // here (i.e. by calling InvokeRuntime directly), as it would
3501 // require to assign fixed registers for the inputs of this
3502 // HInstanceOf instruction (following the runtime calling
3503 // convention), which might be cluttered by the potential first
3504 // read barrier emission at the beginning of this method.
Roland Levillain44015862016-01-22 11:47:17 +00003505 //
3506 // TODO: Introduce a new runtime entry point taking the object
3507 // to test (instead of its class) as argument, and let it deal
3508 // with the read barrier issues. This will let us refactor this
3509 // case of the `switch` code as it was previously (with a direct
3510 // call to the runtime not using a type checking slow path).
3511 // This should also be beneficial for the other cases above.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003512 DCHECK(locations->OnlyCallsOnSlowPath());
3513 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction,
3514 /* is_fatal */ false);
3515 codegen_->AddSlowPath(slow_path);
3516 __ B(slow_path->GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003517 if (zero.IsLinked()) {
3518 __ B(&done);
3519 }
3520 break;
3521 }
3522 }
3523
3524 if (zero.IsLinked()) {
3525 __ Bind(&zero);
Guillaume "Vermeille" Sanchezaf888352015-04-20 14:41:30 +01003526 __ Mov(out, 0);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003527 }
3528
3529 if (done.IsLinked()) {
3530 __ Bind(&done);
3531 }
3532
3533 if (slow_path != nullptr) {
3534 __ Bind(slow_path->GetExitLabel());
3535 }
3536}
3537
3538void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) {
3539 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
3540 bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
3541
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003542 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
3543 switch (type_check_kind) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003544 case TypeCheckKind::kExactCheck:
3545 case TypeCheckKind::kAbstractClassCheck:
3546 case TypeCheckKind::kClassHierarchyCheck:
3547 case TypeCheckKind::kArrayObjectCheck:
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003548 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
3549 LocationSummary::kCallOnSlowPath :
3550 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003551 break;
3552 case TypeCheckKind::kArrayCheck:
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003553 case TypeCheckKind::kUnresolvedCheck:
3554 case TypeCheckKind::kInterfaceCheck:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003555 call_kind = LocationSummary::kCallOnSlowPath;
3556 break;
3557 }
3558
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003559 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3560 locations->SetInAt(0, Location::RequiresRegister());
3561 locations->SetInAt(1, Location::RequiresRegister());
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003562 // Add temps for read barriers and other uses. One is used by TypeCheckSlowPathARM64.
3563 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind));
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003564}
3565
3566void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) {
Roland Levillain44015862016-01-22 11:47:17 +00003567 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003568 LocationSummary* locations = instruction->GetLocations();
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003569 Location obj_loc = locations->InAt(0);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003570 Register obj = InputRegisterAt(instruction, 0);
3571 Register cls = InputRegisterAt(instruction, 1);
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003572 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind);
3573 DCHECK_GE(num_temps, 1u);
3574 DCHECK_LE(num_temps, 3u);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003575 Location temp_loc = locations->GetTemp(0);
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003576 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation();
3577 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation();
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003578 Register temp = WRegisterFrom(temp_loc);
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003579 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3580 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
3581 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
3582 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
3583 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value();
3584 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value();
3585 const uint32_t object_array_data_offset =
3586 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value();
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003587
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003588 bool is_type_check_slow_path_fatal =
3589 (type_check_kind == TypeCheckKind::kExactCheck ||
3590 type_check_kind == TypeCheckKind::kAbstractClassCheck ||
3591 type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
3592 type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
3593 !instruction->CanThrowIntoCatchBlock();
3594 SlowPathCodeARM64* type_check_slow_path =
3595 new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction,
3596 is_type_check_slow_path_fatal);
3597 codegen_->AddSlowPath(type_check_slow_path);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003598
Scott Wakeling97c72b72016-06-24 16:19:36 +01003599 vixl::aarch64::Label done;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003600 // Avoid null check if we know obj is not null.
3601 if (instruction->MustDoNullCheck()) {
Guillaume "Vermeille" Sanchezaf888352015-04-20 14:41:30 +01003602 __ Cbz(obj, &done);
3603 }
Alexandre Rames67555f72014-11-18 10:55:16 +00003604
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003605 switch (type_check_kind) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003606 case TypeCheckKind::kExactCheck:
3607 case TypeCheckKind::kArrayCheck: {
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003608 // /* HeapReference<Class> */ temp = obj->klass_
3609 GenerateReferenceLoadTwoRegisters(instruction,
3610 temp_loc,
3611 obj_loc,
3612 class_offset,
3613 maybe_temp2_loc,
3614 kEmitCompilerReadBarrier);
3615
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003616 __ Cmp(temp, cls);
3617 // Jump to slow path for throwing the exception or doing a
3618 // more involved array check.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003619 __ B(ne, type_check_slow_path->GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003620 break;
3621 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003622
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003623 case TypeCheckKind::kAbstractClassCheck: {
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003624 // /* HeapReference<Class> */ temp = obj->klass_
3625 GenerateReferenceLoadTwoRegisters(instruction,
3626 temp_loc,
3627 obj_loc,
3628 class_offset,
3629 maybe_temp2_loc,
3630 kEmitCompilerReadBarrier);
3631
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003632 // If the class is abstract, we eagerly fetch the super class of the
3633 // object to avoid doing a comparison we know will fail.
Mathieu Chartierb99f4d62016-11-07 16:17:26 -08003634 vixl::aarch64::Label loop;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003635 __ Bind(&loop);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003636 // /* HeapReference<Class> */ temp = temp->super_class_
Roland Levillain44015862016-01-22 11:47:17 +00003637 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003638
Mathieu Chartierb99f4d62016-11-07 16:17:26 -08003639 // If the class reference currently in `temp` is null, jump to the slow path to throw the
3640 // exception.
3641 __ Cbz(temp, type_check_slow_path->GetEntryLabel());
3642 // Otherwise, compare classes.
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003643 __ Cmp(temp, cls);
3644 __ B(ne, &loop);
3645 break;
3646 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003647
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003648 case TypeCheckKind::kClassHierarchyCheck: {
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003649 // /* HeapReference<Class> */ temp = obj->klass_
3650 GenerateReferenceLoadTwoRegisters(instruction,
3651 temp_loc,
3652 obj_loc,
3653 class_offset,
3654 maybe_temp2_loc,
3655 kEmitCompilerReadBarrier);
3656
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003657 // Walk over the class hierarchy to find a match.
Scott Wakeling97c72b72016-06-24 16:19:36 +01003658 vixl::aarch64::Label loop;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003659 __ Bind(&loop);
3660 __ Cmp(temp, cls);
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01003661 __ B(eq, &done);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003662
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003663 // /* HeapReference<Class> */ temp = temp->super_class_
Roland Levillain44015862016-01-22 11:47:17 +00003664 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003665
3666 // If the class reference currently in `temp` is not null, jump
3667 // back at the beginning of the loop.
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003668 __ Cbnz(temp, &loop);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003669 // Otherwise, jump to the slow path to throw the exception.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003670 __ B(type_check_slow_path->GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003671 break;
3672 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003673
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003674 case TypeCheckKind::kArrayObjectCheck: {
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003675 // /* HeapReference<Class> */ temp = obj->klass_
3676 GenerateReferenceLoadTwoRegisters(instruction,
3677 temp_loc,
3678 obj_loc,
3679 class_offset,
3680 maybe_temp2_loc,
3681 kEmitCompilerReadBarrier);
3682
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01003683 // Do an exact check.
3684 __ Cmp(temp, cls);
3685 __ B(eq, &done);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003686
3687 // Otherwise, we need to check that the object's class is a non-primitive array.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003688 // /* HeapReference<Class> */ temp = temp->component_type_
Roland Levillain44015862016-01-22 11:47:17 +00003689 GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, maybe_temp2_loc);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003690
Mathieu Chartierb99f4d62016-11-07 16:17:26 -08003691 // If the component type is null, jump to the slow path to throw the exception.
3692 __ Cbz(temp, type_check_slow_path->GetEntryLabel());
3693 // Otherwise, the object is indeed an array. Further check that this component type is not a
3694 // primitive type.
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003695 __ Ldrh(temp, HeapOperand(temp, primitive_offset));
3696 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
Mathieu Chartierb99f4d62016-11-07 16:17:26 -08003697 __ Cbnz(temp, type_check_slow_path->GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003698 break;
3699 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003700
Calin Juravle98893e12015-10-02 21:05:03 +01003701 case TypeCheckKind::kUnresolvedCheck:
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003702 // We always go into the type check slow path for the unresolved check cases.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003703 //
3704 // We cannot directly call the CheckCast runtime entry point
3705 // without resorting to a type checking slow path here (i.e. by
3706 // calling InvokeRuntime directly), as it would require to
3707 // assign fixed registers for the inputs of this HInstanceOf
3708 // instruction (following the runtime calling convention), which
3709 // might be cluttered by the potential first read barrier
3710 // emission at the beginning of this method.
3711 __ B(type_check_slow_path->GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003712 break;
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07003713 case TypeCheckKind::kInterfaceCheck: {
3714 // /* HeapReference<Class> */ temp = obj->klass_
3715 GenerateReferenceLoadTwoRegisters(instruction,
3716 temp_loc,
3717 obj_loc,
3718 class_offset,
3719 maybe_temp2_loc,
3720 /*emit_read_barrier*/ false);
3721
3722 // /* HeapReference<Class> */ temp = temp->iftable_
3723 GenerateReferenceLoadTwoRegisters(instruction,
3724 temp_loc,
3725 temp_loc,
3726 iftable_offset,
3727 maybe_temp2_loc,
3728 /*emit_read_barrier*/ false);
3729 vixl::aarch64::Label is_null;
3730 // Null iftable means it is empty and will always fail the check.
3731 __ Cbz(temp, &is_null);
3732
3733 // Loop through the iftable and check if any class matches.
3734 __ Ldr(WRegisterFrom(maybe_temp2_loc), HeapOperand(temp.W(), array_length_offset));
3735
3736 vixl::aarch64::Label start_loop;
3737 __ Bind(&start_loop);
3738 __ Ldr(WRegisterFrom(maybe_temp3_loc), HeapOperand(temp.W(), object_array_data_offset));
3739 GetAssembler()->MaybeUnpoisonHeapReference(WRegisterFrom(maybe_temp3_loc));
3740 __ Cmp(cls, WRegisterFrom(maybe_temp3_loc));
3741 __ B(eq, &done); // Return if same class.
3742 // Go to next interface.
3743 __ Add(temp, temp, 2 * kHeapReferenceSize);
3744 __ Sub(WRegisterFrom(maybe_temp2_loc), WRegisterFrom(maybe_temp2_loc), 2);
3745 __ Cbnz(WRegisterFrom(maybe_temp2_loc), &start_loop);
3746 __ Bind(&is_null);
3747
3748 __ B(type_check_slow_path->GetEntryLabel());
3749 break;
3750 }
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003751 }
Nicolas Geoffray75374372015-09-17 17:12:19 +00003752 __ Bind(&done);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00003753
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003754 __ Bind(type_check_slow_path->GetExitLabel());
Alexandre Rames67555f72014-11-18 10:55:16 +00003755}
3756
Alexandre Rames5319def2014-10-23 10:03:10 +01003757void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) {
3758 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
3759 locations->SetOut(Location::ConstantLocation(constant));
3760}
3761
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003762void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003763 // Will be generated at use site.
3764}
3765
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00003766void LocationsBuilderARM64::VisitNullConstant(HNullConstant* constant) {
3767 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
3768 locations->SetOut(Location::ConstantLocation(constant));
3769}
3770
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003771void InstructionCodeGeneratorARM64::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00003772 // Will be generated at use site.
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00003773}
3774
Calin Juravle175dc732015-08-25 15:42:32 +01003775void LocationsBuilderARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
3776 // The trampoline uses the same calling convention as dex calling conventions,
3777 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
3778 // the method_idx.
3779 HandleInvoke(invoke);
3780}
3781
3782void InstructionCodeGeneratorARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
3783 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
3784}
3785
Alexandre Rames5319def2014-10-23 10:03:10 +01003786void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) {
Roland Levillain2d27c8e2015-04-28 15:48:45 +01003787 InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +01003788 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
Alexandre Rames5319def2014-10-23 10:03:10 +01003789}
3790
Alexandre Rames67555f72014-11-18 10:55:16 +00003791void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
3792 HandleInvoke(invoke);
3793}
3794
3795void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
3796 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003797 LocationSummary* locations = invoke->GetLocations();
3798 Register temp = XRegisterFrom(locations->GetTemp(0));
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003799 Location receiver = locations->InAt(0);
Alexandre Rames67555f72014-11-18 10:55:16 +00003800 Offset class_offset = mirror::Object::ClassOffset();
Andreas Gampe542451c2016-07-26 09:02:02 -07003801 Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize);
Alexandre Rames67555f72014-11-18 10:55:16 +00003802
3803 // The register ip1 is required to be used for the hidden argument in
3804 // art_quick_imt_conflict_trampoline, so prevent VIXL from using it.
Alexandre Ramesd921d642015-04-16 15:07:16 +01003805 MacroAssembler* masm = GetVIXLAssembler();
3806 UseScratchRegisterScope scratch_scope(masm);
3807 BlockPoolsScope block_pools(masm);
Alexandre Rames67555f72014-11-18 10:55:16 +00003808 scratch_scope.Exclude(ip1);
3809 __ Mov(ip1, invoke->GetDexMethodIndex());
3810
Alexandre Rames67555f72014-11-18 10:55:16 +00003811 if (receiver.IsStackSlot()) {
Mathieu Chartiere401d142015-04-22 13:56:20 -07003812 __ Ldr(temp.W(), StackOperandFrom(receiver));
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003813 // /* HeapReference<Class> */ temp = temp->klass_
Mathieu Chartiere401d142015-04-22 13:56:20 -07003814 __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset));
Alexandre Rames67555f72014-11-18 10:55:16 +00003815 } else {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003816 // /* HeapReference<Class> */ temp = receiver->klass_
Mathieu Chartiere401d142015-04-22 13:56:20 -07003817 __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
Alexandre Rames67555f72014-11-18 10:55:16 +00003818 }
Calin Juravle77520bc2015-01-12 18:45:46 +00003819 codegen_->MaybeRecordImplicitNullCheck(invoke);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003820 // Instead of simply (possibly) unpoisoning `temp` here, we should
3821 // emit a read barrier for the previous class reference load.
3822 // However this is not required in practice, as this is an
3823 // intermediate/temporary reference and because the current
3824 // concurrent copying collector keeps the from-space memory
3825 // intact/accessible until the end of the marking phase (the
3826 // concurrent copying collector may not in the future).
Roland Levillain4d027112015-07-01 15:41:14 +01003827 GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
Artem Udovichenkoa62cb9b2016-06-30 09:18:25 +00003828 __ Ldr(temp,
3829 MemOperand(temp, mirror::Class::ImtPtrOffset(kArm64PointerSize).Uint32Value()));
3830 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement(
Matthew Gharrity465ecc82016-07-19 21:32:52 +00003831 invoke->GetImtIndex(), kArm64PointerSize));
Alexandre Rames67555f72014-11-18 10:55:16 +00003832 // temp = temp->GetImtEntryAt(method_offset);
Mathieu Chartiere401d142015-04-22 13:56:20 -07003833 __ Ldr(temp, MemOperand(temp, method_offset));
Alexandre Rames67555f72014-11-18 10:55:16 +00003834 // lr = temp->GetEntryPoint();
Mathieu Chartiere401d142015-04-22 13:56:20 -07003835 __ Ldr(lr, MemOperand(temp, entry_point.Int32Value()));
Alexandre Rames67555f72014-11-18 10:55:16 +00003836 // lr();
3837 __ Blr(lr);
3838 DCHECK(!codegen_->IsLeafMethod());
3839 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
3840}
3841
3842void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
Andreas Gampe878d58c2015-01-15 23:24:00 -08003843 IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
3844 if (intrinsic.TryDispatch(invoke)) {
3845 return;
3846 }
3847
Alexandre Rames67555f72014-11-18 10:55:16 +00003848 HandleInvoke(invoke);
3849}
3850
Nicolas Geoffraye53798a2014-12-01 10:31:54 +00003851void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
David Brazdil58282f42016-01-14 12:45:10 +00003852 // Explicit clinit checks triggered by static invokes must have been pruned by
3853 // art::PrepareForRegisterAllocation.
3854 DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
Roland Levillain4c0eb422015-04-24 16:43:49 +01003855
Andreas Gampe878d58c2015-01-15 23:24:00 -08003856 IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
3857 if (intrinsic.TryDispatch(invoke)) {
3858 return;
3859 }
3860
Alexandre Rames67555f72014-11-18 10:55:16 +00003861 HandleInvoke(invoke);
3862}
3863
Andreas Gampe878d58c2015-01-15 23:24:00 -08003864static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codegen) {
3865 if (invoke->GetLocations()->Intrinsified()) {
3866 IntrinsicCodeGeneratorARM64 intrinsic(codegen);
3867 intrinsic.Dispatch(invoke);
3868 return true;
3869 }
3870 return false;
3871}
3872
Vladimir Markodc151b22015-10-15 18:02:30 +01003873HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStaticOrDirectDispatch(
3874 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
Nicolas Geoffray5e4e11e2016-09-22 13:17:41 +01003875 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) {
Roland Levillain44015862016-01-22 11:47:17 +00003876 // On ARM64 we support all dispatch types.
Vladimir Markodc151b22015-10-15 18:02:30 +01003877 return desired_dispatch_info;
3878}
3879
Nicolas Geoffray38207af2015-06-01 15:46:22 +01003880void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
Vladimir Marko58155012015-08-19 12:49:41 +00003881 // For better instruction scheduling we load the direct code pointer before the method pointer.
3882 bool direct_code_loaded = false;
3883 switch (invoke->GetCodePtrLocation()) {
3884 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
3885 // LR = code address from literal pool with link-time patch.
3886 __ Ldr(lr, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod()));
3887 direct_code_loaded = true;
3888 break;
3889 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
3890 // LR = invoke->GetDirectCodePtr();
3891 __ Ldr(lr, DeduplicateUint64Literal(invoke->GetDirectCodePtr()));
3892 direct_code_loaded = true;
3893 break;
3894 default:
3895 break;
3896 }
3897
Andreas Gampe878d58c2015-01-15 23:24:00 -08003898 // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention.
Vladimir Marko58155012015-08-19 12:49:41 +00003899 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
3900 switch (invoke->GetMethodLoadKind()) {
Nicolas Geoffrayda079bb2016-09-26 17:56:07 +01003901 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: {
3902 uint32_t offset =
3903 GetThreadOffset<kArm64PointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
Vladimir Marko58155012015-08-19 12:49:41 +00003904 // temp = thread->string_init_entrypoint
Nicolas Geoffrayda079bb2016-09-26 17:56:07 +01003905 __ Ldr(XRegisterFrom(temp), MemOperand(tr, offset));
Vladimir Marko58155012015-08-19 12:49:41 +00003906 break;
Nicolas Geoffrayda079bb2016-09-26 17:56:07 +01003907 }
Vladimir Marko58155012015-08-19 12:49:41 +00003908 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
Vladimir Markoc53c0792015-11-19 15:48:33 +00003909 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Vladimir Marko58155012015-08-19 12:49:41 +00003910 break;
3911 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
3912 // Load method address from literal pool.
Alexandre Rames6dc01742015-11-12 14:44:19 +00003913 __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress()));
Vladimir Marko58155012015-08-19 12:49:41 +00003914 break;
3915 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
3916 // Load method address from literal pool with a link-time patch.
Alexandre Rames6dc01742015-11-12 14:44:19 +00003917 __ Ldr(XRegisterFrom(temp),
Vladimir Marko58155012015-08-19 12:49:41 +00003918 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod()));
3919 break;
3920 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
3921 // Add ADRP with its PC-relative DexCache access patch.
Nicolas Geoffray5e4e11e2016-09-22 13:17:41 +01003922 const DexFile& dex_file = invoke->GetDexFile();
Vladimir Markocac5a7e2016-02-22 10:39:50 +00003923 uint32_t element_offset = invoke->GetDexCacheArrayOffset();
Scott Wakeling97c72b72016-06-24 16:19:36 +01003924 vixl::aarch64::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset);
Vladimir Markoaad75c62016-10-03 08:46:48 +00003925 EmitAdrpPlaceholder(adrp_label, XRegisterFrom(temp));
Vladimir Marko58155012015-08-19 12:49:41 +00003926 // Add LDR with its PC-relative DexCache access patch.
Scott Wakeling97c72b72016-06-24 16:19:36 +01003927 vixl::aarch64::Label* ldr_label =
Vladimir Markocac5a7e2016-02-22 10:39:50 +00003928 NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label);
Vladimir Markoaad75c62016-10-03 08:46:48 +00003929 EmitLdrOffsetPlaceholder(ldr_label, XRegisterFrom(temp), XRegisterFrom(temp));
Vladimir Marko58155012015-08-19 12:49:41 +00003930 break;
Vladimir Marko9b688a02015-05-06 14:12:42 +01003931 }
Vladimir Marko58155012015-08-19 12:49:41 +00003932 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
Vladimir Markoc53c0792015-11-19 15:48:33 +00003933 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Vladimir Marko58155012015-08-19 12:49:41 +00003934 Register reg = XRegisterFrom(temp);
3935 Register method_reg;
3936 if (current_method.IsRegister()) {
3937 method_reg = XRegisterFrom(current_method);
3938 } else {
3939 DCHECK(invoke->GetLocations()->Intrinsified());
3940 DCHECK(!current_method.IsValid());
3941 method_reg = reg;
3942 __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset));
3943 }
Vladimir Markob2c431e2015-08-19 12:45:42 +00003944
Roland Levillain22ccc3a2015-11-24 13:10:05 +00003945 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
Vladimir Marko05792b92015-08-03 11:56:49 +01003946 __ Ldr(reg.X(),
3947 MemOperand(method_reg.X(),
Andreas Gampe542451c2016-07-26 09:02:02 -07003948 ArtMethod::DexCacheResolvedMethodsOffset(kArm64PointerSize).Int32Value()));
Vladimir Marko58155012015-08-19 12:49:41 +00003949 // temp = temp[index_in_cache];
Vladimir Marko40ecb122016-04-06 17:33:41 +01003950 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file.
3951 uint32_t index_in_cache = invoke->GetDexMethodIndex();
Vladimir Marko58155012015-08-19 12:49:41 +00003952 __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache)));
3953 break;
3954 }
3955 }
3956
3957 switch (invoke->GetCodePtrLocation()) {
3958 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
3959 __ Bl(&frame_entry_label_);
3960 break;
3961 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
Vladimir Markoaad75c62016-10-03 08:46:48 +00003962 relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
3963 invoke->GetTargetMethod().dex_method_index);
Scott Wakeling97c72b72016-06-24 16:19:36 +01003964 vixl::aarch64::Label* label = &relative_call_patches_.back().label;
3965 SingleEmissionCheckScope guard(GetVIXLAssembler());
Alexandre Rames6dc01742015-11-12 14:44:19 +00003966 __ Bind(label);
3967 __ bl(0); // Branch and link to itself. This will be overriden at link time.
Vladimir Marko58155012015-08-19 12:49:41 +00003968 break;
3969 }
3970 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
3971 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
3972 // LR prepared above for better instruction scheduling.
3973 DCHECK(direct_code_loaded);
3974 // lr()
3975 __ Blr(lr);
3976 break;
3977 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
3978 // LR = callee_method->entry_point_from_quick_compiled_code_;
3979 __ Ldr(lr, MemOperand(
Alexandre Rames6dc01742015-11-12 14:44:19 +00003980 XRegisterFrom(callee_method),
Andreas Gampe542451c2016-07-26 09:02:02 -07003981 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize).Int32Value()));
Vladimir Marko58155012015-08-19 12:49:41 +00003982 // lr()
3983 __ Blr(lr);
3984 break;
Nicolas Geoffray1cf95282014-12-12 19:22:03 +00003985 }
Alexandre Rames5319def2014-10-23 10:03:10 +01003986
Andreas Gampe878d58c2015-01-15 23:24:00 -08003987 DCHECK(!IsLeafMethod());
3988}
3989
Andreas Gampebfb5ba92015-09-01 15:45:02 +00003990void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
Nicolas Geoffraye5234232015-12-02 09:06:11 +00003991 // Use the calling convention instead of the location of the receiver, as
3992 // intrinsics may have put the receiver in a different register. In the intrinsics
3993 // slow path, the arguments have been moved to the right place, so here we are
3994 // guaranteed that the receiver is the first register of the calling convention.
3995 InvokeDexCallingConvention calling_convention;
3996 Register receiver = calling_convention.GetRegisterAt(0);
Andreas Gampebfb5ba92015-09-01 15:45:02 +00003997 Register temp = XRegisterFrom(temp_in);
3998 size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
3999 invoke->GetVTableIndex(), kArm64PointerSize).SizeValue();
4000 Offset class_offset = mirror::Object::ClassOffset();
Andreas Gampe542451c2016-07-26 09:02:02 -07004001 Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize);
Andreas Gampebfb5ba92015-09-01 15:45:02 +00004002
4003 BlockPoolsScope block_pools(GetVIXLAssembler());
4004
4005 DCHECK(receiver.IsRegister());
Roland Levillain22ccc3a2015-11-24 13:10:05 +00004006 // /* HeapReference<Class> */ temp = receiver->klass_
Nicolas Geoffraye5234232015-12-02 09:06:11 +00004007 __ Ldr(temp.W(), HeapOperandFrom(LocationFrom(receiver), class_offset));
Andreas Gampebfb5ba92015-09-01 15:45:02 +00004008 MaybeRecordImplicitNullCheck(invoke);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00004009 // Instead of simply (possibly) unpoisoning `temp` here, we should
4010 // emit a read barrier for the previous class reference load.
Roland Levillain22ccc3a2015-11-24 13:10:05 +00004011 // intermediate/temporary reference and because the current
4012 // concurrent copying collector keeps the from-space memory
4013 // intact/accessible until the end of the marking phase (the
4014 // concurrent copying collector may not in the future).
Andreas Gampebfb5ba92015-09-01 15:45:02 +00004015 GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
4016 // temp = temp->GetMethodAt(method_offset);
4017 __ Ldr(temp, MemOperand(temp, method_offset));
4018 // lr = temp->GetEntryPoint();
4019 __ Ldr(lr, MemOperand(temp, entry_point.SizeValue()));
4020 // lr();
4021 __ Blr(lr);
4022}
4023
Scott Wakeling97c72b72016-06-24 16:19:36 +01004024vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeStringPatch(
4025 const DexFile& dex_file,
4026 uint32_t string_index,
4027 vixl::aarch64::Label* adrp_label) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004028 return NewPcRelativePatch(dex_file, string_index, adrp_label, &pc_relative_string_patches_);
4029}
4030
Scott Wakeling97c72b72016-06-24 16:19:36 +01004031vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeTypePatch(
4032 const DexFile& dex_file,
4033 uint32_t type_index,
4034 vixl::aarch64::Label* adrp_label) {
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004035 return NewPcRelativePatch(dex_file, type_index, adrp_label, &pc_relative_type_patches_);
4036}
4037
Scott Wakeling97c72b72016-06-24 16:19:36 +01004038vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch(
4039 const DexFile& dex_file,
4040 uint32_t element_offset,
4041 vixl::aarch64::Label* adrp_label) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004042 return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_);
4043}
4044
Scott Wakeling97c72b72016-06-24 16:19:36 +01004045vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativePatch(
4046 const DexFile& dex_file,
4047 uint32_t offset_or_index,
4048 vixl::aarch64::Label* adrp_label,
4049 ArenaDeque<PcRelativePatchInfo>* patches) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004050 // Add a patch entry and return the label.
4051 patches->emplace_back(dex_file, offset_or_index);
4052 PcRelativePatchInfo* info = &patches->back();
Scott Wakeling97c72b72016-06-24 16:19:36 +01004053 vixl::aarch64::Label* label = &info->label;
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004054 // If adrp_label is null, this is the ADRP patch and needs to point to its own label.
4055 info->pc_insn_label = (adrp_label != nullptr) ? adrp_label : label;
4056 return label;
4057}
4058
Scott Wakeling97c72b72016-06-24 16:19:36 +01004059vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageStringLiteral(
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004060 const DexFile& dex_file, uint32_t string_index) {
4061 return boot_image_string_patches_.GetOrCreate(
4062 StringReference(&dex_file, string_index),
4063 [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); });
4064}
4065
Scott Wakeling97c72b72016-06-24 16:19:36 +01004066vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageTypeLiteral(
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004067 const DexFile& dex_file, uint32_t type_index) {
4068 return boot_image_type_patches_.GetOrCreate(
4069 TypeReference(&dex_file, type_index),
4070 [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); });
4071}
4072
Scott Wakeling97c72b72016-06-24 16:19:36 +01004073vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(
4074 uint64_t address) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004075 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
4076 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
4077 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
4078}
4079
Scott Wakeling97c72b72016-06-24 16:19:36 +01004080vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral(
4081 uint64_t address) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004082 return DeduplicateUint64Literal(address);
4083}
4084
Vladimir Markoaad75c62016-10-03 08:46:48 +00004085void CodeGeneratorARM64::EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label,
4086 vixl::aarch64::Register reg) {
4087 DCHECK(reg.IsX());
4088 SingleEmissionCheckScope guard(GetVIXLAssembler());
4089 __ Bind(fixup_label);
4090 __ adrp(reg, /* offset placeholder */ 0);
4091}
4092
4093void CodeGeneratorARM64::EmitAddPlaceholder(vixl::aarch64::Label* fixup_label,
4094 vixl::aarch64::Register out,
4095 vixl::aarch64::Register base) {
4096 DCHECK(out.IsX());
4097 DCHECK(base.IsX());
4098 SingleEmissionCheckScope guard(GetVIXLAssembler());
4099 __ Bind(fixup_label);
4100 __ add(out, base, Operand(/* offset placeholder */ 0));
4101}
4102
4103void CodeGeneratorARM64::EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label,
4104 vixl::aarch64::Register out,
4105 vixl::aarch64::Register base) {
4106 DCHECK(base.IsX());
4107 SingleEmissionCheckScope guard(GetVIXLAssembler());
4108 __ Bind(fixup_label);
4109 __ ldr(out, MemOperand(base, /* offset placeholder */ 0));
4110}
4111
4112template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
4113inline void CodeGeneratorARM64::EmitPcRelativeLinkerPatches(
4114 const ArenaDeque<PcRelativePatchInfo>& infos,
4115 ArenaVector<LinkerPatch>* linker_patches) {
4116 for (const PcRelativePatchInfo& info : infos) {
4117 linker_patches->push_back(Factory(info.label.GetLocation(),
4118 &info.target_dex_file,
4119 info.pc_insn_label->GetLocation(),
4120 info.offset_or_index));
4121 }
4122}
4123
Vladimir Marko58155012015-08-19 12:49:41 +00004124void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
4125 DCHECK(linker_patches->empty());
4126 size_t size =
4127 method_patches_.size() +
4128 call_patches_.size() +
4129 relative_call_patches_.size() +
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004130 pc_relative_dex_cache_patches_.size() +
4131 boot_image_string_patches_.size() +
4132 pc_relative_string_patches_.size() +
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004133 boot_image_type_patches_.size() +
4134 pc_relative_type_patches_.size() +
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004135 boot_image_address_patches_.size();
Vladimir Marko58155012015-08-19 12:49:41 +00004136 linker_patches->reserve(size);
4137 for (const auto& entry : method_patches_) {
4138 const MethodReference& target_method = entry.first;
Scott Wakeling97c72b72016-06-24 16:19:36 +01004139 vixl::aarch64::Literal<uint64_t>* literal = entry.second;
4140 linker_patches->push_back(LinkerPatch::MethodPatch(literal->GetOffset(),
Vladimir Marko58155012015-08-19 12:49:41 +00004141 target_method.dex_file,
4142 target_method.dex_method_index));
4143 }
4144 for (const auto& entry : call_patches_) {
4145 const MethodReference& target_method = entry.first;
Scott Wakeling97c72b72016-06-24 16:19:36 +01004146 vixl::aarch64::Literal<uint64_t>* literal = entry.second;
4147 linker_patches->push_back(LinkerPatch::CodePatch(literal->GetOffset(),
Vladimir Marko58155012015-08-19 12:49:41 +00004148 target_method.dex_file,
4149 target_method.dex_method_index));
4150 }
Vladimir Markoaad75c62016-10-03 08:46:48 +00004151 for (const PatchInfo<vixl::aarch64::Label>& info : relative_call_patches_) {
4152 linker_patches->push_back(
4153 LinkerPatch::RelativeCodePatch(info.label.GetLocation(), &info.dex_file, info.index));
Vladimir Marko58155012015-08-19 12:49:41 +00004154 }
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004155 for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01004156 linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.GetLocation(),
Vladimir Marko58155012015-08-19 12:49:41 +00004157 &info.target_dex_file,
Scott Wakeling97c72b72016-06-24 16:19:36 +01004158 info.pc_insn_label->GetLocation(),
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004159 info.offset_or_index));
4160 }
4161 for (const auto& entry : boot_image_string_patches_) {
4162 const StringReference& target_string = entry.first;
Scott Wakeling97c72b72016-06-24 16:19:36 +01004163 vixl::aarch64::Literal<uint32_t>* literal = entry.second;
4164 linker_patches->push_back(LinkerPatch::StringPatch(literal->GetOffset(),
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004165 target_string.dex_file,
4166 target_string.string_index));
4167 }
Vladimir Markoaad75c62016-10-03 08:46:48 +00004168 if (!GetCompilerOptions().IsBootImage()) {
4169 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_,
4170 linker_patches);
4171 } else {
4172 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_,
4173 linker_patches);
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004174 }
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004175 for (const auto& entry : boot_image_type_patches_) {
4176 const TypeReference& target_type = entry.first;
Scott Wakeling97c72b72016-06-24 16:19:36 +01004177 vixl::aarch64::Literal<uint32_t>* literal = entry.second;
4178 linker_patches->push_back(LinkerPatch::TypePatch(literal->GetOffset(),
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004179 target_type.dex_file,
4180 target_type.type_index));
4181 }
Vladimir Markoaad75c62016-10-03 08:46:48 +00004182 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_,
4183 linker_patches);
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004184 for (const auto& entry : boot_image_address_patches_) {
4185 DCHECK(GetCompilerOptions().GetIncludePatchInformation());
Scott Wakeling97c72b72016-06-24 16:19:36 +01004186 vixl::aarch64::Literal<uint32_t>* literal = entry.second;
4187 linker_patches->push_back(LinkerPatch::RecordPosition(literal->GetOffset()));
Vladimir Marko58155012015-08-19 12:49:41 +00004188 }
4189}
4190
Scott Wakeling97c72b72016-06-24 16:19:36 +01004191vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value,
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004192 Uint32ToLiteralMap* map) {
4193 return map->GetOrCreate(
4194 value,
4195 [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); });
4196}
4197
Scott Wakeling97c72b72016-06-24 16:19:36 +01004198vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004199 return uint64_literals_.GetOrCreate(
4200 value,
4201 [this, value]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(value); });
Vladimir Marko58155012015-08-19 12:49:41 +00004202}
4203
Scott Wakeling97c72b72016-06-24 16:19:36 +01004204vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral(
Vladimir Marko58155012015-08-19 12:49:41 +00004205 MethodReference target_method,
4206 MethodToLiteralMap* map) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004207 return map->GetOrCreate(
4208 target_method,
4209 [this]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(/* placeholder */ 0u); });
Vladimir Marko58155012015-08-19 12:49:41 +00004210}
4211
Scott Wakeling97c72b72016-06-24 16:19:36 +01004212vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral(
Vladimir Marko58155012015-08-19 12:49:41 +00004213 MethodReference target_method) {
4214 return DeduplicateMethodLiteral(target_method, &method_patches_);
4215}
4216
Scott Wakeling97c72b72016-06-24 16:19:36 +01004217vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodCodeLiteral(
Vladimir Marko58155012015-08-19 12:49:41 +00004218 MethodReference target_method) {
4219 return DeduplicateMethodLiteral(target_method, &call_patches_);
4220}
4221
4222
Andreas Gampe878d58c2015-01-15 23:24:00 -08004223void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
David Brazdil58282f42016-01-14 12:45:10 +00004224 // Explicit clinit checks triggered by static invokes must have been pruned by
4225 // art::PrepareForRegisterAllocation.
4226 DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
Roland Levillain4c0eb422015-04-24 16:43:49 +01004227
Andreas Gampe878d58c2015-01-15 23:24:00 -08004228 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
4229 return;
4230 }
4231
Alexandre Ramesd921d642015-04-16 15:07:16 +01004232 BlockPoolsScope block_pools(GetVIXLAssembler());
Nicolas Geoffray38207af2015-06-01 15:46:22 +01004233 LocationSummary* locations = invoke->GetLocations();
4234 codegen_->GenerateStaticOrDirectCall(
4235 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00004236 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
Alexandre Rames5319def2014-10-23 10:03:10 +01004237}
4238
4239void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
Andreas Gampe878d58c2015-01-15 23:24:00 -08004240 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
4241 return;
4242 }
4243
Andreas Gampebfb5ba92015-09-01 15:45:02 +00004244 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
Alexandre Rames5319def2014-10-23 10:03:10 +01004245 DCHECK(!codegen_->IsLeafMethod());
4246 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
4247}
4248
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004249HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind(
4250 HLoadClass::LoadKind desired_class_load_kind) {
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004251 switch (desired_class_load_kind) {
4252 case HLoadClass::LoadKind::kReferrersClass:
4253 break;
4254 case HLoadClass::LoadKind::kBootImageLinkTimeAddress:
4255 DCHECK(!GetCompilerOptions().GetCompilePic());
4256 break;
4257 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative:
4258 DCHECK(GetCompilerOptions().GetCompilePic());
4259 break;
4260 case HLoadClass::LoadKind::kBootImageAddress:
4261 break;
4262 case HLoadClass::LoadKind::kDexCacheAddress:
4263 DCHECK(Runtime::Current()->UseJitCompilation());
4264 break;
4265 case HLoadClass::LoadKind::kDexCachePcRelative:
4266 DCHECK(!Runtime::Current()->UseJitCompilation());
4267 break;
4268 case HLoadClass::LoadKind::kDexCacheViaMethod:
4269 break;
4270 }
4271 return desired_class_load_kind;
4272}
4273
Alexandre Rames67555f72014-11-18 10:55:16 +00004274void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) {
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004275 if (cls->NeedsAccessCheck()) {
4276 InvokeRuntimeCallingConvention calling_convention;
4277 CodeGenerator::CreateLoadClassLocationSummary(
4278 cls,
4279 LocationFrom(calling_convention.GetRegisterAt(0)),
Scott Wakeling97c72b72016-06-24 16:19:36 +01004280 LocationFrom(vixl::aarch64::x0),
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004281 /* code_generator_supports_read_barrier */ true);
4282 return;
4283 }
4284
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004285 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage();
4286 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier)
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004287 ? LocationSummary::kCallOnSlowPath
4288 : LocationSummary::kNoCall;
4289 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004290 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01004291 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
Vladimir Marko70e97462016-08-09 11:04:26 +01004292 }
4293
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004294 HLoadClass::LoadKind load_kind = cls->GetLoadKind();
4295 if (load_kind == HLoadClass::LoadKind::kReferrersClass ||
4296 load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) {
4297 locations->SetInAt(0, Location::RequiresRegister());
4298 }
4299 locations->SetOut(Location::RequiresRegister());
Alexandre Rames67555f72014-11-18 10:55:16 +00004300}
4301
4302void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) {
Calin Juravle98893e12015-10-02 21:05:03 +01004303 if (cls->NeedsAccessCheck()) {
4304 codegen_->MoveConstant(cls->GetLocations()->GetTemp(0), cls->GetTypeIndex());
Serban Constantinescu22f81d32016-02-18 16:06:31 +00004305 codegen_->InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc());
Roland Levillain888d0672015-11-23 18:53:50 +00004306 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
Calin Juravle580b6092015-10-06 17:35:58 +01004307 return;
4308 }
4309
Roland Levillain22ccc3a2015-11-24 13:10:05 +00004310 Location out_loc = cls->GetLocations()->Out();
Calin Juravle580b6092015-10-06 17:35:58 +01004311 Register out = OutputRegister(cls);
Alexandre Rames67555f72014-11-18 10:55:16 +00004312
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004313 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage();
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004314 bool generate_null_check = false;
4315 switch (cls->GetLoadKind()) {
4316 case HLoadClass::LoadKind::kReferrersClass: {
4317 DCHECK(!cls->CanCallRuntime());
4318 DCHECK(!cls->MustGenerateClinitCheck());
4319 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
4320 Register current_method = InputRegisterAt(cls, 0);
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004321 GenerateGcRootFieldLoad(cls,
4322 out_loc,
4323 current_method,
4324 ArtMethod::DeclaringClassOffset().Int32Value(),
Roland Levillain00468f32016-10-27 18:02:48 +01004325 /* fixup_label */ nullptr,
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004326 requires_read_barrier);
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004327 break;
4328 }
4329 case HLoadClass::LoadKind::kBootImageLinkTimeAddress:
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004330 DCHECK(!requires_read_barrier);
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004331 __ Ldr(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(),
4332 cls->GetTypeIndex()));
4333 break;
4334 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: {
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004335 DCHECK(!requires_read_barrier);
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004336 // Add ADRP with its PC-relative type patch.
4337 const DexFile& dex_file = cls->GetDexFile();
4338 uint32_t type_index = cls->GetTypeIndex();
Scott Wakeling97c72b72016-06-24 16:19:36 +01004339 vixl::aarch64::Label* adrp_label = codegen_->NewPcRelativeTypePatch(dex_file, type_index);
Vladimir Markoaad75c62016-10-03 08:46:48 +00004340 codegen_->EmitAdrpPlaceholder(adrp_label, out.X());
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004341 // Add ADD with its PC-relative type patch.
Scott Wakeling97c72b72016-06-24 16:19:36 +01004342 vixl::aarch64::Label* add_label =
4343 codegen_->NewPcRelativeTypePatch(dex_file, type_index, adrp_label);
Vladimir Markoaad75c62016-10-03 08:46:48 +00004344 codegen_->EmitAddPlaceholder(add_label, out.X(), out.X());
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004345 break;
4346 }
4347 case HLoadClass::LoadKind::kBootImageAddress: {
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004348 DCHECK(!requires_read_barrier);
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004349 DCHECK(cls->GetAddress() != 0u && IsUint<32>(cls->GetAddress()));
4350 __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(cls->GetAddress()));
4351 break;
4352 }
4353 case HLoadClass::LoadKind::kDexCacheAddress: {
4354 DCHECK_NE(cls->GetAddress(), 0u);
4355 // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads
4356 // that gives a 16KiB range. To try and reduce the number of literals if we load
4357 // multiple types, simply split the dex cache address to a 16KiB aligned base
4358 // loaded from a literal and the remaining offset embedded in the load.
4359 static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes.");
4360 DCHECK_ALIGNED(cls->GetAddress(), 4u);
4361 constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2;
4362 uint64_t base_address = cls->GetAddress() & ~MaxInt<uint64_t>(offset_bits);
4363 uint32_t offset = cls->GetAddress() & MaxInt<uint64_t>(offset_bits);
4364 __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address));
4365 // /* GcRoot<mirror::Class> */ out = *(base_address + offset)
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004366 GenerateGcRootFieldLoad(cls,
4367 out_loc,
4368 out.X(),
4369 offset,
Roland Levillain00468f32016-10-27 18:02:48 +01004370 /* fixup_label */ nullptr,
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004371 requires_read_barrier);
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004372 generate_null_check = !cls->IsInDexCache();
4373 break;
4374 }
4375 case HLoadClass::LoadKind::kDexCachePcRelative: {
4376 // Add ADRP with its PC-relative DexCache access patch.
4377 const DexFile& dex_file = cls->GetDexFile();
4378 uint32_t element_offset = cls->GetDexCacheElementOffset();
Scott Wakeling97c72b72016-06-24 16:19:36 +01004379 vixl::aarch64::Label* adrp_label =
4380 codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset);
Vladimir Markoaad75c62016-10-03 08:46:48 +00004381 codegen_->EmitAdrpPlaceholder(adrp_label, out.X());
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004382 // Add LDR with its PC-relative DexCache access patch.
Scott Wakeling97c72b72016-06-24 16:19:36 +01004383 vixl::aarch64::Label* ldr_label =
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004384 codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label);
4385 // /* GcRoot<mirror::Class> */ out = *(base_address + offset) /* PC-relative */
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004386 GenerateGcRootFieldLoad(cls,
4387 out_loc,
4388 out.X(),
4389 /* offset placeholder */ 0,
4390 ldr_label,
4391 requires_read_barrier);
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004392 generate_null_check = !cls->IsInDexCache();
4393 break;
4394 }
4395 case HLoadClass::LoadKind::kDexCacheViaMethod: {
4396 MemberOffset resolved_types_offset =
4397 ArtMethod::DexCacheResolvedTypesOffset(kArm64PointerSize);
4398 // /* GcRoot<mirror::Class>[] */ out =
4399 // current_method.ptr_sized_fields_->dex_cache_resolved_types_
4400 Register current_method = InputRegisterAt(cls, 0);
4401 __ Ldr(out.X(), MemOperand(current_method, resolved_types_offset.Int32Value()));
4402 // /* GcRoot<mirror::Class> */ out = out[type_index]
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004403 GenerateGcRootFieldLoad(cls,
4404 out_loc,
4405 out.X(),
4406 CodeGenerator::GetCacheOffset(cls->GetTypeIndex()),
Roland Levillain00468f32016-10-27 18:02:48 +01004407 /* fixup_label */ nullptr,
Mathieu Chartier31b12e32016-09-02 17:11:57 -07004408 requires_read_barrier);
Vladimir Markodbb7f5b2016-03-30 13:23:58 +01004409 generate_null_check = !cls->IsInDexCache();
4410 break;
4411 }
4412 }
4413
4414 if (generate_null_check || cls->MustGenerateClinitCheck()) {
4415 DCHECK(cls->CanCallRuntime());
4416 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
4417 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4418 codegen_->AddSlowPath(slow_path);
4419 if (generate_null_check) {
4420 __ Cbz(out, slow_path->GetEntryLabel());
4421 }
4422 if (cls->MustGenerateClinitCheck()) {
4423 GenerateClassInitializationCheck(slow_path, out);
4424 } else {
4425 __ Bind(slow_path->GetExitLabel());
Alexandre Rames67555f72014-11-18 10:55:16 +00004426 }
4427 }
4428}
4429
David Brazdilcb1c0552015-08-04 16:22:25 +01004430static MemOperand GetExceptionTlsAddress() {
Andreas Gampe542451c2016-07-26 09:02:02 -07004431 return MemOperand(tr, Thread::ExceptionOffset<kArm64PointerSize>().Int32Value());
David Brazdilcb1c0552015-08-04 16:22:25 +01004432}
4433
Alexandre Rames67555f72014-11-18 10:55:16 +00004434void LocationsBuilderARM64::VisitLoadException(HLoadException* load) {
4435 LocationSummary* locations =
4436 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4437 locations->SetOut(Location::RequiresRegister());
4438}
4439
4440void InstructionCodeGeneratorARM64::VisitLoadException(HLoadException* instruction) {
David Brazdilcb1c0552015-08-04 16:22:25 +01004441 __ Ldr(OutputRegister(instruction), GetExceptionTlsAddress());
4442}
4443
4444void LocationsBuilderARM64::VisitClearException(HClearException* clear) {
4445 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
4446}
4447
4448void InstructionCodeGeneratorARM64::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
4449 __ Str(wzr, GetExceptionTlsAddress());
Alexandre Rames67555f72014-11-18 10:55:16 +00004450}
4451
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004452HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind(
4453 HLoadString::LoadKind desired_string_load_kind) {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004454 switch (desired_string_load_kind) {
4455 case HLoadString::LoadKind::kBootImageLinkTimeAddress:
4456 DCHECK(!GetCompilerOptions().GetCompilePic());
4457 break;
4458 case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
4459 DCHECK(GetCompilerOptions().GetCompilePic());
4460 break;
4461 case HLoadString::LoadKind::kBootImageAddress:
4462 break;
Vladimir Markoaad75c62016-10-03 08:46:48 +00004463 case HLoadString::LoadKind::kBssEntry:
Calin Juravleffc87072016-04-20 14:22:09 +01004464 DCHECK(!Runtime::Current()->UseJitCompilation());
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004465 break;
4466 case HLoadString::LoadKind::kDexCacheViaMethod:
4467 break;
4468 }
4469 return desired_string_load_kind;
4470}
4471
Alexandre Rames67555f72014-11-18 10:55:16 +00004472void LocationsBuilderARM64::VisitLoadString(HLoadString* load) {
Christina Wadsworth1fe89ea2016-08-31 16:14:38 -07004473 LocationSummary::CallKind call_kind = load->NeedsEnvironment()
Vladimir Markoaad75c62016-10-03 08:46:48 +00004474 ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod)
4475 ? LocationSummary::kCallOnMainOnly
4476 : LocationSummary::kCallOnSlowPath)
Nicolas Geoffray917d0162015-11-24 18:25:35 +00004477 : LocationSummary::kNoCall;
4478 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004479 if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) {
Christina Wadsworth1fe89ea2016-08-31 16:14:38 -07004480 InvokeRuntimeCallingConvention calling_convention;
4481 locations->SetOut(calling_convention.GetReturnLocation(load->GetType()));
4482 } else {
4483 locations->SetOut(Location::RequiresRegister());
Vladimir Marko94ce9c22016-09-30 14:50:51 +01004484 if (load->GetLoadKind() == HLoadString::LoadKind::kBssEntry) {
4485 if (!kUseReadBarrier || kUseBakerReadBarrier) {
4486 // Rely on the pResolveString and/or marking to save everything, including temps.
4487 RegisterSet caller_saves = RegisterSet::Empty();
4488 InvokeRuntimeCallingConvention calling_convention;
4489 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode()));
4490 DCHECK_EQ(calling_convention.GetRegisterAt(0).GetCode(),
4491 RegisterFrom(calling_convention.GetReturnLocation(Primitive::kPrimNot),
4492 Primitive::kPrimNot).GetCode());
4493 locations->SetCustomSlowPathCallerSaves(caller_saves);
4494 } else {
4495 // For non-Baker read barrier we have a temp-clobbering call.
4496 }
4497 }
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004498 }
Alexandre Rames67555f72014-11-18 10:55:16 +00004499}
4500
4501void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) {
Alexandre Rames67555f72014-11-18 10:55:16 +00004502 Register out = OutputRegister(load);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00004503
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004504 switch (load->GetLoadKind()) {
4505 case HLoadString::LoadKind::kBootImageLinkTimeAddress:
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004506 __ Ldr(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(),
4507 load->GetStringIndex()));
4508 return; // No dex cache slow path.
4509 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004510 // Add ADRP with its PC-relative String patch.
4511 const DexFile& dex_file = load->GetDexFile();
4512 uint32_t string_index = load->GetStringIndex();
Vladimir Markoaad75c62016-10-03 08:46:48 +00004513 DCHECK(codegen_->GetCompilerOptions().IsBootImage());
Scott Wakeling97c72b72016-06-24 16:19:36 +01004514 vixl::aarch64::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index);
Vladimir Markoaad75c62016-10-03 08:46:48 +00004515 codegen_->EmitAdrpPlaceholder(adrp_label, out.X());
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004516 // Add ADD with its PC-relative String patch.
Scott Wakeling97c72b72016-06-24 16:19:36 +01004517 vixl::aarch64::Label* add_label =
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004518 codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label);
Vladimir Markoaad75c62016-10-03 08:46:48 +00004519 codegen_->EmitAddPlaceholder(add_label, out.X(), out.X());
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004520 return; // No dex cache slow path.
4521 }
4522 case HLoadString::LoadKind::kBootImageAddress: {
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004523 DCHECK(load->GetAddress() != 0u && IsUint<32>(load->GetAddress()));
4524 __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(load->GetAddress()));
4525 return; // No dex cache slow path.
4526 }
Vladimir Markoaad75c62016-10-03 08:46:48 +00004527 case HLoadString::LoadKind::kBssEntry: {
4528 // Add ADRP with its PC-relative String .bss entry patch.
4529 const DexFile& dex_file = load->GetDexFile();
4530 uint32_t string_index = load->GetStringIndex();
4531 DCHECK(!codegen_->GetCompilerOptions().IsBootImage());
Vladimir Marko94ce9c22016-09-30 14:50:51 +01004532 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
4533 Register temp = temps.AcquireX();
Vladimir Markoaad75c62016-10-03 08:46:48 +00004534 vixl::aarch64::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index);
Vladimir Marko94ce9c22016-09-30 14:50:51 +01004535 codegen_->EmitAdrpPlaceholder(adrp_label, temp);
Vladimir Markoaad75c62016-10-03 08:46:48 +00004536 // Add LDR with its PC-relative String patch.
4537 vixl::aarch64::Label* ldr_label =
4538 codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label);
Nicolas Geoffray07c919f2016-11-09 17:29:03 +00004539 // /* GcRoot<mirror::Class> */ out = *(base_address + offset) /* PC-relative */
Vladimir Markoaad75c62016-10-03 08:46:48 +00004540 GenerateGcRootFieldLoad(load,
Nicolas Geoffray07c919f2016-11-09 17:29:03 +00004541 load->GetLocations()->Out(),
Vladimir Marko94ce9c22016-09-30 14:50:51 +01004542 temp,
Roland Levillain00468f32016-10-27 18:02:48 +01004543 /* offset placeholder */ 0u,
4544 ldr_label,
4545 kEmitCompilerReadBarrier);
Vladimir Marko94ce9c22016-09-30 14:50:51 +01004546 SlowPathCodeARM64* slow_path =
4547 new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load, temp, adrp_label);
Vladimir Markoaad75c62016-10-03 08:46:48 +00004548 codegen_->AddSlowPath(slow_path);
4549 __ Cbz(out.X(), slow_path->GetEntryLabel());
4550 __ Bind(slow_path->GetExitLabel());
4551 return;
4552 }
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004553 default:
Christina Wadsworthbf44e0e2016-08-18 10:37:42 -07004554 break;
Vladimir Markocac5a7e2016-02-22 10:39:50 +00004555 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +00004556
Christina Wadsworthbf44e0e2016-08-18 10:37:42 -07004557 // TODO: Re-add the compiler code to do string dex cache lookup again.
Christina Wadsworth1fe89ea2016-08-31 16:14:38 -07004558 InvokeRuntimeCallingConvention calling_convention;
Vladimir Marko94ce9c22016-09-30 14:50:51 +01004559 DCHECK_EQ(calling_convention.GetRegisterAt(0).GetCode(), out.GetCode());
Christina Wadsworth1fe89ea2016-08-31 16:14:38 -07004560 __ Mov(calling_convention.GetRegisterAt(0).W(), load->GetStringIndex());
4561 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc());
4562 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
Alexandre Rames67555f72014-11-18 10:55:16 +00004563}
4564
Alexandre Rames5319def2014-10-23 10:03:10 +01004565void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) {
4566 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
4567 locations->SetOut(Location::ConstantLocation(constant));
4568}
4569
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01004570void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004571 // Will be generated at use site.
4572}
4573
Alexandre Rames67555f72014-11-18 10:55:16 +00004574void LocationsBuilderARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
4575 LocationSummary* locations =
Serban Constantinescu54ff4822016-07-07 18:03:19 +01004576 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly);
Alexandre Rames67555f72014-11-18 10:55:16 +00004577 InvokeRuntimeCallingConvention calling_convention;
4578 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
4579}
4580
4581void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00004582 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject: kQuickUnlockObject,
4583 instruction,
4584 instruction->GetDexPc());
Roland Levillain888d0672015-11-23 18:53:50 +00004585 if (instruction->IsEnter()) {
4586 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
4587 } else {
4588 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
4589 }
Alexandre Rames67555f72014-11-18 10:55:16 +00004590}
4591
Alexandre Rames42d641b2014-10-27 14:00:51 +00004592void LocationsBuilderARM64::VisitMul(HMul* mul) {
4593 LocationSummary* locations =
4594 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
4595 switch (mul->GetResultType()) {
4596 case Primitive::kPrimInt:
4597 case Primitive::kPrimLong:
4598 locations->SetInAt(0, Location::RequiresRegister());
4599 locations->SetInAt(1, Location::RequiresRegister());
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00004600 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames42d641b2014-10-27 14:00:51 +00004601 break;
4602
4603 case Primitive::kPrimFloat:
4604 case Primitive::kPrimDouble:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00004605 locations->SetInAt(0, Location::RequiresFpuRegister());
4606 locations->SetInAt(1, Location::RequiresFpuRegister());
Alexandre Rames67555f72014-11-18 10:55:16 +00004607 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
Alexandre Rames42d641b2014-10-27 14:00:51 +00004608 break;
4609
4610 default:
4611 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
4612 }
4613}
4614
4615void InstructionCodeGeneratorARM64::VisitMul(HMul* mul) {
4616 switch (mul->GetResultType()) {
4617 case Primitive::kPrimInt:
4618 case Primitive::kPrimLong:
4619 __ Mul(OutputRegister(mul), InputRegisterAt(mul, 0), InputRegisterAt(mul, 1));
4620 break;
4621
4622 case Primitive::kPrimFloat:
4623 case Primitive::kPrimDouble:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00004624 __ Fmul(OutputFPRegister(mul), InputFPRegisterAt(mul, 0), InputFPRegisterAt(mul, 1));
Alexandre Rames42d641b2014-10-27 14:00:51 +00004625 break;
4626
4627 default:
4628 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
4629 }
4630}
4631
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004632void LocationsBuilderARM64::VisitNeg(HNeg* neg) {
4633 LocationSummary* locations =
4634 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
4635 switch (neg->GetResultType()) {
4636 case Primitive::kPrimInt:
Alexandre Rames67555f72014-11-18 10:55:16 +00004637 case Primitive::kPrimLong:
Serban Constantinescu2d35d9d2015-02-22 22:08:01 +00004638 locations->SetInAt(0, ARM64EncodableConstantOrRegister(neg->InputAt(0), neg));
Alexandre Rames67555f72014-11-18 10:55:16 +00004639 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004640 break;
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004641
4642 case Primitive::kPrimFloat:
4643 case Primitive::kPrimDouble:
Alexandre Rames67555f72014-11-18 10:55:16 +00004644 locations->SetInAt(0, Location::RequiresFpuRegister());
4645 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004646 break;
4647
4648 default:
4649 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
4650 }
4651}
4652
4653void InstructionCodeGeneratorARM64::VisitNeg(HNeg* neg) {
4654 switch (neg->GetResultType()) {
4655 case Primitive::kPrimInt:
4656 case Primitive::kPrimLong:
4657 __ Neg(OutputRegister(neg), InputOperandAt(neg, 0));
4658 break;
4659
4660 case Primitive::kPrimFloat:
4661 case Primitive::kPrimDouble:
Alexandre Rames67555f72014-11-18 10:55:16 +00004662 __ Fneg(OutputFPRegister(neg), InputFPRegisterAt(neg, 0));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004663 break;
4664
4665 default:
4666 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
4667 }
4668}
4669
4670void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) {
4671 LocationSummary* locations =
Serban Constantinescu54ff4822016-07-07 18:03:19 +01004672 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004673 InvokeRuntimeCallingConvention calling_convention;
4674 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004675 locations->SetOut(LocationFrom(x0));
Andreas Gampe1cc7dba2014-12-17 18:43:01 -08004676 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
Nicolas Geoffray69aa6012015-06-09 10:34:25 +01004677 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(2)));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004678}
4679
4680void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) {
4681 LocationSummary* locations = instruction->GetLocations();
4682 InvokeRuntimeCallingConvention calling_convention;
4683 Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
4684 DCHECK(type_index.Is(w0));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004685 __ Mov(type_index, instruction->GetTypeIndex());
Roland Levillain4d027112015-07-01 15:41:14 +01004686 // Note: if heap poisoning is enabled, the entry point takes cares
4687 // of poisoning the reference.
Serban Constantinescu22f81d32016-02-18 16:06:31 +00004688 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc());
Mathieu Chartiere401d142015-04-22 13:56:20 -07004689 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
Alexandre Ramesfc19de82014-11-07 17:13:31 +00004690}
4691
Alexandre Rames5319def2014-10-23 10:03:10 +01004692void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
4693 LocationSummary* locations =
Serban Constantinescu54ff4822016-07-07 18:03:19 +01004694 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly);
Alexandre Rames5319def2014-10-23 10:03:10 +01004695 InvokeRuntimeCallingConvention calling_convention;
David Brazdil6de19382016-01-08 17:37:10 +00004696 if (instruction->IsStringAlloc()) {
4697 locations->AddTemp(LocationFrom(kArtMethodRegister));
4698 } else {
4699 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
4700 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
4701 }
Alexandre Rames5319def2014-10-23 10:03:10 +01004702 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
4703}
4704
4705void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) {
Roland Levillain4d027112015-07-01 15:41:14 +01004706 // Note: if heap poisoning is enabled, the entry point takes cares
4707 // of poisoning the reference.
David Brazdil6de19382016-01-08 17:37:10 +00004708 if (instruction->IsStringAlloc()) {
4709 // String is allocated through StringFactory. Call NewEmptyString entry point.
4710 Location temp = instruction->GetLocations()->GetTemp(0);
Andreas Gampe542451c2016-07-26 09:02:02 -07004711 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize);
David Brazdil6de19382016-01-08 17:37:10 +00004712 __ Ldr(XRegisterFrom(temp), MemOperand(tr, QUICK_ENTRY_POINT(pNewEmptyString)));
4713 __ Ldr(lr, MemOperand(XRegisterFrom(temp), code_offset.Int32Value()));
4714 __ Blr(lr);
4715 codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4716 } else {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00004717 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc());
David Brazdil6de19382016-01-08 17:37:10 +00004718 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
4719 }
Alexandre Rames5319def2014-10-23 10:03:10 +01004720}
4721
4722void LocationsBuilderARM64::VisitNot(HNot* instruction) {
4723 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
Alexandre Rames4e596512014-11-07 15:56:50 +00004724 locations->SetInAt(0, Location::RequiresRegister());
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00004725 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01004726}
4727
4728void InstructionCodeGeneratorARM64::VisitNot(HNot* instruction) {
Nicolas Geoffrayd8ef2e92015-02-24 16:02:06 +00004729 switch (instruction->GetResultType()) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004730 case Primitive::kPrimInt:
Alexandre Rames5319def2014-10-23 10:03:10 +01004731 case Primitive::kPrimLong:
Roland Levillain55dcfb52014-10-24 18:09:09 +01004732 __ Mvn(OutputRegister(instruction), InputOperandAt(instruction, 0));
Alexandre Rames5319def2014-10-23 10:03:10 +01004733 break;
4734
4735 default:
4736 LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType();
4737 }
4738}
4739
David Brazdil66d126e2015-04-03 16:02:44 +01004740void LocationsBuilderARM64::VisitBooleanNot(HBooleanNot* instruction) {
4741 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
4742 locations->SetInAt(0, Location::RequiresRegister());
4743 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4744}
4745
4746void InstructionCodeGeneratorARM64::VisitBooleanNot(HBooleanNot* instruction) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01004747 __ Eor(OutputRegister(instruction), InputRegisterAt(instruction, 0), vixl::aarch64::Operand(1));
David Brazdil66d126e2015-04-03 16:02:44 +01004748}
4749
Alexandre Rames5319def2014-10-23 10:03:10 +01004750void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01004751 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction);
4752 locations->SetInAt(0, Location::RequiresRegister());
Alexandre Rames5319def2014-10-23 10:03:10 +01004753}
4754
Calin Juravle2ae48182016-03-16 14:05:09 +00004755void CodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) {
4756 if (CanMoveNullCheckToUser(instruction)) {
Calin Juravle77520bc2015-01-12 18:45:46 +00004757 return;
4758 }
Calin Juravlecd6dffe2015-01-08 17:35:35 +00004759
Alexandre Ramesd921d642015-04-16 15:07:16 +01004760 BlockPoolsScope block_pools(GetVIXLAssembler());
4761 Location obj = instruction->GetLocations()->InAt(0);
Calin Juravlecd6dffe2015-01-08 17:35:35 +00004762 __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
Calin Juravle2ae48182016-03-16 14:05:09 +00004763 RecordPcInfo(instruction, instruction->GetDexPc());
Calin Juravlecd6dffe2015-01-08 17:35:35 +00004764}
4765
Calin Juravle2ae48182016-03-16 14:05:09 +00004766void CodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004767 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction);
Calin Juravle2ae48182016-03-16 14:05:09 +00004768 AddSlowPath(slow_path);
Alexandre Rames5319def2014-10-23 10:03:10 +01004769
4770 LocationSummary* locations = instruction->GetLocations();
4771 Location obj = locations->InAt(0);
Calin Juravle77520bc2015-01-12 18:45:46 +00004772
4773 __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel());
Alexandre Rames5319def2014-10-23 10:03:10 +01004774}
4775
Calin Juravlecd6dffe2015-01-08 17:35:35 +00004776void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) {
Calin Juravle2ae48182016-03-16 14:05:09 +00004777 codegen_->GenerateNullCheck(instruction);
Calin Juravlecd6dffe2015-01-08 17:35:35 +00004778}
4779
Alexandre Rames67555f72014-11-18 10:55:16 +00004780void LocationsBuilderARM64::VisitOr(HOr* instruction) {
4781 HandleBinaryOp(instruction);
4782}
4783
4784void InstructionCodeGeneratorARM64::VisitOr(HOr* instruction) {
4785 HandleBinaryOp(instruction);
4786}
4787
Alexandre Rames3e69f162014-12-10 10:36:50 +00004788void LocationsBuilderARM64::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
4789 LOG(FATAL) << "Unreachable";
4790}
4791
4792void InstructionCodeGeneratorARM64::VisitParallelMove(HParallelMove* instruction) {
4793 codegen_->GetMoveResolver()->EmitNativeCode(instruction);
4794}
4795
Alexandre Rames5319def2014-10-23 10:03:10 +01004796void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) {
4797 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
4798 Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
4799 if (location.IsStackSlot()) {
4800 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4801 } else if (location.IsDoubleStackSlot()) {
4802 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4803 }
4804 locations->SetOut(location);
4805}
4806
Nicolas Geoffray76b1e172015-05-27 17:18:33 +01004807void InstructionCodeGeneratorARM64::VisitParameterValue(
4808 HParameterValue* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004809 // Nothing to do, the parameter is already at its location.
Nicolas Geoffray76b1e172015-05-27 17:18:33 +01004810}
4811
4812void LocationsBuilderARM64::VisitCurrentMethod(HCurrentMethod* instruction) {
4813 LocationSummary* locations =
4814 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
Nicolas Geoffray38207af2015-06-01 15:46:22 +01004815 locations->SetOut(LocationFrom(kArtMethodRegister));
Nicolas Geoffray76b1e172015-05-27 17:18:33 +01004816}
4817
4818void InstructionCodeGeneratorARM64::VisitCurrentMethod(
4819 HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
4820 // Nothing to do, the method is already at its location.
Alexandre Rames5319def2014-10-23 10:03:10 +01004821}
4822
4823void LocationsBuilderARM64::VisitPhi(HPhi* instruction) {
4824 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
Vladimir Marko372f10e2016-05-17 16:30:10 +01004825 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004826 locations->SetInAt(i, Location::Any());
4827 }
4828 locations->SetOut(Location::Any());
4829}
4830
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01004831void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004832 LOG(FATAL) << "Unreachable";
4833}
4834
Serban Constantinescu02164b32014-11-13 14:05:07 +00004835void LocationsBuilderARM64::VisitRem(HRem* rem) {
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00004836 Primitive::Type type = rem->GetResultType();
Alexandre Rames542361f2015-01-29 16:57:31 +00004837 LocationSummary::CallKind call_kind =
Serban Constantinescu54ff4822016-07-07 18:03:19 +01004838 Primitive::IsFloatingPointType(type) ? LocationSummary::kCallOnMainOnly
4839 : LocationSummary::kNoCall;
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00004840 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
4841
4842 switch (type) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00004843 case Primitive::kPrimInt:
4844 case Primitive::kPrimLong:
4845 locations->SetInAt(0, Location::RequiresRegister());
Zheng Xuc6667102015-05-15 16:08:45 +08004846 locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
Serban Constantinescu02164b32014-11-13 14:05:07 +00004847 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4848 break;
4849
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00004850 case Primitive::kPrimFloat:
4851 case Primitive::kPrimDouble: {
4852 InvokeRuntimeCallingConvention calling_convention;
4853 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
4854 locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
4855 locations->SetOut(calling_convention.GetReturnLocation(type));
4856
4857 break;
4858 }
4859
Serban Constantinescu02164b32014-11-13 14:05:07 +00004860 default:
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00004861 LOG(FATAL) << "Unexpected rem type " << type;
Serban Constantinescu02164b32014-11-13 14:05:07 +00004862 }
4863}
4864
4865void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) {
4866 Primitive::Type type = rem->GetResultType();
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00004867
Serban Constantinescu02164b32014-11-13 14:05:07 +00004868 switch (type) {
4869 case Primitive::kPrimInt:
4870 case Primitive::kPrimLong: {
Zheng Xuc6667102015-05-15 16:08:45 +08004871 GenerateDivRemIntegral(rem);
Serban Constantinescu02164b32014-11-13 14:05:07 +00004872 break;
4873 }
4874
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00004875 case Primitive::kPrimFloat:
4876 case Primitive::kPrimDouble: {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00004877 QuickEntrypointEnum entrypoint = (type == Primitive::kPrimFloat) ? kQuickFmodf : kQuickFmod;
4878 codegen_->InvokeRuntime(entrypoint, rem, rem->GetDexPc());
Roland Levillain888d0672015-11-23 18:53:50 +00004879 if (type == Primitive::kPrimFloat) {
4880 CheckEntrypointTypes<kQuickFmodf, float, float, float>();
4881 } else {
4882 CheckEntrypointTypes<kQuickFmod, double, double, double>();
4883 }
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00004884 break;
4885 }
4886
Serban Constantinescu02164b32014-11-13 14:05:07 +00004887 default:
4888 LOG(FATAL) << "Unexpected rem type " << type;
Vladimir Marko351dddf2015-12-11 16:34:46 +00004889 UNREACHABLE();
Serban Constantinescu02164b32014-11-13 14:05:07 +00004890 }
4891}
4892
Calin Juravle27df7582015-04-17 19:12:31 +01004893void LocationsBuilderARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
4894 memory_barrier->SetLocations(nullptr);
4895}
4896
4897void InstructionCodeGeneratorARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
Roland Levillain44015862016-01-22 11:47:17 +00004898 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
Calin Juravle27df7582015-04-17 19:12:31 +01004899}
4900
Alexandre Rames5319def2014-10-23 10:03:10 +01004901void LocationsBuilderARM64::VisitReturn(HReturn* instruction) {
4902 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
4903 Primitive::Type return_type = instruction->InputAt(0)->GetType();
Alexandre Ramesa89086e2014-11-07 17:13:25 +00004904 locations->SetInAt(0, ARM64ReturnLocation(return_type));
Alexandre Rames5319def2014-10-23 10:03:10 +01004905}
4906
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01004907void InstructionCodeGeneratorARM64::VisitReturn(HReturn* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004908 codegen_->GenerateFrameExit();
Alexandre Rames5319def2014-10-23 10:03:10 +01004909}
4910
4911void LocationsBuilderARM64::VisitReturnVoid(HReturnVoid* instruction) {
4912 instruction->SetLocations(nullptr);
4913}
4914
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01004915void InstructionCodeGeneratorARM64::VisitReturnVoid(HReturnVoid* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01004916 codegen_->GenerateFrameExit();
Alexandre Rames5319def2014-10-23 10:03:10 +01004917}
4918
Scott Wakeling40a04bf2015-12-11 09:50:36 +00004919void LocationsBuilderARM64::VisitRor(HRor* ror) {
4920 HandleBinaryOp(ror);
4921}
4922
4923void InstructionCodeGeneratorARM64::VisitRor(HRor* ror) {
4924 HandleBinaryOp(ror);
4925}
4926
Serban Constantinescu02164b32014-11-13 14:05:07 +00004927void LocationsBuilderARM64::VisitShl(HShl* shl) {
4928 HandleShift(shl);
4929}
4930
4931void InstructionCodeGeneratorARM64::VisitShl(HShl* shl) {
4932 HandleShift(shl);
4933}
4934
4935void LocationsBuilderARM64::VisitShr(HShr* shr) {
4936 HandleShift(shr);
4937}
4938
4939void InstructionCodeGeneratorARM64::VisitShr(HShr* shr) {
4940 HandleShift(shr);
4941}
4942
Alexandre Rames5319def2014-10-23 10:03:10 +01004943void LocationsBuilderARM64::VisitSub(HSub* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00004944 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01004945}
4946
4947void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00004948 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01004949}
4950
Alexandre Rames67555f72014-11-18 10:55:16 +00004951void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01004952 HandleFieldGet(instruction);
Alexandre Rames67555f72014-11-18 10:55:16 +00004953}
4954
4955void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01004956 HandleFieldGet(instruction, instruction->GetFieldInfo());
Alexandre Rames67555f72014-11-18 10:55:16 +00004957}
4958
4959void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01004960 HandleFieldSet(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01004961}
4962
Alexandre Rames67555f72014-11-18 10:55:16 +00004963void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01004964 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
Alexandre Rames5319def2014-10-23 10:03:10 +01004965}
4966
Calin Juravlee460d1d2015-09-29 04:52:17 +01004967void LocationsBuilderARM64::VisitUnresolvedInstanceFieldGet(
4968 HUnresolvedInstanceFieldGet* instruction) {
4969 FieldAccessCallingConventionARM64 calling_convention;
4970 codegen_->CreateUnresolvedFieldLocationSummary(
4971 instruction, instruction->GetFieldType(), calling_convention);
4972}
4973
4974void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldGet(
4975 HUnresolvedInstanceFieldGet* instruction) {
4976 FieldAccessCallingConventionARM64 calling_convention;
4977 codegen_->GenerateUnresolvedFieldAccess(instruction,
4978 instruction->GetFieldType(),
4979 instruction->GetFieldIndex(),
4980 instruction->GetDexPc(),
4981 calling_convention);
4982}
4983
4984void LocationsBuilderARM64::VisitUnresolvedInstanceFieldSet(
4985 HUnresolvedInstanceFieldSet* instruction) {
4986 FieldAccessCallingConventionARM64 calling_convention;
4987 codegen_->CreateUnresolvedFieldLocationSummary(
4988 instruction, instruction->GetFieldType(), calling_convention);
4989}
4990
4991void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldSet(
4992 HUnresolvedInstanceFieldSet* instruction) {
4993 FieldAccessCallingConventionARM64 calling_convention;
4994 codegen_->GenerateUnresolvedFieldAccess(instruction,
4995 instruction->GetFieldType(),
4996 instruction->GetFieldIndex(),
4997 instruction->GetDexPc(),
4998 calling_convention);
4999}
5000
5001void LocationsBuilderARM64::VisitUnresolvedStaticFieldGet(
5002 HUnresolvedStaticFieldGet* instruction) {
5003 FieldAccessCallingConventionARM64 calling_convention;
5004 codegen_->CreateUnresolvedFieldLocationSummary(
5005 instruction, instruction->GetFieldType(), calling_convention);
5006}
5007
5008void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldGet(
5009 HUnresolvedStaticFieldGet* instruction) {
5010 FieldAccessCallingConventionARM64 calling_convention;
5011 codegen_->GenerateUnresolvedFieldAccess(instruction,
5012 instruction->GetFieldType(),
5013 instruction->GetFieldIndex(),
5014 instruction->GetDexPc(),
5015 calling_convention);
5016}
5017
5018void LocationsBuilderARM64::VisitUnresolvedStaticFieldSet(
5019 HUnresolvedStaticFieldSet* instruction) {
5020 FieldAccessCallingConventionARM64 calling_convention;
5021 codegen_->CreateUnresolvedFieldLocationSummary(
5022 instruction, instruction->GetFieldType(), calling_convention);
5023}
5024
5025void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldSet(
5026 HUnresolvedStaticFieldSet* instruction) {
5027 FieldAccessCallingConventionARM64 calling_convention;
5028 codegen_->GenerateUnresolvedFieldAccess(instruction,
5029 instruction->GetFieldType(),
5030 instruction->GetFieldIndex(),
5031 instruction->GetDexPc(),
5032 calling_convention);
5033}
5034
Alexandre Rames5319def2014-10-23 10:03:10 +01005035void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
Vladimir Marko70e97462016-08-09 11:04:26 +01005036 LocationSummary* locations =
5037 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
Vladimir Marko804b03f2016-09-14 16:26:36 +01005038 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
Alexandre Rames5319def2014-10-23 10:03:10 +01005039}
5040
5041void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00005042 HBasicBlock* block = instruction->GetBlock();
5043 if (block->GetLoopInformation() != nullptr) {
5044 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
5045 // The back edge will generate the suspend check.
5046 return;
5047 }
5048 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
5049 // The goto will generate the suspend check.
5050 return;
5051 }
5052 GenerateSuspendCheck(instruction, nullptr);
Alexandre Rames5319def2014-10-23 10:03:10 +01005053}
5054
Alexandre Rames67555f72014-11-18 10:55:16 +00005055void LocationsBuilderARM64::VisitThrow(HThrow* instruction) {
5056 LocationSummary* locations =
Serban Constantinescu54ff4822016-07-07 18:03:19 +01005057 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly);
Alexandre Rames67555f72014-11-18 10:55:16 +00005058 InvokeRuntimeCallingConvention calling_convention;
5059 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
5060}
5061
5062void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00005063 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc());
Andreas Gampe1cc7dba2014-12-17 18:43:01 -08005064 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
Alexandre Rames67555f72014-11-18 10:55:16 +00005065}
5066
5067void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) {
5068 LocationSummary* locations =
5069 new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
5070 Primitive::Type input_type = conversion->GetInputType();
5071 Primitive::Type result_type = conversion->GetResultType();
Nicolas Geoffray01fcc9e2014-12-01 14:16:20 +00005072 DCHECK_NE(input_type, result_type);
Alexandre Rames67555f72014-11-18 10:55:16 +00005073 if ((input_type == Primitive::kPrimNot) || (input_type == Primitive::kPrimVoid) ||
5074 (result_type == Primitive::kPrimNot) || (result_type == Primitive::kPrimVoid)) {
5075 LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type;
5076 }
5077
Alexandre Rames542361f2015-01-29 16:57:31 +00005078 if (Primitive::IsFloatingPointType(input_type)) {
Alexandre Rames67555f72014-11-18 10:55:16 +00005079 locations->SetInAt(0, Location::RequiresFpuRegister());
5080 } else {
5081 locations->SetInAt(0, Location::RequiresRegister());
5082 }
5083
Alexandre Rames542361f2015-01-29 16:57:31 +00005084 if (Primitive::IsFloatingPointType(result_type)) {
Alexandre Rames67555f72014-11-18 10:55:16 +00005085 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
5086 } else {
5087 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5088 }
5089}
5090
5091void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* conversion) {
5092 Primitive::Type result_type = conversion->GetResultType();
5093 Primitive::Type input_type = conversion->GetInputType();
5094
5095 DCHECK_NE(input_type, result_type);
5096
Alexandre Rames542361f2015-01-29 16:57:31 +00005097 if (Primitive::IsIntegralType(result_type) && Primitive::IsIntegralType(input_type)) {
Alexandre Rames67555f72014-11-18 10:55:16 +00005098 int result_size = Primitive::ComponentSize(result_type);
5099 int input_size = Primitive::ComponentSize(input_type);
Alexandre Rames3e69f162014-12-10 10:36:50 +00005100 int min_size = std::min(result_size, input_size);
Serban Constantinescu02164b32014-11-13 14:05:07 +00005101 Register output = OutputRegister(conversion);
5102 Register source = InputRegisterAt(conversion, 0);
Alexandre Rames8626b742015-11-25 16:28:08 +00005103 if (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimLong) {
Alexandre Rames4dff2fd2015-08-20 13:36:35 +01005104 // 'int' values are used directly as W registers, discarding the top
5105 // bits, so we don't need to sign-extend and can just perform a move.
5106 // We do not pass the `kDiscardForSameWReg` argument to force clearing the
5107 // top 32 bits of the target register. We theoretically could leave those
5108 // bits unchanged, but we would have to make sure that no code uses a
5109 // 32bit input value as a 64bit value assuming that the top 32 bits are
5110 // zero.
5111 __ Mov(output.W(), source.W());
Alexandre Rames8626b742015-11-25 16:28:08 +00005112 } else if (result_type == Primitive::kPrimChar ||
5113 (input_type == Primitive::kPrimChar && input_size < result_size)) {
5114 __ Ubfx(output,
5115 output.IsX() ? source.X() : source.W(),
5116 0, Primitive::ComponentSize(Primitive::kPrimChar) * kBitsPerByte);
Alexandre Rames67555f72014-11-18 10:55:16 +00005117 } else {
Alexandre Rames3e69f162014-12-10 10:36:50 +00005118 __ Sbfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
Alexandre Rames67555f72014-11-18 10:55:16 +00005119 }
Alexandre Rames542361f2015-01-29 16:57:31 +00005120 } else if (Primitive::IsFloatingPointType(result_type) && Primitive::IsIntegralType(input_type)) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00005121 __ Scvtf(OutputFPRegister(conversion), InputRegisterAt(conversion, 0));
Alexandre Rames542361f2015-01-29 16:57:31 +00005122 } else if (Primitive::IsIntegralType(result_type) && Primitive::IsFloatingPointType(input_type)) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00005123 CHECK(result_type == Primitive::kPrimInt || result_type == Primitive::kPrimLong);
5124 __ Fcvtzs(OutputRegister(conversion), InputFPRegisterAt(conversion, 0));
Alexandre Rames542361f2015-01-29 16:57:31 +00005125 } else if (Primitive::IsFloatingPointType(result_type) &&
5126 Primitive::IsFloatingPointType(input_type)) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00005127 __ Fcvt(OutputFPRegister(conversion), InputFPRegisterAt(conversion, 0));
5128 } else {
5129 LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type
5130 << " to " << result_type;
Alexandre Rames67555f72014-11-18 10:55:16 +00005131 }
Serban Constantinescu02164b32014-11-13 14:05:07 +00005132}
Alexandre Rames67555f72014-11-18 10:55:16 +00005133
Serban Constantinescu02164b32014-11-13 14:05:07 +00005134void LocationsBuilderARM64::VisitUShr(HUShr* ushr) {
5135 HandleShift(ushr);
5136}
5137
5138void InstructionCodeGeneratorARM64::VisitUShr(HUShr* ushr) {
5139 HandleShift(ushr);
Alexandre Rames67555f72014-11-18 10:55:16 +00005140}
5141
5142void LocationsBuilderARM64::VisitXor(HXor* instruction) {
5143 HandleBinaryOp(instruction);
5144}
5145
5146void InstructionCodeGeneratorARM64::VisitXor(HXor* instruction) {
5147 HandleBinaryOp(instruction);
5148}
5149
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01005150void LocationsBuilderARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
Calin Juravleb1498f62015-02-16 13:13:29 +00005151 // Nothing to do, this should be removed during prepare for register allocator.
Calin Juravleb1498f62015-02-16 13:13:29 +00005152 LOG(FATAL) << "Unreachable";
5153}
5154
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01005155void InstructionCodeGeneratorARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
Calin Juravleb1498f62015-02-16 13:13:29 +00005156 // Nothing to do, this should be removed during prepare for register allocator.
Calin Juravleb1498f62015-02-16 13:13:29 +00005157 LOG(FATAL) << "Unreachable";
5158}
5159
Mark Mendellfe57faa2015-09-18 09:26:15 -04005160// Simple implementation of packed switch - generate cascaded compare/jumps.
5161void LocationsBuilderARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
5162 LocationSummary* locations =
5163 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
5164 locations->SetInAt(0, Location::RequiresRegister());
5165}
5166
5167void InstructionCodeGeneratorARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
5168 int32_t lower_bound = switch_instr->GetStartValue();
Zheng Xu3927c8b2015-11-18 17:46:25 +08005169 uint32_t num_entries = switch_instr->GetNumEntries();
Mark Mendellfe57faa2015-09-18 09:26:15 -04005170 Register value_reg = InputRegisterAt(switch_instr, 0);
5171 HBasicBlock* default_block = switch_instr->GetDefaultBlock();
5172
Zheng Xu3927c8b2015-11-18 17:46:25 +08005173 // Roughly set 16 as max average assemblies generated per HIR in a graph.
Scott Wakeling97c72b72016-06-24 16:19:36 +01005174 static constexpr int32_t kMaxExpectedSizePerHInstruction = 16 * kInstructionSize;
Zheng Xu3927c8b2015-11-18 17:46:25 +08005175 // ADR has a limited range(+/-1MB), so we set a threshold for the number of HIRs in the graph to
5176 // make sure we don't emit it if the target may run out of range.
5177 // TODO: Instead of emitting all jump tables at the end of the code, we could keep track of ADR
5178 // ranges and emit the tables only as required.
5179 static constexpr int32_t kJumpTableInstructionThreshold = 1* MB / kMaxExpectedSizePerHInstruction;
Mark Mendellfe57faa2015-09-18 09:26:15 -04005180
Vladimir Markof3e0ee22015-12-17 15:23:13 +00005181 if (num_entries <= kPackedSwitchCompareJumpThreshold ||
Zheng Xu3927c8b2015-11-18 17:46:25 +08005182 // Current instruction id is an upper bound of the number of HIRs in the graph.
5183 GetGraph()->GetCurrentInstructionId() > kJumpTableInstructionThreshold) {
5184 // Create a series of compare/jumps.
Vladimir Markof3e0ee22015-12-17 15:23:13 +00005185 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
5186 Register temp = temps.AcquireW();
5187 __ Subs(temp, value_reg, Operand(lower_bound));
5188
Zheng Xu3927c8b2015-11-18 17:46:25 +08005189 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
Vladimir Markof3e0ee22015-12-17 15:23:13 +00005190 // Jump to successors[0] if value == lower_bound.
5191 __ B(eq, codegen_->GetLabelOf(successors[0]));
5192 int32_t last_index = 0;
5193 for (; num_entries - last_index > 2; last_index += 2) {
5194 __ Subs(temp, temp, Operand(2));
5195 // Jump to successors[last_index + 1] if value < case_value[last_index + 2].
5196 __ B(lo, codegen_->GetLabelOf(successors[last_index + 1]));
5197 // Jump to successors[last_index + 2] if value == case_value[last_index + 2].
5198 __ B(eq, codegen_->GetLabelOf(successors[last_index + 2]));
5199 }
5200 if (num_entries - last_index == 2) {
5201 // The last missing case_value.
5202 __ Cmp(temp, Operand(1));
5203 __ B(eq, codegen_->GetLabelOf(successors[last_index + 1]));
Zheng Xu3927c8b2015-11-18 17:46:25 +08005204 }
5205
5206 // And the default for any other value.
5207 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
5208 __ B(codegen_->GetLabelOf(default_block));
5209 }
5210 } else {
Alexandre Ramesc01a6642016-04-15 11:54:06 +01005211 JumpTableARM64* jump_table = codegen_->CreateJumpTable(switch_instr);
Zheng Xu3927c8b2015-11-18 17:46:25 +08005212
5213 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
5214
5215 // Below instructions should use at most one blocked register. Since there are two blocked
5216 // registers, we are free to block one.
5217 Register temp_w = temps.AcquireW();
5218 Register index;
5219 // Remove the bias.
5220 if (lower_bound != 0) {
5221 index = temp_w;
5222 __ Sub(index, value_reg, Operand(lower_bound));
5223 } else {
5224 index = value_reg;
5225 }
5226
5227 // Jump to default block if index is out of the range.
5228 __ Cmp(index, Operand(num_entries));
5229 __ B(hs, codegen_->GetLabelOf(default_block));
5230
5231 // In current VIXL implementation, it won't require any blocked registers to encode the
5232 // immediate value for Adr. So we are free to use both VIXL blocked registers to reduce the
5233 // register pressure.
5234 Register table_base = temps.AcquireX();
5235 // Load jump offset from the table.
5236 __ Adr(table_base, jump_table->GetTableStartLabel());
5237 Register jump_offset = temp_w;
5238 __ Ldr(jump_offset, MemOperand(table_base, index, UXTW, 2));
5239
5240 // Jump to target block by branching to table_base(pc related) + offset.
5241 Register target_address = table_base;
5242 __ Add(target_address, table_base, Operand(jump_offset, SXTW));
5243 __ Br(target_address);
Mark Mendellfe57faa2015-09-18 09:26:15 -04005244 }
5245}
5246
Roland Levillain44015862016-01-22 11:47:17 +00005247void InstructionCodeGeneratorARM64::GenerateReferenceLoadOneRegister(HInstruction* instruction,
5248 Location out,
5249 uint32_t offset,
5250 Location maybe_temp) {
5251 Primitive::Type type = Primitive::kPrimNot;
5252 Register out_reg = RegisterFrom(out, type);
5253 if (kEmitCompilerReadBarrier) {
5254 Register temp_reg = RegisterFrom(maybe_temp, type);
5255 if (kUseBakerReadBarrier) {
5256 // Load with fast path based Baker's read barrier.
5257 // /* HeapReference<Object> */ out = *(out + offset)
5258 codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction,
5259 out,
5260 out_reg,
5261 offset,
5262 temp_reg,
5263 /* needs_null_check */ false,
5264 /* use_load_acquire */ false);
5265 } else {
5266 // Load with slow path based read barrier.
5267 // Save the value of `out` into `maybe_temp` before overwriting it
5268 // in the following move operation, as we will need it for the
5269 // read barrier below.
5270 __ Mov(temp_reg, out_reg);
5271 // /* HeapReference<Object> */ out = *(out + offset)
5272 __ Ldr(out_reg, HeapOperand(out_reg, offset));
5273 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset);
5274 }
5275 } else {
5276 // Plain load with no read barrier.
5277 // /* HeapReference<Object> */ out = *(out + offset)
5278 __ Ldr(out_reg, HeapOperand(out_reg, offset));
5279 GetAssembler()->MaybeUnpoisonHeapReference(out_reg);
5280 }
5281}
5282
5283void InstructionCodeGeneratorARM64::GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
5284 Location out,
5285 Location obj,
5286 uint32_t offset,
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07005287 Location maybe_temp,
5288 bool emit_read_barrier) {
Roland Levillain44015862016-01-22 11:47:17 +00005289 Primitive::Type type = Primitive::kPrimNot;
5290 Register out_reg = RegisterFrom(out, type);
5291 Register obj_reg = RegisterFrom(obj, type);
Mathieu Chartier5c44c1b2016-11-04 18:13:04 -07005292 if (emit_read_barrier) {
5293 DCHECK(kEmitCompilerReadBarrier);
Roland Levillain44015862016-01-22 11:47:17 +00005294 if (kUseBakerReadBarrier) {
5295 // Load with fast path based Baker's read barrier.
5296 Register temp_reg = RegisterFrom(maybe_temp, type);
5297 // /* HeapReference<Object> */ out = *(obj + offset)
5298 codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction,
5299 out,
5300 obj_reg,
5301 offset,
5302 temp_reg,
5303 /* needs_null_check */ false,
5304 /* use_load_acquire */ false);
5305 } else {
5306 // Load with slow path based read barrier.
5307 // /* HeapReference<Object> */ out = *(obj + offset)
5308 __ Ldr(out_reg, HeapOperand(obj_reg, offset));
5309 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset);
5310 }
5311 } else {
5312 // Plain load with no read barrier.
5313 // /* HeapReference<Object> */ out = *(obj + offset)
5314 __ Ldr(out_reg, HeapOperand(obj_reg, offset));
5315 GetAssembler()->MaybeUnpoisonHeapReference(out_reg);
5316 }
5317}
5318
5319void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instruction,
5320 Location root,
Scott Wakeling97c72b72016-06-24 16:19:36 +01005321 Register obj,
Vladimir Markocac5a7e2016-02-22 10:39:50 +00005322 uint32_t offset,
Mathieu Chartier31b12e32016-09-02 17:11:57 -07005323 vixl::aarch64::Label* fixup_label,
5324 bool requires_read_barrier) {
Vladimir Markoaad75c62016-10-03 08:46:48 +00005325 DCHECK(fixup_label == nullptr || offset == 0u);
Roland Levillain44015862016-01-22 11:47:17 +00005326 Register root_reg = RegisterFrom(root, Primitive::kPrimNot);
Mathieu Chartier31b12e32016-09-02 17:11:57 -07005327 if (requires_read_barrier) {
5328 DCHECK(kEmitCompilerReadBarrier);
Roland Levillain44015862016-01-22 11:47:17 +00005329 if (kUseBakerReadBarrier) {
5330 // Fast path implementation of art::ReadBarrier::BarrierForRoot when
5331 // Baker's read barrier are used:
5332 //
5333 // root = obj.field;
5334 // if (Thread::Current()->GetIsGcMarking()) {
5335 // root = ReadBarrier::Mark(root)
5336 // }
5337
5338 // /* GcRoot<mirror::Object> */ root = *(obj + offset)
Vladimir Markocac5a7e2016-02-22 10:39:50 +00005339 if (fixup_label == nullptr) {
5340 __ Ldr(root_reg, MemOperand(obj, offset));
5341 } else {
Vladimir Markoaad75c62016-10-03 08:46:48 +00005342 codegen_->EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj);
Vladimir Markocac5a7e2016-02-22 10:39:50 +00005343 }
Roland Levillain44015862016-01-22 11:47:17 +00005344 static_assert(
5345 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
5346 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
5347 "have different sizes.");
5348 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t),
5349 "art::mirror::CompressedReference<mirror::Object> and int32_t "
5350 "have different sizes.");
5351
Vladimir Marko953437b2016-08-24 08:30:46 +00005352 // Slow path marking the GC root `root`.
Roland Levillain44015862016-01-22 11:47:17 +00005353 SlowPathCodeARM64* slow_path =
Roland Levillain02b75802016-07-13 11:54:35 +01005354 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root);
Roland Levillain44015862016-01-22 11:47:17 +00005355 codegen_->AddSlowPath(slow_path);
5356
5357 MacroAssembler* masm = GetVIXLAssembler();
5358 UseScratchRegisterScope temps(masm);
5359 Register temp = temps.AcquireW();
5360 // temp = Thread::Current()->GetIsGcMarking()
Andreas Gampe542451c2016-07-26 09:02:02 -07005361 __ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
Roland Levillain44015862016-01-22 11:47:17 +00005362 __ Cbnz(temp, slow_path->GetEntryLabel());
5363 __ Bind(slow_path->GetExitLabel());
5364 } else {
5365 // GC root loaded through a slow path for read barriers other
5366 // than Baker's.
5367 // /* GcRoot<mirror::Object>* */ root = obj + offset
Vladimir Markocac5a7e2016-02-22 10:39:50 +00005368 if (fixup_label == nullptr) {
5369 __ Add(root_reg.X(), obj.X(), offset);
5370 } else {
Vladimir Markoaad75c62016-10-03 08:46:48 +00005371 codegen_->EmitAddPlaceholder(fixup_label, root_reg.X(), obj.X());
Vladimir Markocac5a7e2016-02-22 10:39:50 +00005372 }
Roland Levillain44015862016-01-22 11:47:17 +00005373 // /* mirror::Object* */ root = root->Read()
5374 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
5375 }
5376 } else {
5377 // Plain GC root load with no read barrier.
5378 // /* GcRoot<mirror::Object> */ root = *(obj + offset)
Vladimir Markocac5a7e2016-02-22 10:39:50 +00005379 if (fixup_label == nullptr) {
5380 __ Ldr(root_reg, MemOperand(obj, offset));
5381 } else {
Vladimir Markoaad75c62016-10-03 08:46:48 +00005382 codegen_->EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj.X());
Vladimir Markocac5a7e2016-02-22 10:39:50 +00005383 }
Roland Levillain44015862016-01-22 11:47:17 +00005384 // Note that GC roots are not affected by heap poisoning, thus we
5385 // do not have to unpoison `root_reg` here.
5386 }
5387}
5388
5389void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
5390 Location ref,
Scott Wakeling97c72b72016-06-24 16:19:36 +01005391 Register obj,
Roland Levillain44015862016-01-22 11:47:17 +00005392 uint32_t offset,
5393 Register temp,
5394 bool needs_null_check,
5395 bool use_load_acquire) {
5396 DCHECK(kEmitCompilerReadBarrier);
5397 DCHECK(kUseBakerReadBarrier);
5398
5399 // /* HeapReference<Object> */ ref = *(obj + offset)
5400 Location no_index = Location::NoLocation();
Roland Levillaina1aa3b12016-10-26 13:03:38 +01005401 size_t no_scale_factor = 0u;
Roland Levillainbfea3352016-06-23 13:48:47 +01005402 GenerateReferenceLoadWithBakerReadBarrier(instruction,
5403 ref,
5404 obj,
5405 offset,
5406 no_index,
5407 no_scale_factor,
5408 temp,
5409 needs_null_check,
5410 use_load_acquire);
Roland Levillain44015862016-01-22 11:47:17 +00005411}
5412
5413void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
5414 Location ref,
Scott Wakeling97c72b72016-06-24 16:19:36 +01005415 Register obj,
Roland Levillain44015862016-01-22 11:47:17 +00005416 uint32_t data_offset,
5417 Location index,
5418 Register temp,
5419 bool needs_null_check) {
5420 DCHECK(kEmitCompilerReadBarrier);
5421 DCHECK(kUseBakerReadBarrier);
5422
5423 // Array cells are never volatile variables, therefore array loads
5424 // never use Load-Acquire instructions on ARM64.
5425 const bool use_load_acquire = false;
5426
Roland Levillainbfea3352016-06-23 13:48:47 +01005427 static_assert(
5428 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
5429 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
Roland Levillain44015862016-01-22 11:47:17 +00005430 // /* HeapReference<Object> */ ref =
5431 // *(obj + data_offset + index * sizeof(HeapReference<Object>))
Roland Levillainbfea3352016-06-23 13:48:47 +01005432 size_t scale_factor = Primitive::ComponentSizeShift(Primitive::kPrimNot);
5433 GenerateReferenceLoadWithBakerReadBarrier(instruction,
5434 ref,
5435 obj,
5436 data_offset,
5437 index,
5438 scale_factor,
5439 temp,
5440 needs_null_check,
5441 use_load_acquire);
Roland Levillain44015862016-01-22 11:47:17 +00005442}
5443
5444void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
5445 Location ref,
Scott Wakeling97c72b72016-06-24 16:19:36 +01005446 Register obj,
Roland Levillain44015862016-01-22 11:47:17 +00005447 uint32_t offset,
5448 Location index,
Roland Levillainbfea3352016-06-23 13:48:47 +01005449 size_t scale_factor,
Roland Levillain44015862016-01-22 11:47:17 +00005450 Register temp,
5451 bool needs_null_check,
Roland Levillaina1aa3b12016-10-26 13:03:38 +01005452 bool use_load_acquire,
5453 bool always_update_field) {
Roland Levillain44015862016-01-22 11:47:17 +00005454 DCHECK(kEmitCompilerReadBarrier);
5455 DCHECK(kUseBakerReadBarrier);
Roland Levillainbfea3352016-06-23 13:48:47 +01005456 // If we are emitting an array load, we should not be using a
5457 // Load Acquire instruction. In other words:
5458 // `instruction->IsArrayGet()` => `!use_load_acquire`.
5459 DCHECK(!instruction->IsArrayGet() || !use_load_acquire);
Roland Levillain44015862016-01-22 11:47:17 +00005460
5461 MacroAssembler* masm = GetVIXLAssembler();
5462 UseScratchRegisterScope temps(masm);
5463
5464 // In slow path based read barriers, the read barrier call is
5465 // inserted after the original load. However, in fast path based
5466 // Baker's read barriers, we need to perform the load of
5467 // mirror::Object::monitor_ *before* the original reference load.
5468 // This load-load ordering is required by the read barrier.
5469 // The fast path/slow path (for Baker's algorithm) should look like:
5470 //
5471 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
5472 // lfence; // Load fence or artificial data dependency to prevent load-load reordering
5473 // HeapReference<Object> ref = *src; // Original reference load.
Hiroshi Yamauchi12b58b22016-11-01 11:55:29 -07005474 // bool is_gray = (rb_state == ReadBarrier::GrayState());
Roland Levillain44015862016-01-22 11:47:17 +00005475 // if (is_gray) {
5476 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
5477 // }
5478 //
5479 // Note: the original implementation in ReadBarrier::Barrier is
5480 // slightly more complex as it performs additional checks that we do
5481 // not do here for performance reasons.
5482
5483 Primitive::Type type = Primitive::kPrimNot;
5484 Register ref_reg = RegisterFrom(ref, type);
5485 DCHECK(obj.IsW());
5486 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
5487
5488 // /* int32_t */ monitor = obj->monitor_
5489 __ Ldr(temp, HeapOperand(obj, monitor_offset));
5490 if (needs_null_check) {
5491 MaybeRecordImplicitNullCheck(instruction);
5492 }
5493 // /* LockWord */ lock_word = LockWord(monitor)
5494 static_assert(sizeof(LockWord) == sizeof(int32_t),
5495 "art::LockWord and int32_t have different sizes.");
Roland Levillain44015862016-01-22 11:47:17 +00005496
Vladimir Marko877a0332016-07-11 19:30:56 +01005497 // Introduce a dependency on the lock_word including rb_state,
5498 // to prevent load-load reordering, and without using
Roland Levillain44015862016-01-22 11:47:17 +00005499 // a memory barrier (which would be more expensive).
Roland Levillain0b671c02016-08-19 12:02:34 +01005500 // `obj` is unchanged by this operation, but its value now depends
5501 // on `temp`.
Vladimir Marko877a0332016-07-11 19:30:56 +01005502 __ Add(obj.X(), obj.X(), Operand(temp.X(), LSR, 32));
Roland Levillain44015862016-01-22 11:47:17 +00005503
5504 // The actual reference load.
5505 if (index.IsValid()) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01005506 // Load types involving an "index": ArrayGet,
5507 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
5508 // intrinsics.
Roland Levillainbfea3352016-06-23 13:48:47 +01005509 if (use_load_acquire) {
5510 // UnsafeGetObjectVolatile intrinsic case.
5511 // Register `index` is not an index in an object array, but an
5512 // offset to an object reference field within object `obj`.
5513 DCHECK(instruction->IsInvoke()) << instruction->DebugName();
5514 DCHECK(instruction->GetLocations()->Intrinsified());
5515 DCHECK(instruction->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)
5516 << instruction->AsInvoke()->GetIntrinsic();
Roland Levillaina1aa3b12016-10-26 13:03:38 +01005517 DCHECK_EQ(offset, 0u);
5518 DCHECK_EQ(scale_factor, 0u);
5519 DCHECK_EQ(needs_null_check, 0u);
Roland Levillainbfea3352016-06-23 13:48:47 +01005520 // /* HeapReference<Object> */ ref = *(obj + index)
5521 MemOperand field = HeapOperand(obj, XRegisterFrom(index));
5522 LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false);
Roland Levillain44015862016-01-22 11:47:17 +00005523 } else {
Roland Levillainbfea3352016-06-23 13:48:47 +01005524 // ArrayGet and UnsafeGetObject intrinsics cases.
5525 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
5526 if (index.IsConstant()) {
5527 uint32_t computed_offset = offset + (Int64ConstantFrom(index) << scale_factor);
5528 Load(type, ref_reg, HeapOperand(obj, computed_offset));
5529 } else {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01005530 Register temp3 = temps.AcquireW();
5531 __ Add(temp3, obj, offset);
5532 Load(type, ref_reg, HeapOperand(temp3, XRegisterFrom(index), LSL, scale_factor));
5533 temps.Release(temp3);
Roland Levillainbfea3352016-06-23 13:48:47 +01005534 }
Roland Levillain44015862016-01-22 11:47:17 +00005535 }
Roland Levillain44015862016-01-22 11:47:17 +00005536 } else {
5537 // /* HeapReference<Object> */ ref = *(obj + offset)
5538 MemOperand field = HeapOperand(obj, offset);
5539 if (use_load_acquire) {
5540 LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false);
5541 } else {
5542 Load(type, ref_reg, field);
5543 }
5544 }
5545
5546 // Object* ref = ref_addr->AsMirrorPtr()
5547 GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
5548
Vladimir Marko953437b2016-08-24 08:30:46 +00005549 // Slow path marking the object `ref` when it is gray.
Roland Levillaina1aa3b12016-10-26 13:03:38 +01005550 SlowPathCodeARM64* slow_path;
5551 if (always_update_field) {
5552 // ReadBarrierMarkAndUpdateFieldSlowPathARM64 only supports
5553 // address of the form `obj + field_offset`, where `obj` is a
5554 // register and `field_offset` is a register. Thus `offset` and
5555 // `scale_factor` above are expected to be null in this code path.
5556 DCHECK_EQ(offset, 0u);
5557 DCHECK_EQ(scale_factor, 0u); /* "times 1" */
5558 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM64(
5559 instruction, ref, obj, /* field_offset */ index, temp);
5560 } else {
5561 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, ref);
5562 }
Roland Levillain44015862016-01-22 11:47:17 +00005563 AddSlowPath(slow_path);
5564
Hiroshi Yamauchi12b58b22016-11-01 11:55:29 -07005565 // if (rb_state == ReadBarrier::GrayState())
Roland Levillain44015862016-01-22 11:47:17 +00005566 // ref = ReadBarrier::Mark(ref);
Vladimir Marko877a0332016-07-11 19:30:56 +01005567 // Given the numeric representation, it's enough to check the low bit of the rb_state.
Hiroshi Yamauchi12b58b22016-11-01 11:55:29 -07005568 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
5569 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
Vladimir Marko877a0332016-07-11 19:30:56 +01005570 __ Tbnz(temp, LockWord::kReadBarrierStateShift, slow_path->GetEntryLabel());
Roland Levillain44015862016-01-22 11:47:17 +00005571 __ Bind(slow_path->GetExitLabel());
5572}
5573
5574void CodeGeneratorARM64::GenerateReadBarrierSlow(HInstruction* instruction,
5575 Location out,
5576 Location ref,
5577 Location obj,
5578 uint32_t offset,
5579 Location index) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005580 DCHECK(kEmitCompilerReadBarrier);
5581
Roland Levillain44015862016-01-22 11:47:17 +00005582 // Insert a slow path based read barrier *after* the reference load.
5583 //
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005584 // If heap poisoning is enabled, the unpoisoning of the loaded
5585 // reference will be carried out by the runtime within the slow
5586 // path.
5587 //
5588 // Note that `ref` currently does not get unpoisoned (when heap
5589 // poisoning is enabled), which is alright as the `ref` argument is
5590 // not used by the artReadBarrierSlow entry point.
5591 //
5592 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
5593 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena())
5594 ReadBarrierForHeapReferenceSlowPathARM64(instruction, out, ref, obj, offset, index);
5595 AddSlowPath(slow_path);
5596
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005597 __ B(slow_path->GetEntryLabel());
5598 __ Bind(slow_path->GetExitLabel());
5599}
5600
Roland Levillain44015862016-01-22 11:47:17 +00005601void CodeGeneratorARM64::MaybeGenerateReadBarrierSlow(HInstruction* instruction,
5602 Location out,
5603 Location ref,
5604 Location obj,
5605 uint32_t offset,
5606 Location index) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005607 if (kEmitCompilerReadBarrier) {
Roland Levillain44015862016-01-22 11:47:17 +00005608 // Baker's read barriers shall be handled by the fast path
5609 // (CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier).
5610 DCHECK(!kUseBakerReadBarrier);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005611 // If heap poisoning is enabled, unpoisoning will be taken care of
5612 // by the runtime within the slow path.
Roland Levillain44015862016-01-22 11:47:17 +00005613 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index);
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005614 } else if (kPoisonHeapReferences) {
5615 GetAssembler()->UnpoisonHeapReference(WRegisterFrom(out));
5616 }
5617}
5618
Roland Levillain44015862016-01-22 11:47:17 +00005619void CodeGeneratorARM64::GenerateReadBarrierForRootSlow(HInstruction* instruction,
5620 Location out,
5621 Location root) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005622 DCHECK(kEmitCompilerReadBarrier);
5623
Roland Levillain44015862016-01-22 11:47:17 +00005624 // Insert a slow path based read barrier *after* the GC root load.
5625 //
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005626 // Note that GC roots are not affected by heap poisoning, so we do
5627 // not need to do anything special for this here.
5628 SlowPathCodeARM64* slow_path =
5629 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM64(instruction, out, root);
5630 AddSlowPath(slow_path);
5631
Roland Levillain22ccc3a2015-11-24 13:10:05 +00005632 __ B(slow_path->GetEntryLabel());
5633 __ Bind(slow_path->GetExitLabel());
5634}
5635
Nicolas Geoffraya42363f2015-12-17 14:57:09 +00005636void LocationsBuilderARM64::VisitClassTableGet(HClassTableGet* instruction) {
5637 LocationSummary* locations =
5638 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5639 locations->SetInAt(0, Location::RequiresRegister());
5640 locations->SetOut(Location::RequiresRegister());
5641}
5642
5643void InstructionCodeGeneratorARM64::VisitClassTableGet(HClassTableGet* instruction) {
5644 LocationSummary* locations = instruction->GetLocations();
Vladimir Markoa1de9182016-02-25 11:37:38 +00005645 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) {
Nicolas Geoffrayff484b92016-07-13 14:13:48 +01005646 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
Nicolas Geoffraya42363f2015-12-17 14:57:09 +00005647 instruction->GetIndex(), kArm64PointerSize).SizeValue();
Nicolas Geoffrayff484b92016-07-13 14:13:48 +01005648 __ Ldr(XRegisterFrom(locations->Out()),
5649 MemOperand(XRegisterFrom(locations->InAt(0)), method_offset));
Nicolas Geoffraya42363f2015-12-17 14:57:09 +00005650 } else {
Nicolas Geoffrayff484b92016-07-13 14:13:48 +01005651 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement(
Matthew Gharrity465ecc82016-07-19 21:32:52 +00005652 instruction->GetIndex(), kArm64PointerSize));
Artem Udovichenkoa62cb9b2016-06-30 09:18:25 +00005653 __ Ldr(XRegisterFrom(locations->Out()), MemOperand(XRegisterFrom(locations->InAt(0)),
5654 mirror::Class::ImtPtrOffset(kArm64PointerSize).Uint32Value()));
Nicolas Geoffrayff484b92016-07-13 14:13:48 +01005655 __ Ldr(XRegisterFrom(locations->Out()),
5656 MemOperand(XRegisterFrom(locations->Out()), method_offset));
Nicolas Geoffraya42363f2015-12-17 14:57:09 +00005657 }
Nicolas Geoffraya42363f2015-12-17 14:57:09 +00005658}
5659
Nicolas Geoffray07c919f2016-11-09 17:29:03 +00005660
Nicolas Geoffraya42363f2015-12-17 14:57:09 +00005661
Alexandre Rames67555f72014-11-18 10:55:16 +00005662#undef __
5663#undef QUICK_ENTRY_POINT
5664
Alexandre Rames5319def2014-10-23 10:03:10 +01005665} // namespace arm64
5666} // namespace art