src/compiler/codegen/GenInvoke.cc

/*
 * Copyright (C) 2012 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "oat/runtime/oat_support_entrypoints.h"

namespace art {

/*
 * This source files contains "gen" codegen routines that should
 * be applicable to most targets.  Only mid-level support utilities
 * and "op" calls may be used here.
 */

typedef int (*NextCallInsn)(CompilationUnit*, MIR*, int, uint32_t dexIdx,
                            uint32_t methodIdx, uintptr_t directCode,
                            uintptr_t directMethod, InvokeType type);
LIR* opCondBranch(CompilationUnit* cUnit, ConditionCode cc, LIR* target);

/*
 * If there are any ins passed in registers that have not been promoted
 * to a callee-save register, flush them to the frame.  Perform intial
 * assignment of promoted arguments.
 */
void flushIns(CompilationUnit* cUnit)
{
    /*
     * Dummy up a RegLocation for the incoming Method*
     * It will attempt to keep rARG0 live (or copy it to home location
     * if promoted).
     */
    RegLocation rlSrc = cUnit->regLocation[cUnit->methodSReg];
    RegLocation rlMethod = cUnit->regLocation[cUnit->methodSReg];
    rlSrc.location = kLocPhysReg;
    rlSrc.lowReg = rARG0;
    rlSrc.home = false;
    oatMarkLive(cUnit, rlSrc.lowReg, rlSrc.sRegLow);
    storeValue(cUnit, rlMethod, rlSrc);
    // If Method* has been promoted, explicitly flush
    if (rlMethod.location == kLocPhysReg) {
        storeWordDisp(cUnit, rSP, 0, rARG0);
    }

    if (cUnit->numIns == 0)
        return;
    const int numArgRegs = 3;
    static int argRegs[] = {rARG1, rARG2, rARG3};
    int startVReg = cUnit->numDalvikRegisters - cUnit->numIns;
    /*
     * Copy incoming arguments to their proper home locations.
     * NOTE: an older version of dx had an issue in which
     * it would reuse static method argument registers.
     * This could result in the same Dalvik virtual register
     * being promoted to both core and fp regs. To account for this,
     * we only copy to the corresponding promoted physical register
     * if it matches the type of the SSA name for the incoming
     * argument.  It is also possible that long and double arguments
     * end up half-promoted.  In those cases, we must flush the promoted
     * half to memory as well.
     */
    for (int i = 0; i < cUnit->numIns; i++) {
        PromotionMap* vMap = &cUnit->promotionMap[startVReg + i];
        if (i < numArgRegs) {
            // If arriving in register
            bool needFlush = true;
            RegLocation* tLoc = &cUnit->regLocation[startVReg + i];
            if ((vMap->coreLocation == kLocPhysReg) && !tLoc->fp) {
                opRegCopy(cUnit, vMap->coreReg, argRegs[i]);
                needFlush = false;
            } else if ((vMap->fpLocation == kLocPhysReg) && tLoc->fp) {
                opRegCopy(cUnit, vMap->fpReg, argRegs[i]);
                needFlush = false;
            } else {
                needFlush = true;
            }

            // For wide args, force flush if only half is promoted
            if (tLoc->wide) {
                PromotionMap* pMap = vMap + (tLoc->highWord ? -1 : +1);
                needFlush |= (pMap->coreLocation != vMap->coreLocation) ||
                             (pMap->fpLocation != vMap->fpLocation);
            }
            if (needFlush) {
                storeBaseDisp(cUnit, rSP, oatSRegOffset(cUnit, startVReg + i),
                              argRegs[i], kWord);
            }
        } else {
            // If arriving in frame & promoted
            if (vMap->coreLocation == kLocPhysReg) {
                loadWordDisp(cUnit, rSP, oatSRegOffset(cUnit, startVReg + i),
                             vMap->coreReg);
            }
            if (vMap->fpLocation == kLocPhysReg) {
                loadWordDisp(cUnit, rSP, oatSRegOffset(cUnit, startVReg + i),
                             vMap->fpReg);
            }
        }
    }
}

void scanMethodLiteralPool(CompilationUnit* cUnit, LIR** methodTarget, LIR** codeTarget, const DexFile* dexFile, uint32_t dexMethodIdx)
{
    LIR* curTarget = cUnit->methodLiteralList;
    LIR* nextTarget = curTarget != NULL ? curTarget->next : NULL;
    while (curTarget != NULL && nextTarget != NULL) {
      if (curTarget->operands[0] == (int)dexFile &&
          nextTarget->operands[0] == (int)dexMethodIdx) {
        *codeTarget = curTarget;
        *methodTarget = nextTarget;
        DCHECK((*codeTarget)->next == *methodTarget);
        DCHECK_EQ((*codeTarget)->operands[0], (int)dexFile);
        DCHECK_EQ((*methodTarget)->operands[0], (int)dexMethodIdx);
        break;
      }
      curTarget = nextTarget->next;
      nextTarget = curTarget != NULL ? curTarget->next : NULL;
    }
}

/*
 * Bit of a hack here - in the absence of a real scheduling pass,
 * emit the next instruction in static & direct invoke sequences.
 */
int nextSDCallInsn(CompilationUnit* cUnit, MIR* mir,
                   int state, uint32_t dexIdx, uint32_t unused,
                   uintptr_t directCode, uintptr_t directMethod,
                   InvokeType type)
{
#if !defined(TARGET_ARM)
    directCode = 0;
    directMethod = 0;
#endif
    if (directCode != 0 && directMethod != 0) {
        switch (state) {
        case 0:  // Get the current Method* [sets rARG0]
            if (directCode != (uintptr_t)-1) {
                loadConstant(cUnit, rINVOKE_TGT, directCode);
            } else {
                LIR* dataTarget = scanLiteralPool(cUnit->codeLiteralList, dexIdx, 0);
                if (dataTarget == NULL) {
                    dataTarget = addWordData(cUnit, &cUnit->codeLiteralList, dexIdx);
                    dataTarget->operands[1] = type;
                }
#if defined(TARGET_ARM)
                LIR* loadPcRel = rawLIR(cUnit, cUnit->currentDalvikOffset,
                                        kThumb2LdrPcRel12, rINVOKE_TGT, 0, 0, 0, 0, dataTarget);
                oatAppendLIR(cUnit, loadPcRel);
#else
                UNIMPLEMENTED(FATAL) << (void*)dataTarget;
#endif
            }
            if (directMethod != (uintptr_t)-1) {
                loadConstant(cUnit, rARG0, directMethod);
            } else {
                LIR* dataTarget = scanLiteralPool(cUnit->methodLiteralList, dexIdx, 0);
                if (dataTarget == NULL) {
                    dataTarget = addWordData(cUnit, &cUnit->methodLiteralList, dexIdx);
                    dataTarget->operands[1] = type;
                }
#if defined(TARGET_ARM)
                LIR* loadPcRel = rawLIR(cUnit, cUnit->currentDalvikOffset,
                                        kThumb2LdrPcRel12, rARG0, 0, 0, 0, 0, dataTarget);
                oatAppendLIR(cUnit, loadPcRel);
#else
                UNIMPLEMENTED(FATAL) << (void*)dataTarget;
#endif
            }
            break;
        default:
            return -1;
      }
    } else {
        switch (state) {
        case 0:  // Get the current Method* [sets rARG0]
            // TUNING: we can save a reg copy if Method* has been promoted
            loadCurrMethodDirect(cUnit, rARG0);
            break;
        case 1:  // Get method->dex_cache_resolved_methods_
            loadWordDisp(cUnit, rARG0,
                Method::DexCacheResolvedMethodsOffset().Int32Value(),
                rARG0);
            // Set up direct code if known.
            if (directCode != 0) {
                if (directCode != (uintptr_t)-1) {
                    loadConstant(cUnit, rINVOKE_TGT, directCode);
                } else {
                    LIR* dataTarget = scanLiteralPool(cUnit->codeLiteralList, dexIdx, 0);
                    if (dataTarget == NULL) {
                        dataTarget = addWordData(cUnit, &cUnit->codeLiteralList, dexIdx);
                        dataTarget->operands[1] = type;
                    }
#if defined(TARGET_ARM)
                    LIR* loadPcRel = rawLIR(cUnit, cUnit->currentDalvikOffset,
                                            kThumb2LdrPcRel12, rINVOKE_TGT, 0, 0, 0, 0, dataTarget);
                    oatAppendLIR(cUnit, loadPcRel);
#else
                    UNIMPLEMENTED(FATAL) << (void*)dataTarget;
#endif
                }
            }
            break;
        case 2:  // Grab target method*
            loadWordDisp(cUnit, rARG0,
                Array::DataOffset(sizeof(Object*)).Int32Value() + dexIdx * 4,
                rARG0);
            break;
#if !defined(TARGET_X86)
        case 3:  // Grab the code from the method*
            if (directCode == 0) {
                loadWordDisp(cUnit, rARG0, Method::GetCodeOffset().Int32Value(),
                             rINVOKE_TGT);
            }
            break;
#endif
        default:
            return -1;
        }
    }
    return state + 1;
}

/*
 * Bit of a hack here - in the absence of a real scheduling pass,
 * emit the next instruction in a virtual invoke sequence.
 * We can use rLR as a temp prior to target address loading
 * Note also that we'll load the first argument ("this") into
 * rARG1 here rather than the standard loadArgRegs.
 */
int nextVCallInsn(CompilationUnit* cUnit, MIR* mir,
                  int state, uint32_t dexIdx, uint32_t methodIdx,
                  uintptr_t unused, uintptr_t unused2, InvokeType unused3)
{
    RegLocation rlArg;
    /*
     * This is the fast path in which the target virtual method is
     * fully resolved at compile time.
     */
    switch (state) {
        case 0:  // Get "this" [set rARG1]
            rlArg = oatGetSrc(cUnit, mir, 0);
            loadValueDirectFixed(cUnit, rlArg, rARG1);
            break;
        case 1: // Is "this" null? [use rARG1]
            genNullCheck(cUnit, oatSSASrc(mir,0), rARG1, mir);
            // get this->klass_ [use rARG1, set rINVOKE_TGT]
            loadWordDisp(cUnit, rARG1, Object::ClassOffset().Int32Value(),
                         rINVOKE_TGT);
            break;
        case 2: // Get this->klass_->vtable [usr rINVOKE_TGT, set rINVOKE_TGT]
            loadWordDisp(cUnit, rINVOKE_TGT, Class::VTableOffset().Int32Value(),
                         rINVOKE_TGT);
            break;
        case 3: // Get target method [use rINVOKE_TGT, set rARG0]
            loadWordDisp(cUnit, rINVOKE_TGT, (methodIdx * 4) +
                         Array::DataOffset(sizeof(Object*)).Int32Value(),
                         rARG0);
            break;
#if !defined(TARGET_X86)
        case 4: // Get the compiled code address [uses rARG0, sets rINVOKE_TGT]
            loadWordDisp(cUnit, rARG0, Method::GetCodeOffset().Int32Value(),
                         rINVOKE_TGT);
            break;
#endif
        default:
            return -1;
    }
    return state + 1;
}

int nextInvokeInsnSP(CompilationUnit* cUnit, MIR* mir, int trampoline,
                     int state, uint32_t dexIdx, uint32_t methodIdx)
{
    /*
     * This handles the case in which the base method is not fully
     * resolved at compile time, we bail to a runtime helper.
     */
    if (state == 0) {
#if !defined(TARGET_X86)
        // Load trampoline target
        loadWordDisp(cUnit, rSELF, trampoline, rINVOKE_TGT);
#endif
        // Load rARG0 with method index
        loadConstant(cUnit, rARG0, dexIdx);
        return 1;
    }
    return -1;
}

int nextStaticCallInsnSP(CompilationUnit* cUnit, MIR* mir,
                         int state, uint32_t dexIdx, uint32_t methodIdx,
                         uintptr_t unused, uintptr_t unused2,
                         InvokeType unused3)
{
  int trampoline = ENTRYPOINT_OFFSET(pInvokeStaticTrampolineWithAccessCheck);
  return nextInvokeInsnSP(cUnit, mir, trampoline, state, dexIdx, 0);
}

int nextDirectCallInsnSP(CompilationUnit* cUnit, MIR* mir, int state,
                         uint32_t dexIdx, uint32_t methodIdx, uintptr_t unused,
                         uintptr_t unused2, InvokeType unused3)
{
  int trampoline = ENTRYPOINT_OFFSET(pInvokeDirectTrampolineWithAccessCheck);
  return nextInvokeInsnSP(cUnit, mir, trampoline, state, dexIdx, 0);
}

int nextSuperCallInsnSP(CompilationUnit* cUnit, MIR* mir, int state,
                        uint32_t dexIdx, uint32_t methodIdx, uintptr_t unused,
                        uintptr_t unused2, InvokeType unused3)
{
  int trampoline = ENTRYPOINT_OFFSET(pInvokeSuperTrampolineWithAccessCheck);
  return nextInvokeInsnSP(cUnit, mir, trampoline, state, dexIdx, 0);
}

int nextVCallInsnSP(CompilationUnit* cUnit, MIR* mir, int state,
                    uint32_t dexIdx, uint32_t methodIdx, uintptr_t unused,
                    uintptr_t unused2, InvokeType unused3)
{
  int trampoline = ENTRYPOINT_OFFSET(pInvokeVirtualTrampolineWithAccessCheck);
  return nextInvokeInsnSP(cUnit, mir, trampoline, state, dexIdx, 0);
}

/*
 * All invoke-interface calls bounce off of art_invoke_interface_trampoline,
 * which will locate the target and continue on via a tail call.
 */
int nextInterfaceCallInsn(CompilationUnit* cUnit, MIR* mir, int state,
                          uint32_t dexIdx, uint32_t unused, uintptr_t unused2,
                          uintptr_t unused3, InvokeType unused4)
{
  int trampoline = ENTRYPOINT_OFFSET(pInvokeInterfaceTrampoline);
  return nextInvokeInsnSP(cUnit, mir, trampoline, state, dexIdx, 0);
}

int nextInterfaceCallInsnWithAccessCheck(CompilationUnit* cUnit, MIR* mir,
                                         int state, uint32_t dexIdx,
                                         uint32_t unused, uintptr_t unused2,
                                         uintptr_t unused3, InvokeType unused4)
{
  int trampoline = ENTRYPOINT_OFFSET(pInvokeInterfaceTrampolineWithAccessCheck);
  return nextInvokeInsnSP(cUnit, mir, trampoline, state, dexIdx, 0);
}

int loadArgRegs(CompilationUnit* cUnit, MIR* mir, DecodedInstruction* dInsn,
                int callState, NextCallInsn nextCallInsn, uint32_t dexIdx,
                uint32_t methodIdx, uintptr_t directCode,
                uintptr_t directMethod, InvokeType type, bool skipThis)
{
    int lastArgReg = rARG3;
    int nextReg = rARG1;
    int nextArg = 0;
    if (skipThis) {
        nextReg++;
        nextArg++;
    }
    for (; (nextReg <= lastArgReg) && (nextArg < mir->ssaRep->numUses); nextReg++) {
        RegLocation rlArg = oatGetRawSrc(cUnit, mir, nextArg++);
        rlArg = oatUpdateRawLoc(cUnit, rlArg);
        if (rlArg.wide && (nextReg <= rARG2)) {
            loadValueDirectWideFixed(cUnit, rlArg, nextReg, nextReg + 1);
            nextReg++;
            nextArg++;
        } else {
            rlArg.wide = false;
            loadValueDirectFixed(cUnit, rlArg, nextReg);
        }
        callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                                 directCode, directMethod, type);
    }
    return callState;
}

/*
 * Load up to 5 arguments, the first three of which will be in
 * rARG1 .. rARG3.  On entry rARG0 contains the current method pointer,
 * and as part of the load sequence, it must be replaced with
 * the target method pointer.  Note, this may also be called
 * for "range" variants if the number of arguments is 5 or fewer.
 */
int genDalvikArgsNoRange(CompilationUnit* cUnit, MIR* mir,
                         DecodedInstruction* dInsn, int callState,
                         LIR** pcrLabel, NextCallInsn nextCallInsn,
                         uint32_t dexIdx, uint32_t methodIdx,
                         uintptr_t directCode, uintptr_t directMethod,
                         InvokeType type, bool skipThis)
{
    RegLocation rlArg;

    /* If no arguments, just return */
    if (dInsn->vA == 0)
        return callState;

    callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                             directCode, directMethod, type);

    DCHECK_LE(dInsn->vA, 5U);
    if (dInsn->vA > 3) {
        uint32_t nextUse = 3;
        //Detect special case of wide arg spanning arg3/arg4
        RegLocation rlUse0 = oatGetRawSrc(cUnit, mir, 0);
        RegLocation rlUse1 = oatGetRawSrc(cUnit, mir, 1);
        RegLocation rlUse2 = oatGetRawSrc(cUnit, mir, 2);
        if (((!rlUse0.wide && !rlUse1.wide) || rlUse0.wide) &&
            rlUse2.wide) {
            int reg = -1;
            // Wide spans, we need the 2nd half of uses[2].
            rlArg = oatUpdateLocWide(cUnit, rlUse2);
            if (rlArg.location == kLocPhysReg) {
                reg = rlArg.highReg;
            } else {
                // rARG2 & rARG3 can safely be used here
                reg = rARG3;
                loadWordDisp(cUnit, rSP,
                             oatSRegOffset(cUnit, rlArg.sRegLow) + 4, reg);
                callState = nextCallInsn(cUnit, mir, callState, dexIdx,
                                         methodIdx, directCode, directMethod,
                                         type);
            }
            storeBaseDisp(cUnit, rSP, (nextUse + 1) * 4, reg, kWord);
            storeBaseDisp(cUnit, rSP, 16 /* (3+1)*4 */, reg, kWord);
            callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                                     directCode, directMethod, type);
            nextUse++;
        }
        // Loop through the rest
        while (nextUse < dInsn->vA) {
            int lowReg;
            int highReg = -1;
            rlArg = oatGetRawSrc(cUnit, mir, nextUse);
            rlArg = oatUpdateRawLoc(cUnit, rlArg);
            if (rlArg.location == kLocPhysReg) {
                lowReg = rlArg.lowReg;
                highReg = rlArg.highReg;
            } else {
                lowReg = rARG2;
                if (rlArg.wide) {
                    highReg = rARG3;
                    loadValueDirectWideFixed(cUnit, rlArg, lowReg, highReg);
                } else {
                    loadValueDirectFixed(cUnit, rlArg, lowReg);
                }
                callState = nextCallInsn(cUnit, mir, callState, dexIdx,
                                         methodIdx, directCode, directMethod,
                                         type);
            }
            int outsOffset = (nextUse + 1) * 4;
            if (rlArg.wide) {
                storeBaseDispWide(cUnit, rSP, outsOffset, lowReg, highReg);
                nextUse += 2;
            } else {
                storeWordDisp(cUnit, rSP, outsOffset, lowReg);
                nextUse++;
            }
            callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                                     directCode, directMethod, type);
        }
    }

    callState = loadArgRegs(cUnit, mir, dInsn, callState, nextCallInsn,
                            dexIdx, methodIdx, directCode, directMethod,
                            type, skipThis);

    if (pcrLabel) {
        *pcrLabel = genNullCheck(cUnit, oatSSASrc(mir,0), rARG1, mir);
    }
    return callState;
}

/*
 * May have 0+ arguments (also used for jumbo).  Note that
 * source virtual registers may be in physical registers, so may
 * need to be flushed to home location before copying.  This
 * applies to arg3 and above (see below).
 *
 * Two general strategies:
 *    If < 20 arguments
 *       Pass args 3-18 using vldm/vstm block copy
 *       Pass arg0, arg1 & arg2 in rARG1-rARG3
 *    If 20+ arguments
 *       Pass args arg19+ using memcpy block copy
 *       Pass arg0, arg1 & arg2 in rARG1-rARG3
 *
 */
int genDalvikArgsRange(CompilationUnit* cUnit, MIR* mir,
                       DecodedInstruction* dInsn, int callState,
                       LIR** pcrLabel, NextCallInsn nextCallInsn,
                       uint32_t dexIdx, uint32_t methodIdx,
                       uintptr_t directCode, uintptr_t directMethod,
                       InvokeType type, bool skipThis)
{
    int firstArg = dInsn->vC;
    int numArgs = dInsn->vA;

    // If we can treat it as non-range (Jumbo ops will use range form)
    if (numArgs <= 5)
        return genDalvikArgsNoRange(cUnit, mir, dInsn, callState, pcrLabel,
                                    nextCallInsn, dexIdx, methodIdx,
                                    directCode, directMethod, type, skipThis);
    /*
     * Make sure range list doesn't span the break between in normal
     * Dalvik vRegs and the ins.
     */
    int highestArg = oatGetSrc(cUnit, mir, numArgs-1).sRegLow;
    int boundaryReg = cUnit->numDalvikRegisters - cUnit->numIns;
    if ((firstArg < boundaryReg) && (highestArg >= boundaryReg)) {
        LOG(FATAL) << "Argument list spanned locals & args";
    }

    /*
     * First load the non-register arguments.  Both forms expect all
     * of the source arguments to be in their home frame location, so
     * scan the sReg names and flush any that have been promoted to
     * frame backing storage.
     */
    // Scan the rest of the args - if in physReg flush to memory
    for (int nextArg = 0; nextArg < numArgs;) {
        RegLocation loc = oatGetRawSrc(cUnit, mir, nextArg);
        if (loc.wide) {
            loc = oatUpdateLocWide(cUnit, loc);
            if ((nextArg >= 2) && (loc.location == kLocPhysReg)) {
                storeBaseDispWide(cUnit, rSP,
                                  oatSRegOffset(cUnit, loc.sRegLow),
                                  loc.lowReg, loc.highReg);
            }
            nextArg += 2;
        } else {
            loc = oatUpdateLoc(cUnit, loc);
            if ((nextArg >= 3) && (loc.location == kLocPhysReg)) {
                storeBaseDisp(cUnit, rSP, oatSRegOffset(cUnit, loc.sRegLow),
                              loc.lowReg, kWord);
            }
            nextArg++;
        }
    }

    int startOffset = oatSRegOffset(cUnit,
        cUnit->regLocation[mir->ssaRep->uses[3]].sRegLow);
    int outsOffset = 4 /* Method* */ + (3 * 4);
#if defined(TARGET_MIPS) || defined(TARGET_X86)
    // Generate memcpy
    opRegRegImm(cUnit, kOpAdd, rARG0, rSP, outsOffset);
    opRegRegImm(cUnit, kOpAdd, rARG1, rSP, startOffset);
    callRuntimeHelperRegRegImm(cUnit, ENTRYPOINT_OFFSET(pMemcpy),
                               rARG0, rARG1, (numArgs - 3) * 4);
#else
    if (numArgs >= 20) {
        // Generate memcpy
        opRegRegImm(cUnit, kOpAdd, rARG0, rSP, outsOffset);
        opRegRegImm(cUnit, kOpAdd, rARG1, rSP, startOffset);
        callRuntimeHelperRegRegImm(cUnit, ENTRYPOINT_OFFSET(pMemcpy),
                                   rARG0, rARG1, (numArgs - 3) * 4);
    } else {
        // Use vldm/vstm pair using rARG3 as a temp
        int regsLeft = std::min(numArgs - 3, 16);
        callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                                 directCode, directMethod, type);
        opRegRegImm(cUnit, kOpAdd, rARG3, rSP, startOffset);
        LIR* ld = newLIR3(cUnit, kThumb2Vldms, rARG3, fr0, regsLeft);
        //TUNING: loosen barrier
        ld->defMask = ENCODE_ALL;
        setMemRefType(ld, true /* isLoad */, kDalvikReg);
        callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                                 directCode, directMethod, type);
        opRegRegImm(cUnit, kOpAdd, rARG3, rSP, 4 /* Method* */ + (3 * 4));
        callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                                 directCode, directMethod, type);
        LIR* st = newLIR3(cUnit, kThumb2Vstms, rARG3, fr0, regsLeft);
        setMemRefType(st, false /* isLoad */, kDalvikReg);
        st->defMask = ENCODE_ALL;
        callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                                 directCode, directMethod, type);

    }
#endif

    callState = loadArgRegs(cUnit, mir, dInsn, callState, nextCallInsn,
                            dexIdx, methodIdx, directCode, directMethod,
                            type, skipThis);

    callState = nextCallInsn(cUnit, mir, callState, dexIdx, methodIdx,
                             directCode, directMethod, type);
    if (pcrLabel) {
        *pcrLabel = genNullCheck(cUnit, oatSSASrc(mir,0), rARG1, mir);
    }
    return callState;
}

RegLocation inlineTarget(CompilationUnit* cUnit, BasicBlock* bb, MIR* mir)
{
    RegLocation res;
    mir = oatFindMoveResult(cUnit, bb, mir, false);
    if (mir == NULL) {
        res = oatGetReturn(cUnit, false);
    } else {
        res = oatGetDest(cUnit, mir, 0);
        mir->dalvikInsn.opcode = Instruction::NOP;
    }
    return res;
}

RegLocation inlineTargetWide(CompilationUnit* cUnit, BasicBlock* bb, MIR* mir)
{
    RegLocation res;
    mir = oatFindMoveResult(cUnit, bb, mir, true);
    if (mir == NULL) {
        res = oatGetReturnWide(cUnit, false);
    } else {
        res = oatGetDestWide(cUnit, mir, 0, 1);
        mir->dalvikInsn.opcode = Instruction::NOP;
    }
    return res;
}

bool genInlinedCharAt(CompilationUnit* cUnit, BasicBlock* bb, MIR* mir,
                      InvokeType type, bool isRange)
{
#if defined(TARGET_ARM)
    // Location of reference to data array
    int valueOffset = String::ValueOffset().Int32Value();
    // Location of count
    int countOffset = String::CountOffset().Int32Value();
    // Starting offset within data array
    int offsetOffset = String::OffsetOffset().Int32Value();
    // Start of char data with array_
    int dataOffset = Array::DataOffset(sizeof(uint16_t)).Int32Value();

    RegLocation rlObj = oatGetSrc(cUnit, mir, 0);
    RegLocation rlIdx = oatGetSrc(cUnit, mir, 1);
    rlObj = loadValue(cUnit, rlObj, kCoreReg);
    rlIdx = loadValue(cUnit, rlIdx, kCoreReg);
    int regMax;
    int regOff = oatAllocTemp(cUnit);
    int regPtr = oatAllocTemp(cUnit);
    genNullCheck(cUnit, rlObj.sRegLow, rlObj.lowReg, mir);
    bool rangeCheck = (!(mir->optimizationFlags & MIR_IGNORE_RANGE_CHECK));
    if (rangeCheck) {
        regMax = oatAllocTemp(cUnit);
        loadWordDisp(cUnit, rlObj.lowReg, countOffset, regMax);
    }
    loadWordDisp(cUnit, rlObj.lowReg, offsetOffset, regOff);
    loadWordDisp(cUnit, rlObj.lowReg, valueOffset, regPtr);
    LIR* launchPad = NULL;
    if (rangeCheck) {
        // Set up a launch pad to allow retry in case of bounds violation */
        launchPad = rawLIR(cUnit, 0, kPseudoIntrinsicRetry, (int)mir, type);
        oatInsertGrowableList(cUnit, &cUnit->intrinsicLaunchpads,
                              (intptr_t)launchPad);
        opRegReg(cUnit, kOpCmp, rlIdx.lowReg, regMax);
        oatFreeTemp(cUnit, regMax);
        opCondBranch(cUnit, kCondCs, launchPad);
    }
    opRegImm(cUnit, kOpAdd, regPtr, dataOffset);
    opRegReg(cUnit, kOpAdd, regOff, rlIdx.lowReg);
    RegLocation rlDest = inlineTarget(cUnit, bb, mir);
    RegLocation rlResult = oatEvalLoc(cUnit, rlDest, kCoreReg, true);
    loadBaseIndexed(cUnit, regPtr, regOff, rlResult.lowReg, 1, kUnsignedHalf);
    oatFreeTemp(cUnit, regOff);
    oatFreeTemp(cUnit, regPtr);
    storeValue(cUnit, rlDest, rlResult);
    if (rangeCheck) {
        launchPad->operands[2] = NULL;  // no resumption
        launchPad->operands[3] = (uintptr_t)bb;
    }
    // Record that we've already inlined & null checked
    mir->optimizationFlags |= (MIR_INLINED | MIR_IGNORE_NULL_CHECK);
    return true;
#else
    return false;
#endif
}

bool genInlinedMinMaxInt(CompilationUnit *cUnit, BasicBlock* bb, MIR *mir,
                         bool isMin)
{
#if defined(TARGET_ARM)
    RegLocation rlSrc1 = oatGetSrc(cUnit, mir, 0);
    RegLocation rlSrc2 = oatGetSrc(cUnit, mir, 1);
    rlSrc1 = loadValue(cUnit, rlSrc1, kCoreReg);
    rlSrc2 = loadValue(cUnit, rlSrc2, kCoreReg);
    RegLocation rlDest = inlineTarget(cUnit, bb, mir);
    RegLocation rlResult = oatEvalLoc(cUnit, rlDest, kCoreReg, true);
    opRegReg(cUnit, kOpCmp, rlSrc1.lowReg, rlSrc2.lowReg);
    opIT(cUnit, (isMin) ? kArmCondGt : kArmCondLt, "E");
    opRegReg(cUnit, kOpMov, rlResult.lowReg, rlSrc2.lowReg);
    opRegReg(cUnit, kOpMov, rlResult.lowReg, rlSrc1.lowReg);
    genBarrier(cUnit);
    storeValue(cUnit, rlDest, rlResult);
    return true;
#else
    return false;
#endif
}

// Generates an inlined String.isEmpty or String.length.
bool genInlinedStringIsEmptyOrLength(CompilationUnit* cUnit,
                                            BasicBlock* bb, MIR* mir,
                                            bool isEmpty)
{
#if defined(TARGET_ARM)
    // dst = src.length();
    RegLocation rlObj = oatGetSrc(cUnit, mir, 0);
    rlObj = loadValue(cUnit, rlObj, kCoreReg);
    RegLocation rlDest = inlineTarget(cUnit, bb, mir);
    RegLocation rlResult = oatEvalLoc(cUnit, rlDest, kCoreReg, true);
    genNullCheck(cUnit, rlObj.sRegLow, rlObj.lowReg, mir);
    loadWordDisp(cUnit, rlObj.lowReg, String::CountOffset().Int32Value(),
                 rlResult.lowReg);
    if (isEmpty) {
        // dst = (dst == 0);
        int tReg = oatAllocTemp(cUnit);
        opRegReg(cUnit, kOpNeg, tReg, rlResult.lowReg);
        opRegRegReg(cUnit, kOpAdc, rlResult.lowReg, rlResult.lowReg, tReg);
    }
    storeValue(cUnit, rlDest, rlResult);
    return true;
#else
    return false;
#endif
}

bool genInlinedAbsInt(CompilationUnit *cUnit, BasicBlock* bb, MIR *mir)
{
#if defined(TARGET_ARM)
    RegLocation rlSrc = oatGetSrc(cUnit, mir, 0);
    rlSrc = loadValue(cUnit, rlSrc, kCoreReg);
    RegLocation rlDest = inlineTarget(cUnit, bb, mir);
    RegLocation rlResult = oatEvalLoc(cUnit, rlDest, kCoreReg, true);
    int signReg = oatAllocTemp(cUnit);
    // abs(x) = y<=x>>31, (x+y)^y.
    opRegRegImm(cUnit, kOpAsr, signReg, rlSrc.lowReg, 31);
    opRegRegReg(cUnit, kOpAdd, rlResult.lowReg, rlSrc.lowReg, signReg);
    opRegReg(cUnit, kOpXor, rlResult.lowReg, signReg);
    storeValue(cUnit, rlDest, rlResult);
    return true;
#else
    return false;
#endif
}

bool genInlinedAbsLong(CompilationUnit *cUnit, BasicBlock* bb, MIR *mir)
{
#if defined(TARGET_ARM)
    RegLocation rlSrc = oatGetSrcWide(cUnit, mir, 0, 1);
    rlSrc = loadValueWide(cUnit, rlSrc, kCoreReg);
    RegLocation rlDest = inlineTargetWide(cUnit, bb, mir);
    RegLocation rlResult = oatEvalLoc(cUnit, rlDest, kCoreReg, true);
    int signReg = oatAllocTemp(cUnit);
    // abs(x) = y<=x>>31, (x+y)^y.
    opRegRegImm(cUnit, kOpAsr, signReg, rlSrc.highReg, 31);
    opRegRegReg(cUnit, kOpAdd, rlResult.lowReg, rlSrc.lowReg, signReg);
    opRegRegReg(cUnit, kOpAdc, rlResult.highReg, rlSrc.highReg, signReg);
    opRegReg(cUnit, kOpXor, rlResult.lowReg, signReg);
    opRegReg(cUnit, kOpXor, rlResult.highReg, signReg);
    storeValueWide(cUnit, rlDest, rlResult);
    return true;
#else
    return false;
#endif
}

bool genInlinedFloatCvt(CompilationUnit *cUnit, BasicBlock* bb, MIR *mir)
{
#if defined(TARGET_ARM)
    RegLocation rlSrc = oatGetSrc(cUnit, mir, 0);
    RegLocation rlDest = inlineTarget(cUnit, bb, mir);
    storeValue(cUnit, rlDest, rlSrc);
    return true;
#else
    return false;
#endif
}

bool genInlinedDoubleCvt(CompilationUnit *cUnit, BasicBlock* bb, MIR *mir)
{
#if defined(TARGET_ARM)
    RegLocation rlSrc = oatGetSrcWide(cUnit, mir, 0, 1);
    RegLocation rlDest = inlineTargetWide(cUnit, bb, mir);
    storeValueWide(cUnit, rlDest, rlSrc);
    return true;
#else
    return false;
#endif
}

/*
 * Fast string.indexOf(I) & (II).  Tests for simple case of char <= 0xffff,
 * otherwise bails to standard library code.
 */
bool genInlinedIndexOf(CompilationUnit* cUnit, BasicBlock* bb, MIR* mir,
                       InvokeType type, bool zeroBased)
{
#if defined(TARGET_ARM)

    oatClobberCalleeSave(cUnit);
    oatLockCallTemps(cUnit);  // Using fixed registers
    int regPtr = rARG0;
    int regChar = rARG1;
    int regStart = rARG2;

    RegLocation rlObj = oatGetSrc(cUnit, mir, 0);
    RegLocation rlChar = oatGetSrc(cUnit, mir, 1);
    RegLocation rlStart = oatGetSrc(cUnit, mir, 2);
    loadValueDirectFixed(cUnit, rlObj, regPtr);
    loadValueDirectFixed(cUnit, rlChar, regChar);
    if (zeroBased) {
        loadConstant(cUnit, regStart, 0);
    } else {
        loadValueDirectFixed(cUnit, rlStart, regStart);
    }
    int rTgt = loadHelper(cUnit, ENTRYPOINT_OFFSET(pIndexOf));
    genNullCheck(cUnit, rlObj.sRegLow, regPtr, mir);
    LIR* launchPad = rawLIR(cUnit, 0, kPseudoIntrinsicRetry, (int)mir, type);
    oatInsertGrowableList(cUnit, &cUnit->intrinsicLaunchpads,
                          (intptr_t)launchPad);
    opCmpImmBranch(cUnit, kCondGt, regChar, 0xFFFF, launchPad);
    opReg(cUnit, kOpBlx, rTgt);
    LIR* resumeTgt = newLIR0(cUnit, kPseudoTargetLabel);
    launchPad->operands[2] = (uintptr_t)resumeTgt;
    launchPad->operands[3] = (uintptr_t)bb;
    // Record that we've already inlined & null checked
    mir->optimizationFlags |= (MIR_INLINED | MIR_IGNORE_NULL_CHECK);
    return true;
#else
    return false;
#endif
}

/* Fast string.compareTo(Ljava/lang/string;)I. */
bool genInlinedStringCompareTo(CompilationUnit* cUnit, BasicBlock* bb,
                               MIR* mir, InvokeType type)
{
#if defined(TARGET_ARM)
    oatClobberCalleeSave(cUnit);
    oatLockCallTemps(cUnit);  // Using fixed registers
    int regThis = rARG0;
    int regCmp = rARG1;

    RegLocation rlThis = oatGetSrc(cUnit, mir, 0);
    RegLocation rlCmp = oatGetSrc(cUnit, mir, 1);
    loadValueDirectFixed(cUnit, rlThis, regThis);
    loadValueDirectFixed(cUnit, rlCmp, regCmp);
    int rTgt = loadHelper(cUnit, ENTRYPOINT_OFFSET(pStringCompareTo));
    genNullCheck(cUnit, rlThis.sRegLow, regThis, mir);
    //TUNING: check if rlCmp.sRegLow is already null checked
    LIR* launchPad = rawLIR(cUnit, 0, kPseudoIntrinsicRetry, (int)mir, type);
    oatInsertGrowableList(cUnit, &cUnit->intrinsicLaunchpads,
                          (intptr_t)launchPad);
    opCmpImmBranch(cUnit, kCondEq, regCmp, 0, launchPad);
    opReg(cUnit, kOpBlx, rTgt);
    launchPad->operands[2] = NULL;  // No return possible
    launchPad->operands[3] = (uintptr_t)bb;
    // Record that we've already inlined & null checked
    mir->optimizationFlags |= (MIR_INLINED | MIR_IGNORE_NULL_CHECK);
    return true;
#else
    return false;
#endif
}

bool genIntrinsic(CompilationUnit* cUnit, BasicBlock* bb, MIR* mir,
                  InvokeType type, bool isRange)
{
    if ((mir->optimizationFlags & MIR_INLINED) || isRange)  {
        return false;
    }
    /*
     * TODO: move these to a target-specific structured constant array
     * and use a generic match function.  The list of intrinsics may be
     * slightly different depending on target.
     * TODO: Fold this into a matching function that runs during
     * basic block building.  This should be part of the action for
     * small method inlining and recognition of the special object init
     * method.  By doing this during basic block construction, we can also
     * take advantage of/generate new useful dataflow info.
     */
    std::string tgtMethod = PrettyMethod(mir->dalvikInsn.vB, *cUnit->dex_file);
    if (tgtMethod.compare("char java.lang.String.charAt(int)") == 0) {
        return genInlinedCharAt(cUnit, bb, mir, type, isRange);
    }
    if (tgtMethod.compare("int java.lang.Math.min(int, int)") == 0) {
        return genInlinedMinMaxInt(cUnit, bb, mir, true /* isMin */);
    }
    if (tgtMethod.compare("int java.lang.Math.max(int, int)") == 0) {
        return genInlinedMinMaxInt(cUnit, bb, mir, false /* isMin */);
    }
    if (tgtMethod.compare("int java.lang.String.length()") == 0) {
        return genInlinedStringIsEmptyOrLength(cUnit, bb, mir, false /* isEmpty */);
    }
    if (tgtMethod.compare("boolean java.lang.String.isEmpty()") == 0) {
        return genInlinedStringIsEmptyOrLength(cUnit, bb, mir, true /* isEmpty */);
    }
    if (tgtMethod.compare("int java.lang.Math.abs(int)") == 0) {
        return genInlinedAbsInt(cUnit, bb, mir);
    }
    if (tgtMethod.compare("long java.lang.Math.abs(long)") == 0) {
        return genInlinedAbsLong(cUnit, bb, mir);
    }
    if (tgtMethod.compare("int java.lang.Float.floatToRawIntBits(float)") == 0) {
        return genInlinedFloatCvt(cUnit, bb, mir);
    }
    if (tgtMethod.compare("float java.lang.Float.intBitsToFloat(int)") == 0) {
        return genInlinedFloatCvt(cUnit, bb, mir);
    }
    if (tgtMethod.compare("long java.lang.Double.doubleToRawLongBits(double)") == 0) {
        return genInlinedDoubleCvt(cUnit, bb, mir);
    }
    if (tgtMethod.compare("double java.lang.Double.longBitsToDouble(long)") == 0) {
        return genInlinedDoubleCvt(cUnit, bb, mir);
    }
    if (tgtMethod.compare("int java.lang.String.indexOf(int, int)") == 0) {
        return genInlinedIndexOf(cUnit, bb, mir, type, false /* base 0 */);
    }
    if (tgtMethod.compare("int java.lang.String.indexOf(int)") == 0) {
        return genInlinedIndexOf(cUnit, bb, mir, type, true /* base 0 */);
    }
    if (tgtMethod.compare("int java.lang.String.compareTo(java.lang.String)") == 0) {
        return genInlinedStringCompareTo(cUnit, bb, mir, type);
    }
    return false;
}


}  // namespace art