diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 5979303b2c55a..ac2e25f95a11d 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -308,6 +308,11 @@ regMaskTP LinearScan::getMatchingConstants(regMaskTP mask, Interval* currentInte return result; } +void LinearScan::clearAllNextIntervalRef() +{ + memset(nextIntervalRef, MaxLocation, AVAILABLE_REG_COUNT * sizeof(LsraLocation)); +} + void LinearScan::clearNextIntervalRef(regNumber reg, var_types regType) { nextIntervalRef[reg] = MaxLocation; @@ -321,6 +326,11 @@ void LinearScan::clearNextIntervalRef(regNumber reg, var_types regType) #endif } +void LinearScan::clearAllSpillCost() +{ + memset(spillCost, 0, AVAILABLE_REG_COUNT * sizeof(weight_t)); +} + void LinearScan::clearSpillCost(regNumber reg, var_types regType) { spillCost[reg] = 0; @@ -723,7 +733,6 @@ LinearScan::LinearScan(Compiler* theCompiler) } #endif // TARGET_XARCH - regSelector = new (theCompiler, CMK_LSRA) RegisterSelection(this); firstColdLoc = MaxLocation; #ifdef DEBUG @@ -762,6 +771,9 @@ LinearScan::LinearScan(Compiler* theCompiler) // set won't be recomputed until after Lowering (and this constructor is called prior to Lowering), // so we don't want to check that yet. enregisterLocalVars = compiler->compEnregLocals(); + + regSelector = new (theCompiler, CMK_LSRA) RegisterSelection(this); + #ifdef TARGET_ARM64 availableIntRegs = (RBM_ALLINT & ~(RBM_PR | RBM_FP | RBM_LR) & ~compiler->codeGen->regSet.rsMaskResvd); #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) @@ -1395,7 +1407,14 @@ PhaseStatus LinearScan::doLinearScan() else #endif // TARGET_ARM64 { - allocateRegisters(); + if (enregisterLocalVars || compiler->opts.OptimizationEnabled()) + { + allocateRegisters(); + } + else + { + allocateRegistersMinimal(); + } } allocationPassComplete = true; @@ -1898,7 +1917,7 @@ void LinearScan::identifyCandidates() // the same register assignment throughout varDsc->lvRegister = false; - if (!localVarsEnregistered || !isRegCandidate(varDsc)) + if (!isRegCandidate(varDsc)) { varDsc->lvLRACandidate = 0; if (varDsc->lvTracked) @@ -2898,6 +2917,43 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo return false; } +//------------------------------------------------------------------------ +// allocateRegMinimal: Find a register that satisfies the requirements for refPosition, +// taking into account the preferences for the given Interval, +// and possibly spilling a lower weight Interval. +// +// Note: This is a minimal version of `allocateReg()` and used when localVars are not set +// for enregistration. It simply finds the register to be assigned, if it was assigned to something +// else, then will unassign it and then assign to the currentInterval +// +regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, + RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) +{ + assert(!enregisterLocalVars); + regNumber foundReg; + regMaskTP foundRegBit; + RegRecord* availablePhysRegRecord; + foundRegBit = regSelector->selectMinimal(currentInterval, refPosition DEBUG_ARG(registerScore)); + if (foundRegBit == RBM_NONE) + { + return REG_NA; + } + + foundReg = genRegNumFromMask(foundRegBit); + availablePhysRegRecord = getRegisterRecord(foundReg); + Interval* assignedInterval = availablePhysRegRecord->assignedInterval; + if ((assignedInterval != currentInterval) && + isAssigned(availablePhysRegRecord ARM_ARG(getRegisterType(currentInterval, refPosition)))) + { + // For minopts, just unassign `foundReg` from existing interval + unassignPhysReg(availablePhysRegRecord ARM_ARG(currentInterval->registerType)); + } + + assignPhysReg(availablePhysRegRecord, currentInterval); + refPosition->registerAssignment = foundRegBit; + return foundReg; +} + //------------------------------------------------------------------------ // allocateReg: Find a register that satisfies the requirements for refPosition, // taking into account the preferences for the given Interval, @@ -2938,7 +2994,6 @@ regNumber LinearScan::allocateReg(Interval* currentInterval, { regMaskTP foundRegBit = regSelector->select(currentInterval, refPosition DEBUG_ARG(registerScore)); - if (foundRegBit == RBM_NONE) { return REG_NA; @@ -3000,6 +3055,7 @@ regNumber LinearScan::allocateReg(Interval* currentInterval, } } } + assignPhysReg(availablePhysRegRecord, currentInterval); refPosition->registerAssignment = foundRegBit; return foundReg; @@ -3205,6 +3261,58 @@ bool LinearScan::isSpillCandidate(Interval* current, RefPosition* refPosition, R return canSpill; } +// ---------------------------------------------------------- +// assignCopyRegMinimal: Grab a register to use to copy and then immediately use. +// This is called only for localVar intervals that already have a register +// assignment that is not compatible with the current RefPosition. +// This is not like regular assignment, because we don't want to change +// any preferences or existing register assignments. +// +// Arguments: +// refPosition - Refposition within the interval for which register needs to be selected. +// +// Return Values: +// Register bit selected (a single register) and REG_NA if no register was selected. +// +regNumber LinearScan::assignCopyRegMinimal(RefPosition* refPosition) +{ + Interval* currentInterval = refPosition->getInterval(); + assert(currentInterval != nullptr); + assert(currentInterval->isActive); + + // Save the relatedInterval, if any, so that it doesn't get modified during allocation. + Interval* savedRelatedInterval = currentInterval->relatedInterval; + currentInterval->relatedInterval = nullptr; + + // We don't want really want to change the default assignment, + // so 1) pretend this isn't active, and 2) remember the old reg + regNumber oldPhysReg = currentInterval->physReg; + RegRecord* oldRegRecord = currentInterval->assignedReg; + assert(oldRegRecord->regNum == oldPhysReg); + currentInterval->isActive = false; + + // We *must* allocate a register, and it will be a copyReg. Set that field now, so that + // refPosition->RegOptional() will return false. + refPosition->copyReg = true; + + RegisterScore registerScore = NONE; + + regNumber allocatedReg = allocateRegMinimal(currentInterval, refPosition DEBUG_ARG(®isterScore)); + assert(allocatedReg != REG_NA); + + // restore the related interval + currentInterval->relatedInterval = savedRelatedInterval; + + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_COPY_REG, currentInterval, allocatedReg, nullptr, registerScore)); + + // Now restore the old info + currentInterval->physReg = oldPhysReg; + currentInterval->assignedReg = oldRegRecord; + currentInterval->isActive = true; + + return allocatedReg; +} + // Grab a register to use to copy and then immediately use. // This is called only for localVar intervals that already have a register // assignment that is not compatible with the current RefPosition. @@ -3236,9 +3344,9 @@ regNumber LinearScan::assignCopyReg(RefPosition* refPosition) refPosition->copyReg = true; RegisterScore registerScore = NONE; - regNumber allocatedReg = - allocateReg(currentInterval, refPosition DEBUG_ARG(®isterScore)); + regNumber allocatedReg = + allocateReg(currentInterval, refPosition DEBUG_ARG(®isterScore)); assert(allocatedReg != REG_NA); // restore the related interval @@ -3403,6 +3511,12 @@ void LinearScan::setIntervalAsSplit(Interval* interval) // void LinearScan::setIntervalAsSpilled(Interval* interval) { + if (!enregisterLocalVars) + { + interval->isSpilled = true; + return; + } + #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE if (interval->isUpperVector) { @@ -3905,12 +4019,13 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) // Calls processBlockEndLocations() to set the outVarToRegMap, then gets the next block, // and sets the inVarToRegMap appropriately. // +template void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) { assert(currentBlock != nullptr); markBlockVisited(currentBlock); - if (enregisterLocalVars) + if (localVarsEnregistered) { processBlockEndLocations(currentBlock); @@ -4181,17 +4296,17 @@ void LinearScan::resetAllRegistersState() assert(!enregisterLocalVars); // Just clear any constant registers and return. resetAvailableRegs(); + clearAllNextIntervalRef(); + clearAllSpillCost(); + for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { - RegRecord* physRegRecord = getRegisterRecord(reg); - Interval* assignedInterval = physRegRecord->assignedInterval; - clearNextIntervalRef(reg, physRegRecord->registerType); - clearSpillCost(reg, physRegRecord->registerType); - if (assignedInterval != nullptr) - { - assert(assignedInterval->isConstant); - physRegRecord->assignedInterval = nullptr; - } + RegRecord* physRegRecord = getRegisterRecord(reg); +#ifdef DEBUG + Interval* assignedInterval = physRegRecord->assignedInterval; + assert(assignedInterval == nullptr || assignedInterval->isConstant); +#endif + physRegRecord->assignedInterval = nullptr; } } @@ -4214,9 +4329,9 @@ void LinearScan::resetAllRegistersState() // void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) { - // If we have no register candidates we should only call this method during allocation. + // We should only call this method if we have register candidates. - assert(enregisterLocalVars || !allocationPassComplete); + assert(enregisterLocalVars); unsigned predBBNum = blockInfo[currentBlock->bbNum].predBBNum; VarToRegMap predVarToRegMap = getOutVarToRegMap(predBBNum); @@ -4732,73 +4847,33 @@ void LinearScan::freeRegisters(regMaskTP regsToFree) } //------------------------------------------------------------------------ -// LinearScan::allocateRegisters: Perform the actual register allocation by iterating over -// all of the previously constructed Intervals +// LinearScan::allocateRegistersMinimal: Perform the actual register allocation when localVars +// are not enregistered. // -#ifdef TARGET_ARM64 -template -#endif -void LinearScan::allocateRegisters() +void LinearScan::allocateRegistersMinimal() { - JITDUMP("*************** In LinearScan::allocateRegisters()\n"); - DBEXEC(VERBOSE, lsraDumpIntervals("before allocateRegisters")); + assert(!enregisterLocalVars); + JITDUMP("*************** In LinearScan::allocateRegistersMinimal()\n"); + DBEXEC(VERBOSE, lsraDumpIntervals("before allocateRegistersMinimal")); // at start, nothing is active except for register args for (Interval& interval : intervals) { Interval* currentInterval = &interval; currentInterval->recentRefPosition = nullptr; - currentInterval->isActive = false; - if (currentInterval->isLocalVar) - { - LclVarDsc* varDsc = currentInterval->getLocalVar(compiler); - if (varDsc->lvIsRegArg && currentInterval->firstRefPosition != nullptr) - { - currentInterval->isActive = true; - } - } - } - - if (enregisterLocalVars) - { -#if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - VarSetOps::Iter largeVectorVarsIter(compiler, largeVectorVars); - unsigned largeVectorVarIndex = 0; - while (largeVectorVarsIter.NextElem(&largeVectorVarIndex)) - { - Interval* lclVarInterval = getIntervalForLocalVar(largeVectorVarIndex); - lclVarInterval->isPartiallySpilled = false; - } -#endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE + assert(!currentInterval->isActive); } resetRegState(); + clearAllNextIntervalRef(); + clearAllSpillCost(); + for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { RegRecord* physRegRecord = getRegisterRecord(reg); physRegRecord->recentRefPosition = nullptr; updateNextFixedRef(physRegRecord, physRegRecord->firstRefPosition); - - // Is this an incoming arg register? (Note that we don't, currently, consider reassigning - // an incoming arg register as having spill cost.) - Interval* interval = physRegRecord->assignedInterval; - if (interval != nullptr) - { -#ifdef TARGET_ARM - if ((interval->registerType != TYP_DOUBLE) || genIsValidDoubleReg(reg)) -#endif // TARGET_ARM - { - updateNextIntervalRef(reg, interval); - updateSpillCost(reg, interval); - setRegInUse(reg, interval->registerType); - INDEBUG(registersToDump |= getRegMask(reg, interval->registerType)); - } - } - else - { - clearNextIntervalRef(reg, physRegRecord->registerType); - clearSpillCost(reg, physRegRecord->registerType); - } + assert(physRegRecord->assignedInterval == nullptr); } #ifdef DEBUG @@ -4878,13 +4953,15 @@ void LinearScan::allocateRegisters() Interval* currentInterval = nullptr; Referenceable* currentReferent = nullptr; RefType refType = currentRefPosition.refType; + assert(!((refType == RefTypeDummyDef) || (refType == RefTypeParamDef) || (refType == RefTypeZeroInit) || + (refType == RefTypeExpUse))); currentReferent = currentRefPosition.referent; if (spillAlways() && lastAllocatedRefPosition != nullptr && !lastAllocatedRefPosition->IsPhysRegRef() && !lastAllocatedRefPosition->getInterval()->isInternal && (!lastAllocatedRefPosition->RegOptional() || (lastAllocatedRefPosition->registerAssignment != RBM_NONE)) && - (RefTypeIsDef(lastAllocatedRefPosition->refType) || lastAllocatedRefPosition->getInterval()->isLocalVar)) + (RefTypeIsDef(lastAllocatedRefPosition->refType))) { assert(lastAllocatedRefPosition->registerAssignment != RBM_NONE); RegRecord* regRecord = lastAllocatedRefPosition->getInterval()->assignedReg; @@ -4916,12 +4993,6 @@ void LinearScan::allocateRegisters() copyRegsToFree = RBM_NONE; regsInUseThisLocation = regsInUseNextLocation; regsInUseNextLocation = RBM_NONE; -#ifdef TARGET_ARM64 - if (hasConsecutiveRegister) - { - consecutiveRegsInUseThisLocation = RBM_NONE; - } -#endif if ((regsToFree | delayRegsToFree) != RBM_NONE) { freeRegisters(regsToFree); @@ -4939,131 +5010,11 @@ void LinearScan::allocateRegisters() delayRegsToFree = RBM_NONE; #ifdef DEBUG - // Validate the current state just after we've freed the registers. This ensures that any pending - // freed registers will have had their state updated to reflect the intervals they were holding. - for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) - { - regMaskTP regMask = genRegMask(reg); - // If this isn't available or if it's still waiting to be freed (i.e. it was in - // delayRegsToFree and so now it's in regsToFree), then skip it. - if ((regMask & allAvailableRegs & ~regsToFree) == RBM_NONE) - { - continue; - } - RegRecord* physRegRecord = getRegisterRecord(reg); - Interval* assignedInterval = physRegRecord->assignedInterval; - if (assignedInterval != nullptr) - { - bool isAssignedReg = (assignedInterval->physReg == reg); - RefPosition* recentRefPosition = assignedInterval->recentRefPosition; - // If we have a copyReg or a moveReg, we might have assigned this register to an Interval, - // but that isn't considered its assignedReg. - if (recentRefPosition != nullptr) - { - if (recentRefPosition->refType == RefTypeExpUse) - { - // We don't update anything on these, as they're just placeholders to extend the - // lifetime. - continue; - } - // For copyReg or moveReg, we don't have anything further to assert. - if (recentRefPosition->copyReg || recentRefPosition->moveReg) - { - continue; - } - assert(assignedInterval->isConstant == isRegConstant(reg, assignedInterval->registerType)); - if (assignedInterval->isActive) - { - // If this is not the register most recently allocated, it must be from a copyReg, - // it was placed there by the inVarToRegMap or it might be one of the upper vector - // save/restore refPosition. - // In either case it must be a lclVar. - - if (!isAssignedToInterval(assignedInterval, physRegRecord)) - { - // We'd like to assert that this was either set by the inVarToRegMap, or by - // a copyReg, but we can't traverse backward to check for a copyReg, because - // we only have recentRefPosition, and there may be a previous RefPosition - // at the same Location with a copyReg. - - bool sanityCheck = assignedInterval->isLocalVar; - // For upper vector interval, make sure it was one of the save/restore only. - if (assignedInterval->IsUpperVector()) - { - sanityCheck |= (recentRefPosition->refType == RefTypeUpperVectorSave) || - (recentRefPosition->refType == RefTypeUpperVectorRestore); - } - - assert(sanityCheck); - } - if (isAssignedReg) - { - assert(nextIntervalRef[reg] == assignedInterval->getNextRefLocation()); - assert(!isRegAvailable(reg, assignedInterval->registerType)); - assert((recentRefPosition == nullptr) || - (spillCost[reg] == getSpillWeight(physRegRecord))); - } - else - { - assert((nextIntervalRef[reg] == MaxLocation) || - isRegBusy(reg, assignedInterval->registerType)); - } - } - else - { - if ((assignedInterval->physReg == reg) && !assignedInterval->isConstant) - { - assert(nextIntervalRef[reg] == assignedInterval->getNextRefLocation()); - } - else - { - assert(nextIntervalRef[reg] == MaxLocation); - assert(isRegAvailable(reg, assignedInterval->registerType)); - assert(spillCost[reg] == 0); - } - } - } - } - else - { - // Available registers should not hold constants - assert(isRegAvailable(reg, physRegRecord->registerType)); - assert(!isRegConstant(reg, physRegRecord->registerType) || spillAlways()); - assert(nextIntervalRef[reg] == MaxLocation); - assert(spillCost[reg] == 0); - } - LsraLocation thisNextFixedRef = physRegRecord->getNextRefLocation(); - assert(nextFixedRef[reg] == thisNextFixedRef); -#ifdef TARGET_ARM - // If this is occupied by a double interval, skip the corresponding float reg. - if ((assignedInterval != nullptr) && (assignedInterval->registerType == TYP_DOUBLE)) - { - reg = REG_NEXT(reg); - } -#endif - } + verifyFreeRegisters(regsToFree); #endif // DEBUG } } prevLocation = currentLocation; -#ifdef TARGET_ARM64 - -#ifdef DEBUG - if (hasConsecutiveRegister) - { - if (currentRefPosition.needsConsecutive) - { - // track all the refpositions around the location that is also - // allocating consecutive registers. - consecutiveRegistersLocation = currentLocation; - } - else if (consecutiveRegistersLocation < currentLocation) - { - consecutiveRegistersLocation = MinLocation; - } - } -#endif // DEBUG -#endif // TARGET_ARM64 // get previous refposition, then current refpos is the new previous if (currentReferent != nullptr) @@ -5094,7 +5045,7 @@ void LinearScan::allocateRegisters() } #endif // DEBUG - if (!handledBlockEnd && (refType == RefTypeBB || refType == RefTypeDummyDef)) + if (!handledBlockEnd && refType == RefTypeBB) { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); @@ -5110,7 +5061,7 @@ void LinearScan::allocateRegisters() } else { - processBlockEndAllocation(currentBlock); + processBlockEndAllocation(currentBlock); currentBlock = moveToNextBlock(); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_START_BB, nullptr, REG_NA, currentBlock)); } @@ -5158,10 +5109,10 @@ void LinearScan::allocateRegisters() } regsInUseThisLocation |= currentRefPosition.registerAssignment; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); - continue; } - if (refType == RefTypeKill) + else { + assert(refType == RefTypeKill); if (assignedInterval != nullptr) { unassignPhysReg(regRecord, assignedInterval->recentRefPosition); @@ -5170,21 +5121,6 @@ void LinearScan::allocateRegisters() } clearRegBusyUntilKill(regRecord->regNum); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_KEPT_ALLOCATION, nullptr, regRecord->regNum)); - continue; - } - } - - // If this is an exposed use, do nothing - this is merely a placeholder to attempt to - // ensure that a register is allocated for the full lifetime. The resolution logic - // will take care of moving to the appropriate register if needed. - - if (refType == RefTypeExpUse) - { - INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_EXP_USE)); - currentInterval = currentRefPosition.getInterval(); - if (currentInterval->physReg != REG_NA) - { - updateNextIntervalRef(currentInterval->physReg, currentInterval); } continue; } @@ -5194,18 +5130,742 @@ void LinearScan::allocateRegisters() assert(currentRefPosition.isIntervalRef()); currentInterval = currentRefPosition.getInterval(); assert(currentInterval != nullptr); + assert(!currentInterval->isLocalVar); assignedRegister = currentInterval->physReg; // Identify the special cases where we decide up-front not to allocate bool allocate = true; bool didDump = false; - if (refType == RefTypeParamDef || refType == RefTypeZeroInit) +#ifdef FEATURE_SIMD +#if FEATURE_PARTIAL_SIMD_CALLEE_SAVE + if (refType == RefTypeUpperVectorSave) { - if (nextRefPosition == nullptr) - { - // If it has no actual references, mark it as "lastUse"; since they're not actually part - // of any flow they won't have been marked during dataflow. Otherwise, if we allocate a + // Note that this case looks a lot like the case below, but in this case we need to spill + // at the previous RefPosition. + // We may want to consider allocating two callee-save registers for this case, but it happens rarely + // enough that it may not warrant the additional complexity. + if (assignedRegister != REG_NA) + { + RegRecord* regRecord = getRegisterRecord(assignedRegister); + Interval* assignedInterval = regRecord->assignedInterval; + unassignPhysReg(regRecord, currentInterval->firstRefPosition); + if (assignedInterval->isConstant) + { + clearConstantReg(assignedRegister, assignedInterval->registerType); + } + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, currentInterval)); + } + currentRefPosition.registerAssignment = RBM_NONE; + continue; + } +#endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE +#endif // FEATURE_SIMD + + if (assignedRegister == REG_NA && RefTypeIsUse(refType)) + { + currentRefPosition.reload = true; + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_RELOAD, currentInterval, assignedRegister)); + } + + regMaskTP assignedRegBit = RBM_NONE; + bool isInRegister = false; + if (assignedRegister != REG_NA) + { + isInRegister = true; + assignedRegBit = genRegMask(assignedRegister); + if (!currentInterval->isActive) + { + assert(!RefTypeIsUse(refType)); + + currentInterval->isActive = true; + setRegInUse(assignedRegister, currentInterval->registerType); + updateSpillCost(assignedRegister, currentInterval); + updateNextIntervalRef(assignedRegister, currentInterval); + } + assert(currentInterval->assignedReg != nullptr && + currentInterval->assignedReg->regNum == assignedRegister && + currentInterval->assignedReg->assignedInterval == currentInterval); + } + + if (previousRefPosition != nullptr) + { + assert(previousRefPosition->nextRefPosition == ¤tRefPosition); + assert(assignedRegister == REG_NA || assignedRegBit == previousRefPosition->registerAssignment || + currentRefPosition.outOfOrder || previousRefPosition->copyReg); + } + + if (assignedRegister != REG_NA) + { + RegRecord* physRegRecord = getRegisterRecord(assignedRegister); + assert((assignedRegBit == currentRefPosition.registerAssignment) || + (physRegRecord->assignedInterval == currentInterval) || + !isRegInUse(assignedRegister, currentInterval->registerType)); + + if (conflictingFixedRegReference(assignedRegister, ¤tRefPosition)) + { + // We may have already reassigned the register to the conflicting reference. + // If not, we need to unassign this interval. + if (physRegRecord->assignedInterval == currentInterval) + { + unassignPhysRegNoSpill(physRegRecord); + clearConstantReg(assignedRegister, currentInterval->registerType); + } + currentRefPosition.moveReg = true; + assignedRegister = REG_NA; + currentRefPosition.registerAssignment &= ~assignedRegBit; + setIntervalAsSplit(currentInterval); + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_MOVE_REG, currentInterval, assignedRegister)); + } + else if ((genRegMask(assignedRegister) & currentRefPosition.registerAssignment) != 0) + { + currentRefPosition.registerAssignment = assignedRegBit; + if (!currentInterval->isActive) + { + currentRefPosition.reload = true; + } + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_KEPT_ALLOCATION, currentInterval, assignedRegister)); + } + else + { + // It's already in a register, but not one we need. + if (!RefTypeIsDef(currentRefPosition.refType)) + { + regNumber copyReg = assignCopyRegMinimal(¤tRefPosition); + + lastAllocatedRefPosition = ¤tRefPosition; + regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + + // For consecutive register, although it shouldn't matter what the assigned register was, + // because we have just assigned it `copyReg` and that's the one in-use, and not the + // one that was assigned previously. However, in situation where an upper-vector restore + // happened to be restored in assignedReg, we would need assignedReg to stay alive because + // we will copy the entire vector value from it to the `copyReg`. + updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, + &delayRegsToFree DEBUG_ARG(currentInterval) DEBUG_ARG(assignedRegister)); + if (!currentRefPosition.lastUse) + { + copyRegsToFree |= copyRegMask; + } + + // For tree temp (non-localVar) interval, we will need an explicit move. + currentRefPosition.moveReg = true; + currentRefPosition.copyReg = false; + clearNextIntervalRef(copyReg, currentInterval->registerType); + clearSpillCost(copyReg, currentInterval->registerType); + updateNextIntervalRef(assignedRegister, currentInterval); + updateSpillCost(assignedRegister, currentInterval); + continue; + } + else + { + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); + regsToFree |= getRegMask(assignedRegister, currentInterval->registerType); + // We want a new register, but we don't want this to be considered a spill. + assignedRegister = REG_NA; + if (physRegRecord->assignedInterval == currentInterval) + { + unassignPhysRegNoSpill(physRegRecord); + } + } + } + } + + if (assignedRegister == REG_NA) + { + if (currentRefPosition.RegOptional()) + { + // We can avoid allocating a register if it is a last use requiring a reload. + if (currentRefPosition.lastUse && currentRefPosition.reload) + { + allocate = false; + } + +#if FEATURE_PARTIAL_SIMD_CALLEE_SAVE && defined(TARGET_XARCH) + // We can also avoid allocating a register (in fact we don't want to) if we have + // an UpperVectorRestore on xarch where the value is on the stack. + if ((currentRefPosition.refType == RefTypeUpperVectorRestore) && (currentInterval->physReg == REG_NA)) + { + assert(currentRefPosition.regOptional); + allocate = false; + } +#endif + +#ifdef DEBUG + // Under stress mode, don't allocate registers to RegOptional RefPositions. + if (allocate && regOptionalNoAlloc()) + { + allocate = false; + } +#endif + } + + RegisterScore registerScore = NONE; + if (allocate) + { + // Allocate a register, if we must, or if it is profitable to do so. + // If we have a fixed reg requirement, and the interval is inactive in another register, + // unassign that register. + if (currentRefPosition.isFixedRegRef && !currentInterval->isActive && + (currentInterval->assignedReg != nullptr) && + (currentInterval->assignedReg->assignedInterval == currentInterval) && + (genRegMask(currentInterval->assignedReg->regNum) != currentRefPosition.registerAssignment)) + { + unassignPhysReg(currentInterval->assignedReg, nullptr); + } + assignedRegister = allocateRegMinimal(currentInterval, ¤tRefPosition DEBUG_ARG(®isterScore)); + } + + // If no register was found, this RefPosition must not require a register. + if (assignedRegister == REG_NA) + { + assert(currentRefPosition.RegOptional()); + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, currentInterval)); + currentRefPosition.registerAssignment = RBM_NONE; + currentRefPosition.reload = false; + currentInterval->isActive = false; + setIntervalAsSpilled(currentInterval); + } +#ifdef DEBUG + else + { + if (VERBOSE) + { + if (currentInterval->isConstant && (currentRefPosition.treeNode != nullptr) && + currentRefPosition.treeNode->IsReuseRegVal()) + { + dumpLsraAllocationEvent(LSRA_EVENT_REUSE_REG, currentInterval, assignedRegister, currentBlock, + registerScore); + } + else + { + dumpLsraAllocationEvent(LSRA_EVENT_ALLOC_REG, currentInterval, assignedRegister, currentBlock, + registerScore); + } + } + } +#endif // DEBUG + + // If we allocated a register, and this is a use of a spilled value, + // it should have been marked for reload above. + if (assignedRegister != REG_NA && RefTypeIsUse(refType) && !isInRegister) + { + assert(currentRefPosition.reload); + } + } + + // If we allocated a register, record it + if (assignedRegister != REG_NA) + { + assignedRegBit = genRegMask(assignedRegister); + regMaskTP regMask = getRegMask(assignedRegister, currentInterval->registerType); + regsInUseThisLocation |= regMask; + if (currentRefPosition.delayRegFree) + { + regsInUseNextLocation |= regMask; + } + currentRefPosition.registerAssignment = assignedRegBit; + + currentInterval->physReg = assignedRegister; + regsToFree &= ~regMask; // we'll set it again later if it's dead + + // If this interval is dead, free the register. + // The interval could be dead if this is a user variable, or if the + // node is being evaluated for side effects, or a call whose result + // is not used, etc. + // If this is an UpperVector we'll neither free it nor preference it + // (it will be freed when it is used). + bool unassign = false; + if (currentRefPosition.lastUse || (currentRefPosition.nextRefPosition == nullptr)) + { + assert(currentRefPosition.isIntervalRef()); + // If this isn't a final use, we'll mark the register as available, but keep the association. + if ((refType != RefTypeExpUse) && (currentRefPosition.nextRefPosition == nullptr)) + { + unassign = true; + } + else + { + if (currentRefPosition.delayRegFree) + { + delayRegsToMakeInactive |= regMask; + } + else + { + regsToMakeInactive |= regMask; + } + // TODO-Cleanup: this makes things consistent with previous, and will enable preferences + // to be propagated, but it seems less than ideal. + currentInterval->isActive = false; + } + // Update the register preferences for the relatedInterval, if this is 'preferencedToDef'. + // Don't propagate to subsequent relatedIntervals; that will happen as they are allocated, and we + // don't know yet whether the register will be retained. + if (currentInterval->relatedInterval != nullptr) + { + currentInterval->relatedInterval->updateRegisterPreferences(assignedRegBit); + } + } + + if (unassign) + { + if (currentRefPosition.delayRegFree) + { + delayRegsToFree |= regMask; + + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE_DELAYED)); + } + else + { + regsToFree |= regMask; + + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE)); + } + } + if (!unassign) + { + updateNextIntervalRef(assignedRegister, currentInterval); + updateSpillCost(assignedRegister, currentInterval); + } + } + lastAllocatedRefPosition = ¤tRefPosition; + } + + // Free registers to clear associated intervals for resolution phase + CLANG_FORMAT_COMMENT_ANCHOR; + +#ifdef DEBUG + if (getLsraExtendLifeTimes()) + { + // If we have extended lifetimes, we need to make sure all the registers are freed. + for (size_t regNumIndex = 0; regNumIndex <= REG_FP_LAST; regNumIndex++) + { + RegRecord& regRecord = physRegs[regNumIndex]; + Interval* interval = regRecord.assignedInterval; + if (interval != nullptr) + { + interval->isActive = false; + unassignPhysReg(®Record, nullptr); + } + } + } + else +#endif // DEBUG + { + freeRegisters(regsToFree | delayRegsToFree); + } + +#ifdef DEBUG + if (VERBOSE) + { + // Dump the RegRecords after the last RefPosition is handled. + dumpRegRecords(); + printf("\n"); + + dumpRefPositions("AFTER ALLOCATION"); + dumpVarRefPositions("AFTER ALLOCATION"); + + // Dump the intervals that remain active + printf("Active intervals at end of allocation:\n"); + + // We COULD just reuse the intervalIter from above, but ArrayListIterator doesn't + // provide a Reset function (!) - we'll probably replace this so don't bother + // adding it + + for (Interval& interval : intervals) + { + if (interval.isActive) + { + printf("Active "); + interval.dump(this->compiler); + } + } + + printf("\n"); + } +#endif // DEBUG +} + +//------------------------------------------------------------------------ +// LinearScan::allocateRegisters: Perform the actual register allocation by iterating over +// all of the previously constructed Intervals +// +#ifdef TARGET_ARM64 +template +#endif +void LinearScan::allocateRegisters() +{ + JITDUMP("*************** In LinearScan::allocateRegisters()\n"); + DBEXEC(VERBOSE, lsraDumpIntervals("before allocateRegisters")); + + // at start, nothing is active except for register args + for (Interval& interval : intervals) + { + Interval* currentInterval = &interval; + currentInterval->recentRefPosition = nullptr; + currentInterval->isActive = false; + if (currentInterval->isLocalVar) + { + LclVarDsc* varDsc = currentInterval->getLocalVar(compiler); + if (varDsc->lvIsRegArg && currentInterval->firstRefPosition != nullptr) + { + currentInterval->isActive = true; + } + } + } + +#if FEATURE_PARTIAL_SIMD_CALLEE_SAVE + if (enregisterLocalVars) + { + VarSetOps::Iter largeVectorVarsIter(compiler, largeVectorVars); + unsigned largeVectorVarIndex = 0; + while (largeVectorVarsIter.NextElem(&largeVectorVarIndex)) + { + Interval* lclVarInterval = getIntervalForLocalVar(largeVectorVarIndex); + lclVarInterval->isPartiallySpilled = false; + } + } +#endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE + + resetRegState(); + for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) + { + RegRecord* physRegRecord = getRegisterRecord(reg); + physRegRecord->recentRefPosition = nullptr; + updateNextFixedRef(physRegRecord, physRegRecord->firstRefPosition); + + // Is this an incoming arg register? (Note that we don't, currently, consider reassigning + // an incoming arg register as having spill cost.) + Interval* interval = physRegRecord->assignedInterval; + if (interval != nullptr) + { +#ifdef TARGET_ARM + if ((interval->registerType != TYP_DOUBLE) || genIsValidDoubleReg(reg)) +#endif // TARGET_ARM + { + updateNextIntervalRef(reg, interval); + updateSpillCost(reg, interval); + setRegInUse(reg, interval->registerType); + INDEBUG(registersToDump |= getRegMask(reg, interval->registerType)); + } + } + else + { + clearNextIntervalRef(reg, physRegRecord->registerType); + clearSpillCost(reg, physRegRecord->registerType); + } + } + +#ifdef DEBUG + if (VERBOSE) + { + dumpRefPositions("BEFORE ALLOCATION"); + dumpVarRefPositions("BEFORE ALLOCATION"); + + printf("\n\nAllocating Registers\n" + "--------------------\n"); + // Start with a small set of commonly used registers, so that we don't keep having to print a new title. + // Include all the arg regs, as they may already have values assigned to them. + registersToDump = LsraLimitSmallIntSet | LsraLimitSmallFPSet | RBM_ARG_REGS; + dumpRegRecordHeader(); + // Now print an empty "RefPosition", since we complete the dump of the regs at the beginning of the loop. + printf(indentFormat, ""); + } +#endif // DEBUG + + BasicBlock* currentBlock = nullptr; + + LsraLocation prevLocation = MinLocation; + regMaskTP regsToFree = RBM_NONE; + regMaskTP delayRegsToFree = RBM_NONE; + regMaskTP regsToMakeInactive = RBM_NONE; + regMaskTP delayRegsToMakeInactive = RBM_NONE; + regMaskTP copyRegsToFree = RBM_NONE; + regsInUseThisLocation = RBM_NONE; + regsInUseNextLocation = RBM_NONE; + + // This is the most recent RefPosition for which a register was allocated + // - currently only used for DEBUG but maintained in non-debug, for clarity of code + // (and will be optimized away because in non-debug spillAlways() unconditionally returns false) + RefPosition* lastAllocatedRefPosition = nullptr; + + bool handledBlockEnd = false; + + for (RefPosition& currentRefPosition : refPositions) + { + RefPosition* nextRefPosition = currentRefPosition.nextRefPosition; + + // TODO: Can we combine this with the freeing of registers below? It might + // mess with the dump, since this was previously being done before the call below + // to dumpRegRecords. + regMaskTP tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + while (tempRegsToMakeInactive != RBM_NONE) + { + regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); + RegRecord* regRecord = getRegisterRecord(nextReg); + clearSpillCost(regRecord->regNum, regRecord->registerType); + makeRegisterInactive(regRecord); + } + if (currentRefPosition.nodeLocation > prevLocation) + { + makeRegsAvailable(regsToMakeInactive); + // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want + // to mark them as free yet. + regsToMakeInactive |= delayRegsToMakeInactive; + regsToMakeInactive = delayRegsToMakeInactive; + delayRegsToMakeInactive = RBM_NONE; + } + +#ifdef DEBUG + // Set the activeRefPosition to null until we're done with any boundary handling. + activeRefPosition = nullptr; + if (VERBOSE) + { + // We're really dumping the RegRecords "after" the previous RefPosition, but it's more convenient + // to do this here, since there are a number of "continue"s in this loop. + dumpRegRecords(); + } +#endif // DEBUG + + // This is the previousRefPosition of the current Referent, if any + RefPosition* previousRefPosition = nullptr; + + Interval* currentInterval = nullptr; + Referenceable* currentReferent = nullptr; + RefType refType = currentRefPosition.refType; + + currentReferent = currentRefPosition.referent; + + if (spillAlways() && lastAllocatedRefPosition != nullptr && !lastAllocatedRefPosition->IsPhysRegRef() && + !lastAllocatedRefPosition->getInterval()->isInternal && + (!lastAllocatedRefPosition->RegOptional() || (lastAllocatedRefPosition->registerAssignment != RBM_NONE)) && + (RefTypeIsDef(lastAllocatedRefPosition->refType) || lastAllocatedRefPosition->getInterval()->isLocalVar)) + { + assert(lastAllocatedRefPosition->registerAssignment != RBM_NONE); + RegRecord* regRecord = lastAllocatedRefPosition->getInterval()->assignedReg; + + INDEBUG(activeRefPosition = lastAllocatedRefPosition); + unassignPhysReg(regRecord, lastAllocatedRefPosition); + INDEBUG(activeRefPosition = nullptr); + + // Now set lastAllocatedRefPosition to null, so that we don't try to spill it again + lastAllocatedRefPosition = nullptr; + } + + // We wait to free any registers until we've completed all the + // uses for the current node. + // This avoids reusing registers too soon. + // We free before the last true def (after all the uses & internal + // registers), and then again at the beginning of the next node. + // This is made easier by assigning two LsraLocations per node - one + // for all the uses, internal registers & all but the last def, and + // another for the final def (if any). + + LsraLocation currentLocation = currentRefPosition.nodeLocation; + + // Free at a new location. + if (currentLocation > prevLocation) + { + // CopyRegs are simply made available - we don't want to make the associated interval inactive. + makeRegsAvailable(copyRegsToFree); + copyRegsToFree = RBM_NONE; + regsInUseThisLocation = regsInUseNextLocation; + regsInUseNextLocation = RBM_NONE; +#ifdef TARGET_ARM64 + if (hasConsecutiveRegister) + { + consecutiveRegsInUseThisLocation = RBM_NONE; + } +#endif + if ((regsToFree | delayRegsToFree) != RBM_NONE) + { + freeRegisters(regsToFree); + if ((currentLocation > (prevLocation + 1)) && (delayRegsToFree != RBM_NONE)) + { + // We should never see a delayReg that is delayed until a Location that has no RefPosition + // (that would be the RefPosition that it was supposed to interfere with). + assert(!"Found a delayRegFree associated with Location with no reference"); + // However, to be cautious for the Release build case, we will free them. + freeRegisters(delayRegsToFree); + delayRegsToFree = RBM_NONE; + regsInUseThisLocation = RBM_NONE; + } + regsToFree = delayRegsToFree; + delayRegsToFree = RBM_NONE; +#ifdef DEBUG + verifyFreeRegisters(regsToFree); +#endif + } + } + prevLocation = currentLocation; +#ifdef TARGET_ARM64 + +#ifdef DEBUG + if (hasConsecutiveRegister) + { + if (currentRefPosition.needsConsecutive) + { + // track all the refpositions around the location that is also + // allocating consecutive registers. + consecutiveRegistersLocation = currentLocation; + } + else if (consecutiveRegistersLocation < currentLocation) + { + consecutiveRegistersLocation = MinLocation; + } + } +#endif // DEBUG +#endif // TARGET_ARM64 + + // get previous refposition, then current refpos is the new previous + if (currentReferent != nullptr) + { + previousRefPosition = currentReferent->recentRefPosition; + currentReferent->recentRefPosition = ¤tRefPosition; + } + else + { + assert((refType == RefTypeBB) || (refType == RefTypeKillGCRefs)); + } + +#ifdef DEBUG + activeRefPosition = ¤tRefPosition; + + // For the purposes of register resolution, we handle the DummyDefs before + // the block boundary - so the RefTypeBB is after all the DummyDefs. + // However, for the purposes of allocation, we want to handle the block + // boundary first, so that we can free any registers occupied by lclVars + // that aren't live in the next block and make them available for the + // DummyDefs. + + // If we've already handled the BlockEnd, but now we're seeing the RefTypeBB, + // dump it now. + if ((refType == RefTypeBB) && handledBlockEnd) + { + dumpNewBlock(currentBlock, currentRefPosition.nodeLocation); + } +#endif // DEBUG + + if (!handledBlockEnd && (refType == RefTypeBB || refType == RefTypeDummyDef)) + { + // Free any delayed regs (now in regsToFree) before processing the block boundary + freeRegisters(regsToFree); + regsToFree = RBM_NONE; + regsInUseThisLocation = RBM_NONE; + regsInUseNextLocation = RBM_NONE; + handledBlockEnd = true; + curBBStartLocation = currentRefPosition.nodeLocation; + if (currentBlock == nullptr) + { + currentBlock = startBlockSequence(); + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_START_BB, nullptr, REG_NA, compiler->fgFirstBB)); + } + else + { + if (enregisterLocalVars) + { + processBlockEndAllocation(currentBlock); + } + else + { + processBlockEndAllocation(currentBlock); + } + currentBlock = moveToNextBlock(); + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_START_BB, nullptr, REG_NA, currentBlock)); + } + } + + if (refType == RefTypeBB) + { + handledBlockEnd = false; + continue; + } + + if (refType == RefTypeKillGCRefs) + { + spillGCRefs(¤tRefPosition); + continue; + } + + if (currentRefPosition.isPhysRegRef) + { + RegRecord* regRecord = currentRefPosition.getReg(); + Interval* assignedInterval = regRecord->assignedInterval; + + updateNextFixedRef(regRecord, currentRefPosition.nextRefPosition); + + // If this is a FixedReg, disassociate any inactive constant interval from this register. + // Otherwise, do nothing. + if (refType == RefTypeFixedReg) + { + if (assignedInterval != nullptr && !assignedInterval->isActive && assignedInterval->isConstant) + { + clearConstantReg(regRecord->regNum, assignedInterval->registerType); + regRecord->assignedInterval = nullptr; + spillCost[regRecord->regNum] = 0; + +#ifdef TARGET_ARM + // Update overlapping floating point register for TYP_DOUBLE + if (assignedInterval->registerType == TYP_DOUBLE) + { + RegRecord* otherRegRecord = findAnotherHalfRegRec(regRecord); + assert(otherRegRecord->assignedInterval == assignedInterval); + otherRegRecord->assignedInterval = nullptr; + spillCost[otherRegRecord->regNum] = 0; + } +#endif // TARGET_ARM + } + regsInUseThisLocation |= currentRefPosition.registerAssignment; + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); + } + else + { + assert(refType == RefTypeKill); + if (assignedInterval != nullptr) + { + unassignPhysReg(regRecord, assignedInterval->recentRefPosition); + clearConstantReg(regRecord->regNum, assignedInterval->registerType); + makeRegAvailable(regRecord->regNum, assignedInterval->registerType); + } + clearRegBusyUntilKill(regRecord->regNum); + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_KEPT_ALLOCATION, nullptr, regRecord->regNum)); + } + continue; + } + + // If this is an exposed use, do nothing - this is merely a placeholder to attempt to + // ensure that a register is allocated for the full lifetime. The resolution logic + // will take care of moving to the appropriate register if needed. + + if (refType == RefTypeExpUse) + { + INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_EXP_USE)); + currentInterval = currentRefPosition.getInterval(); + if (currentInterval->physReg != REG_NA) + { + updateNextIntervalRef(currentInterval->physReg, currentInterval); + } + continue; + } + + regNumber assignedRegister = REG_NA; + + assert(currentRefPosition.isIntervalRef()); + currentInterval = currentRefPosition.getInterval(); + assert(currentInterval != nullptr); + assignedRegister = currentInterval->physReg; + + // Identify the special cases where we decide up-front not to allocate + bool allocate = true; + bool didDump = false; + + if (refType == RefTypeParamDef || refType == RefTypeZeroInit) + { + if (nextRefPosition == nullptr) + { + // If it has no actual references, mark it as "lastUse"; since they're not actually part + // of any flow they won't have been marked during dataflow. Otherwise, if we allocate a // register we won't unassign it. INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_ZERO_REF, currentInterval)); currentRefPosition.lastUse = true; @@ -5627,6 +6287,7 @@ void LinearScan::allocateRegisters() { regNumber copyReg; #ifdef TARGET_ARM64 + if (hasConsecutiveRegister && currentRefPosition.needsConsecutive && currentRefPosition.refType == RefTypeUse) { @@ -5635,7 +6296,7 @@ void LinearScan::allocateRegisters() else #endif { - copyReg = assignCopyReg(¤tRefPosition); + copyReg = assignCopyReg(¤tRefPosition); } lastAllocatedRefPosition = ¤tRefPosition; @@ -5956,7 +6617,7 @@ void LinearScan::allocateRegisters() // For the JIT32_GCENCODER, when lvaKeepAliveAndReportThis is true, we must either keep the "this" pointer // in the same register for the entire method, or keep it on the stack. Rather than imposing this constraint // as we allocate, we will force all refs to the stack if it is split or spilled. - if (enregisterLocalVars && compiler->lvaKeepAliveAndReportThis()) + if (compiler->lvaKeepAliveAndReportThis()) { LclVarDsc* thisVarDsc = compiler->lvaGetDesc(compiler->info.compThisArg); if (thisVarDsc->lvLRACandidate) @@ -7347,6 +8008,11 @@ void LinearScan::resolveRegisters() // In those cases, treeNode will be nullptr. if (treeNode == nullptr) { + if (localVarsEnregistered) + { + continue; + } + // This is either a use, a dead def, or a field of a struct Interval* interval = currentRefPosition->getInterval(); assert(currentRefPosition->refType == RefTypeUse || @@ -7380,7 +8046,8 @@ void LinearScan::resolveRegisters() { writeRegisters(currentRefPosition, treeNode); - if (treeNode->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && currentRefPosition->getInterval()->isLocalVar) + if (localVarsEnregistered && treeNode->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && + currentRefPosition->getInterval()->isLocalVar) { resolveLocalRef(block, treeNode->AsLclVar(), currentRefPosition); } @@ -10866,19 +11533,130 @@ bool LinearScan::IsResolutionNode(LIR::Range& containingRange, GenTree* node) { if (IsResolutionMove(node)) { - return true; + return true; + } + + if (!IsLsraAdded(node) || (node->OperGet() != GT_LCL_VAR)) + { + return false; + } + + LIR::Use use; + bool foundUse = containingRange.TryGetUse(node, &use); + assert(foundUse); + + node = use.User(); + } +} + +// verifyFreeRegisters: Validate the current state just after we've freed the registers. +// This ensures that any pending freed registers will have had their state updated to +// reflect the intervals they were holding. +// +// Arguments: +// regsToFree - Registers that were just freed. +// +void LinearScan::verifyFreeRegisters(regMaskTP regsToFree) +{ + for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) + { + regMaskTP regMask = genRegMask(reg); + // If this isn't available or if it's still waiting to be freed (i.e. it was in + // delayRegsToFree and so now it's in regsToFree), then skip it. + if ((regMask & allAvailableRegs & ~regsToFree) == RBM_NONE) + { + continue; + } + RegRecord* physRegRecord = getRegisterRecord(reg); + Interval* assignedInterval = physRegRecord->assignedInterval; + if (assignedInterval != nullptr) + { + bool isAssignedReg = (assignedInterval->physReg == reg); + RefPosition* recentRefPosition = assignedInterval->recentRefPosition; + // If we have a copyReg or a moveReg, we might have assigned this register to an Interval, + // but that isn't considered its assignedReg. + if (recentRefPosition != nullptr) + { + if (recentRefPosition->refType == RefTypeExpUse) + { + // We don't update anything on these, as they're just placeholders to extend the + // lifetime. + continue; + } + + // For copyReg or moveReg, we don't have anything further to assert. + if (recentRefPosition->copyReg || recentRefPosition->moveReg) + { + continue; + } + assert(assignedInterval->isConstant == isRegConstant(reg, assignedInterval->registerType)); + if (assignedInterval->isActive) + { + // If this is not the register most recently allocated, it must be from a copyReg, + // it was placed there by the inVarToRegMap or it might be one of the upper vector + // save/restore refPosition. + // In either case it must be a lclVar. + + if (!isAssignedToInterval(assignedInterval, physRegRecord)) + { + // We'd like to assert that this was either set by the inVarToRegMap, or by + // a copyReg, but we can't traverse backward to check for a copyReg, because + // we only have recentRefPosition, and there may be a previous RefPosition + // at the same Location with a copyReg. + + bool sanityCheck = assignedInterval->isLocalVar; + // For upper vector interval, make sure it was one of the save/restore only. + if (assignedInterval->IsUpperVector()) + { + sanityCheck |= (recentRefPosition->refType == RefTypeUpperVectorSave) || + (recentRefPosition->refType == RefTypeUpperVectorRestore); + } + + assert(sanityCheck); + } + if (isAssignedReg) + { + assert(nextIntervalRef[reg] == assignedInterval->getNextRefLocation()); + assert(!isRegAvailable(reg, assignedInterval->registerType)); + assert((recentRefPosition == nullptr) || (spillCost[reg] == getSpillWeight(physRegRecord))); + } + else + { + assert((nextIntervalRef[reg] == MaxLocation) || isRegBusy(reg, assignedInterval->registerType)); + } + } + else + { + if ((assignedInterval->physReg == reg) && !assignedInterval->isConstant) + { + assert(nextIntervalRef[reg] == assignedInterval->getNextRefLocation()); + } + else + { + assert(nextIntervalRef[reg] == MaxLocation); + assert(isRegAvailable(reg, assignedInterval->registerType)); + assert(spillCost[reg] == 0); + } + } + } + } + else + { + // Available registers should not hold constants + assert(isRegAvailable(reg, physRegRecord->registerType)); + assert(!isRegConstant(reg, physRegRecord->registerType) || spillAlways()); + assert(nextIntervalRef[reg] == MaxLocation); + assert(spillCost[reg] == 0); } - - if (!IsLsraAdded(node) || (node->OperGet() != GT_LCL_VAR)) + LsraLocation thisNextFixedRef = physRegRecord->getNextRefLocation(); + assert(nextFixedRef[reg] == thisNextFixedRef); +#ifdef TARGET_ARM + // If this is occupied by a double interval, skip the corresponding float reg. + if ((assignedInterval != nullptr) && (assignedInterval->registerType == TYP_DOUBLE)) { - return false; + reg = REG_NEXT(reg); } - - LIR::Use use; - bool foundUse = containingRange.TryGetUse(node, &use); - assert(foundUse); - - node = use.User(); +#endif } } @@ -11518,6 +12296,11 @@ LinearScan::RegisterSelection::RegisterSelection(LinearScan* linearScan) if (ordering == nullptr) { ordering = W("ABCDEFGHIJKLMNOPQ"); + + if (!linearScan->enregisterLocalVars && linearScan->compiler->opts.OptimizationDisabled()) + { + ordering = W("MQQQQQQQQQQQQQQQQ"); + } } for (int orderId = 0; orderId < REGSELECT_HEURISTIC_COUNT; orderId++) @@ -11540,6 +12323,19 @@ LinearScan::RegisterSelection::RegisterSelection(LinearScan* linearScan) #endif // DEBUG } +// ---------------------------------------------------------- +// reset: Resets the values of all the fields used for register selection. +// +void LinearScan::RegisterSelection::resetMinimal(Interval* interval, RefPosition* refPos) +{ + currentInterval = interval; + refPosition = refPos; + + regType = linearScan->getRegisterType(currentInterval, refPosition); + candidates = refPosition->registerAssignment; + found = false; +} + // ---------------------------------------------------------- // reset: Resets the values of all the fields used for register selection. // @@ -11630,10 +12426,16 @@ bool LinearScan::RegisterSelection::applySingleRegSelection(int selectionScore, void LinearScan::RegisterSelection::try_FREE() { assert(!found); +#ifdef DEBUG + // We try the "free" heuristics only if we have free candidates and this is + // guarded by a check in Release mode. + // In Debug mode, JitLsraOrdering can reorder the heuristics and we can come + // here in any order, hence we should check if it is RBM_NONE or not. if (freeCandidates == RBM_NONE) { return; } +#endif found = applySelection(FREE, freeCandidates); } @@ -11646,10 +12448,13 @@ void LinearScan::RegisterSelection::try_FREE() void LinearScan::RegisterSelection::try_CONST_AVAILABLE() { assert(!found); +#ifdef DEBUG + // See comments in try_FREE if (freeCandidates == RBM_NONE) { return; } +#endif if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) { @@ -11669,10 +12474,13 @@ void LinearScan::RegisterSelection::try_CONST_AVAILABLE() void LinearScan::RegisterSelection::try_THIS_ASSIGNED() { assert(!found); +#ifdef DEBUG + // See comments in try_FREE if (freeCandidates == RBM_NONE) { return; } +#endif if (currentInterval->assignedReg != nullptr) { @@ -11687,6 +12495,14 @@ void LinearScan::RegisterSelection::try_COVERS() { assert(!found); +#ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } +#endif + calculateCoversSets(); found = applySelection(COVERS, coversSet & preferenceSet); @@ -11702,6 +12518,11 @@ void LinearScan::RegisterSelection::try_OWN_PREFERENCE() assert(!found); #ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } calculateCoversSets(); #endif @@ -11716,6 +12537,11 @@ void LinearScan::RegisterSelection::try_COVERS_RELATED() assert(!found); #ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } calculateCoversSets(); #endif @@ -11729,6 +12555,14 @@ void LinearScan::RegisterSelection::try_RELATED_PREFERENCE() { assert(!found); +#ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } +#endif + found = applySelection(RELATED_PREFERENCE, relatedPreferences & freeCandidates); } @@ -11739,6 +12573,14 @@ void LinearScan::RegisterSelection::try_CALLER_CALLEE() { assert(!found); +#ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } +#endif + found = applySelection(CALLER_CALLEE, callerCalleePrefs & freeCandidates); } @@ -11750,6 +12592,11 @@ void LinearScan::RegisterSelection::try_UNASSIGNED() assert(!found); #ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } calculateCoversSets(); #endif @@ -11764,6 +12611,11 @@ void LinearScan::RegisterSelection::try_COVERS_FULL() assert(!found); #ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } calculateCoversSets(); #endif @@ -11783,10 +12635,13 @@ void LinearScan::RegisterSelection::try_BEST_FIT() { assert(!found); +#ifdef DEBUG + // See comments in try_FREE if (freeCandidates == RBM_NONE) { return; } +#endif regMaskTP bestFitSet = RBM_NONE; // If the best score includes COVERS_FULL, pick the one that's killed soonest. @@ -11858,6 +12713,13 @@ void LinearScan::RegisterSelection::try_BEST_FIT() // void LinearScan::RegisterSelection::try_IS_PREV_REG() { +#ifdef DEBUG + // See comments in try_FREE + if (freeCandidates == RBM_NONE) + { + return; + } +#endif // TODO: We do not check found here. if ((currentInterval->assignedReg != nullptr) && coversFullApplied) { @@ -11872,10 +12734,13 @@ void LinearScan::RegisterSelection::try_REG_ORDER() { assert(!found); +#ifdef DEBUG + // See comments in try_FREE if (freeCandidates == RBM_NONE) { return; } +#endif // This will always result in a single candidate. That is, it is the tie-breaker // for free candidates, and doesn't make sense as anything other than the last @@ -12168,6 +13033,32 @@ void LinearScan::RegisterSelection::try_REG_NUM() found = applySingleRegSelection(REG_NUM, genFindLowestBit(candidates)); } +// ---------------------------------------------------------- +// calculateUnassignedSets: Calculate the necessary unassigned sets. +// +void LinearScan::RegisterSelection::calculateUnassignedSets() // TODO: Seperate the calculation of unassigned set +{ + if (freeCandidates == RBM_NONE || coversSetsCalculated) + { + return; + } + + regMaskTP coversCandidates = candidates; + for (; coversCandidates != RBM_NONE;) + { + regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); + regMaskTP coversCandidateBit = genRegMask(coversCandidateRegNum); + coversCandidates ^= coversCandidateBit; + + // The register is considered unassigned if it has no assignedInterval, OR + // if its next reference is beyond the range of this interval. + if (linearScan->nextIntervalRef[coversCandidateRegNum] > lastLocation) + { + unassignedSet |= coversCandidateBit; + } + } +} + // ---------------------------------------------------------- // calculateCoversSets: Calculate the necessary covers set registers to be used // for heuristics lke COVERS, COVERS_RELATED, COVERS_FULL. @@ -12699,9 +13590,201 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, return RBM_NONE; } - calculateCoversSets(); + calculateUnassignedSets(); assert(found && isSingleRegister(candidates)); foundRegBit = candidates; return candidates; } + +// ---------------------------------------------------------- +// selectMinimal: For given `currentInterval` and `refPosition`, selects a register to be assigned. +// +// Arguments: +// currentInterval - Current interval for which register needs to be selected. +// refPosition - Refposition within the interval for which register needs to be selected. +// +// Return Values: +// Register bit selected (a single register) and REG_NA if no register was selected. +// +// Note - This routine just eliminates the busy candidates and the ones that has conflicting use and +// select the REG_ORDER heuristics (if there are any free candidates) or REG_NUM (if all registers +// are busy). +// +regMaskTP LinearScan::RegisterSelection::selectMinimal(Interval* currentInterval, + RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) +{ + assert(!linearScan->enregisterLocalVars); +#ifdef DEBUG + *registerScore = NONE; +#endif + +#ifdef TARGET_ARM64 + assert(!refPosition->needsConsecutive); +#endif + + resetMinimal(currentInterval, refPosition); + + // process data-structures + if (RefTypeIsDef(refPosition->refType)) + { + RefPosition* nextRefPos = refPosition->nextRefPosition; + if (currentInterval->hasConflictingDefUse) + { + linearScan->resolveConflictingDefAndUse(currentInterval, refPosition); + candidates = refPosition->registerAssignment; + } + // Otherwise, check for the case of a fixed-reg def of a reg that will be killed before the + // use, or interferes at the point of use (which shouldn't happen, but Lower doesn't mark + // the contained nodes as interfering). + // Note that we may have a ParamDef RefPosition that is marked isFixedRegRef, but which + // has had its registerAssignment changed to no longer be a single register. + else if (refPosition->isFixedRegRef && nextRefPos != nullptr && RefTypeIsUse(nextRefPos->refType) && + !nextRefPos->isFixedRegRef && genMaxOneBit(refPosition->registerAssignment)) + { + regNumber defReg = refPosition->assignedReg(); + RegRecord* defRegRecord = linearScan->getRegisterRecord(defReg); + + RefPosition* currFixedRegRefPosition = defRegRecord->recentRefPosition; + assert(currFixedRegRefPosition != nullptr && + currFixedRegRefPosition->nodeLocation == refPosition->nodeLocation); + + // If there is another fixed reference to this register before the use, change the candidates + // on this RefPosition to include that of nextRefPos. + RefPosition* nextFixedRegRefPosition = defRegRecord->getNextRefPosition(); + if (nextFixedRegRefPosition != nullptr && + nextFixedRegRefPosition->nodeLocation <= nextRefPos->getRefEndLocation()) + { + candidates |= nextRefPos->registerAssignment; + } + } + } + +#ifdef DEBUG + candidates = linearScan->stressLimitRegs(refPosition, candidates); +#endif + assert(candidates != RBM_NONE); + + bool avoidByteRegs = false; +#ifdef TARGET_X86 + if ((relatedPreferences & ~RBM_BYTE_REGS) != RBM_NONE) + { + avoidByteRegs = true; + } +#endif + + // Is this a fixedReg? + regMaskTP fixedRegMask = RBM_NONE; + if (refPosition->isFixedRegRef) + { + assert(genMaxOneBit(refPosition->registerAssignment)); + if (candidates == refPosition->registerAssignment) + { + found = true; + foundRegBit = candidates; + return candidates; + } + fixedRegMask = refPosition->registerAssignment; + } + + // Eliminate candidates that are in-use or busy. + // TODO-CQ: We assign same registerAssignment to UPPER_RESTORE and the next USE. + // When we allocate for USE, we see that the register is busy at current location + // and we end up with that candidate is no longer available. + regMaskTP busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; + candidates &= ~busyRegs; + +#ifdef TARGET_ARM + // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half + // is also available. Thus, for every busyReg that is an odd floating-point register, we need to + // remove from candidates the corresponding even floating-point register. For example, if busyRegs + // contains `f3`, we need to remove `f2` from the candidates for a double register interval. The + // clause below creates a mask to do this. + if (currentInterval->registerType == TYP_DOUBLE) + { + candidates &= ~((busyRegs & RBM_ALLDOUBLE_HIGH) >> 1); + } +#endif // TARGET_ARM + + // Also eliminate as busy any register with a conflicting fixed reference at this or + // the next location. + // Note that this will eliminate the fixedReg, if any, but we'll add it back below. + regMaskTP checkConflictMask = candidates & linearScan->fixedRegs; + while (checkConflictMask != RBM_NONE) + { + regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); + regMaskTP checkConflictBit = genRegMask(checkConflictReg); + checkConflictMask ^= checkConflictBit; + + LsraLocation checkConflictLocation = linearScan->nextFixedRef[checkConflictReg]; + + if ((checkConflictLocation == refPosition->nodeLocation) || + (refPosition->delayRegFree && (checkConflictLocation == (refPosition->nodeLocation + 1)))) + { + candidates &= ~checkConflictBit; + } + } + candidates |= fixedRegMask; + found = isSingleRegister(candidates); + + // TODO-Cleanup: Previously, the "reverseSelect" stress mode reversed the order of the heuristics. + // It needs to be re-engineered with this refactoring. + // In non-debug builds, this will simply get optimized away + bool reverseSelect = false; +#ifdef DEBUG + reverseSelect = linearScan->doReverseSelect(); +#endif // DEBUG + + if (!found) + { + if (candidates == RBM_NONE) + { + assert(refPosition->RegOptional()); + currentInterval->assignedReg = nullptr; + return RBM_NONE; + } + + freeCandidates = linearScan->getFreeCandidates(candidates ARM_ARG(regType)); + + if (freeCandidates != RBM_NONE) + { + candidates = freeCandidates; + + try_REG_ORDER(); + } + +#ifdef DEBUG +#if TRACK_LSRA_STATS + INTRACK_STATS_IF(found, linearScan->updateLsraStat(linearScan->getLsraStatFromScore(RegisterScore::REG_ORDER), + refPosition->bbNum)); +#endif // TRACK_LSRA_STATS + if (found) + { + *registerScore = RegisterScore::REG_ORDER; + } +#endif // DEBUG + } + if (!found) + { + if (refPosition->RegOptional() || !refPosition->IsActualRef()) + { + // We won't spill if this refPosition is not an actual ref or is RegOptional + currentInterval->assignedReg = nullptr; + return RBM_NONE; + } + else + { + try_REG_NUM(); +#ifdef DEBUG +#if TRACK_LSRA_STATS + INTRACK_STATS_IF(found, linearScan->updateLsraStat(linearScan->getLsraStatFromScore(RegisterScore::REG_NUM), + refPosition->bbNum)); +#endif // TRACK_LSRA_STATS + *registerScore = RegisterScore::REG_NUM; +#endif // DEBUG + } + } + + assert(found && isSingleRegister(candidates)); + return candidates; +} diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index a708662882a06..c0e0f5d2fdbd3 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -637,12 +637,15 @@ class LinearScan : public LinearScanInterface template void buildIntervals(); + // This is where the actual assignment is done for scenarios where + // no local var enregistration is done. + void allocateRegistersMinimal(); + // This is where the actual assignment is done #ifdef TARGET_ARM64 template #endif void allocateRegisters(); - // This is the resolution phase, where cross-block mismatches are fixed up template void resolveRegisters(); @@ -937,6 +940,7 @@ class LinearScan : public LinearScanInterface static bool IsResolutionMove(GenTree* node); static bool IsResolutionNode(LIR::Range& containingRange, GenTree* node); + void verifyFreeRegisters(regMaskTP regsToFree); void verifyFinalAllocation(); void verifyResolutionMove(GenTree* resolutionNode, LsraLocation currentLocation); #else // !DEBUG @@ -1001,6 +1005,7 @@ class LinearScan : public LinearScanInterface // Update allocations at start/end of block void unassignIntervalBlockStart(RegRecord* regRecord, VarToRegMap inVarToRegMap); + template void processBlockEndAllocation(BasicBlock* current); // Record variable locations at start/end of block @@ -1158,13 +1163,6 @@ class LinearScan : public LinearScanInterface regMaskTP mask, unsigned multiRegIdx = 0); - // This creates a RefTypeUse at currentLoc. It sets the treeNode to nullptr if it is not a - // lclVar interval. - RefPosition* newUseRefPosition(Interval* theInterval, - GenTree* theTreeNode, - regMaskTP mask, - unsigned multiRegIdx = 0); - RefPosition* newRefPosition( regNumber reg, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, regMaskTP mask); @@ -1186,8 +1184,10 @@ class LinearScan : public LinearScanInterface #endif template regNumber allocateReg(Interval* current, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); + regNumber allocateRegMinimal(Interval* current, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); template regNumber assignCopyReg(RefPosition* refPosition); + regNumber assignCopyRegMinimal(RefPosition* refPosition); bool isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPosition); bool isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord); @@ -1261,6 +1261,9 @@ class LinearScan : public LinearScanInterface FORCEINLINE regMaskTP select(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); + FORCEINLINE regMaskTP selectMinimal(Interval* currentInterval, + RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); + // If the register is from unassigned set such that it was not already // assigned to the current interval FORCEINLINE bool foundUnassignedReg() @@ -1344,7 +1347,9 @@ class LinearScan : public LinearScanInterface bool applySelection(int selectionScore, regMaskTP selectionCandidates); bool applySingleRegSelection(int selectionScore, regMaskTP selectionCandidate); FORCEINLINE void calculateCoversSets(); + FORCEINLINE void calculateUnassignedSets(); FORCEINLINE void reset(Interval* interval, RefPosition* refPosition); + FORCEINLINE void resetMinimal(Interval* interval, RefPosition* refPosition); #define REG_SEL_DEF(stat, value, shortname, orderSeqId) FORCEINLINE void try_##stat(); #define BUSY_REG_SEL_DEF(stat, value, shortname, orderSeqId) REG_SEL_DEF(stat, value, shortname, orderSeqId) @@ -1777,9 +1782,11 @@ class LinearScan : public LinearScanInterface makeRegsAvailable(regMask); } + void clearAllNextIntervalRef(); void clearNextIntervalRef(regNumber reg, var_types regType); void updateNextIntervalRef(regNumber reg, Interval* interval); + void clearAllSpillCost(); void clearSpillCost(regNumber reg, var_types regType); void updateSpillCost(regNumber reg, Interval* interval); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 824555ed1840e..0c1d3f74475c8 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -185,6 +185,11 @@ RefPosition* LinearScan::newRefPositionRaw(LsraLocation nodeLocation, GenTree* t // the allocation table. currBuildNode = nullptr; newRP->rpNum = static_cast(refPositions.size() - 1); + if (!enregisterLocalVars) + { + assert(!((refType == RefTypeParamDef) || (refType == RefTypeZeroInit) || (refType == RefTypeDummyDef) || + (refType == RefTypeExpUse))); + } #endif // DEBUG return newRP; } @@ -645,41 +650,6 @@ RefPosition* LinearScan::newRefPosition(Interval* theInterval, return newRP; } -//--------------------------------------------------------------------------- -// newUseRefPosition: allocate and initialize a RefTypeUse RefPosition at currentLoc. -// -// Arguments: -// theInterval - interval to which RefPosition is associated with. -// theTreeNode - GenTree node for which this RefPosition is created -// mask - Set of valid registers for this RefPosition -// multiRegIdx - register position if this RefPosition corresponds to a -// multi-reg call node. -// minRegCount - Minimum number registers that needs to be ensured while -// constraining candidates for this ref position under -// LSRA stress. This is a DEBUG only arg. -// -// Return Value: -// a new RefPosition -// -// Notes: -// If the caller knows that 'theTreeNode' is NOT a candidate local, newRefPosition -// can/should be called directly. -// -RefPosition* LinearScan::newUseRefPosition(Interval* theInterval, - GenTree* theTreeNode, - regMaskTP mask, - unsigned multiRegIdx) -{ - GenTree* treeNode = isCandidateLocalRef(theTreeNode) ? theTreeNode : nullptr; - - RefPosition* pos = newRefPosition(theInterval, currentLoc, RefTypeUse, treeNode, mask, multiRegIdx); - if (theTreeNode->IsRegOptional()) - { - pos->setRegOptional(true); - } - return pos; -} - //------------------------------------------------------------------------ // IsContainableMemoryOp: Checks whether this is a memory op that can be contained. //