From 8dec7cf95478273269ecbc2062b7a3254b485d5e Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Sat, 12 Jul 2025 01:40:59 +0800 Subject: [PATCH] [VPlan] Allow generating vectors with VPInstruction::ptradd. NFC Currently a ptradd can only generate a scalar, or a series of scalars per-lane. In an upcoming patch to expand VPWidenPointerRecipe into smaller recipes, we need to be able to generate a vector ptradd, which currently we can't do. This adds support for generating vectors by checking to see if the offset operand is a vector: If it isn't, it will generate per-lane scalars as per usual. --- llvm/lib/Transforms/Vectorize/VPlan.h | 8 +++++--- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 14 +++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 9a6e4b36397b3..0d9af0210a393 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -958,8 +958,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, ExtractPenultimateElement, LogicalAnd, // Non-poison propagating logical And. // Add an offset in bytes (second operand) to a base pointer (first - // operand). Only generates scalar values (either for the first lane only or - // for all lanes, depending on its uses). + // operand). The base pointer must be scalar, but the offset can be a + // scalar, multiple scalars, or a vector. If the offset is multiple scalars + // then it will generate multiple scalar values (either for the first lane + // only or for all lanes, depending on its uses). PtrAdd, // Returns a scalar boolean value, which is true if any lane of its // (boolean) vector operands is true. It produces the reduced value across @@ -998,7 +1000,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, /// values per all lanes, stemming from an original ingredient. This method /// identifies the (rare) cases of VPInstructions that do so as well, w/o an /// underlying ingredient. - bool doesGeneratePerAllLanes() const; + bool doesGeneratePerAllLanes(VPTransformState &State) const; /// Returns true if we can generate a scalar for the first lane only if /// needed. diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 75ade13b09d9c..4b7d21edbb48a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -494,8 +494,9 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) { } #endif -bool VPInstruction::doesGeneratePerAllLanes() const { - return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this); +bool VPInstruction::doesGeneratePerAllLanes(VPTransformState &State) const { + return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this) && + !State.hasVectorValue(getOperand(1)); } bool VPInstruction::canGenerateScalarForFirstLane() const { @@ -848,10 +849,8 @@ Value *VPInstruction::generate(VPTransformState &State) { return Builder.CreateLogicalAnd(A, B, Name); } case VPInstruction::PtrAdd: { - assert(vputils::onlyFirstLaneUsed(this) && - "can only generate first lane for PtrAdd"); Value *Ptr = State.get(getOperand(0), VPLane(0)); - Value *Addend = State.get(getOperand(1), VPLane(0)); + Value *Addend = State.get(getOperand(1), vputils::onlyFirstLaneUsed(this)); return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags()); } case VPInstruction::AnyOf: { @@ -911,9 +910,6 @@ InstructionCost VPInstruction::computeCost(ElementCount VF, } } - assert(!doesGeneratePerAllLanes() && - "Should only generate a vector value or single scalar, not scalars " - "for all lanes."); return Ctx.TTI.getArithmeticInstrCost(getOpcode(), ResTy, Ctx.CostKind); } @@ -1001,7 +997,7 @@ void VPInstruction::execute(VPTransformState &State) { bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() && (vputils::onlyFirstLaneUsed(this) || isVectorToScalar() || isSingleScalar()); - bool GeneratesPerAllLanes = doesGeneratePerAllLanes(); + bool GeneratesPerAllLanes = doesGeneratePerAllLanes(State); if (GeneratesPerAllLanes) { for (unsigned Lane = 0, NumLanes = State.VF.getFixedValue(); Lane != NumLanes; ++Lane) {