Skip to content

Commit

Permalink
[LLVM][XTHeadVector] Implement intrinsic for unit-stride segment load…
Browse files Browse the repository at this point in the history 
…s and stores. (llvm#37)

* [LLVM][RVV 0.7.1] Rename elements in search
table to match T-Head suggested naming scheme.

* [LLVM][XTHeadVector] Define intrinsic functions
for unit-strided segment load.

* [LLVM][XTHeadVector] Define search table for vlseg
intrinsic.

* [LLVM][XTHeadVector] Define pseudo nodes for vlseg
intrinsic.

* [LLVM][XTHeadVector] Expand vlseg load pseudo
nodes.

* [LLVM][XTHeadVector] Set load/store info for newly
added intrinsic.

* [LLVM][XTHeadVector] Add tests for new intrinsic.

* [LLVM][XTHeadVector] Define intrinsic for segment
store.

* [LLVM][XTHeadVector] Define search table for seg
store intrinsic.

* [LLVM][XTHeadVector] Define pseudo nodes for
segment store intrinsic.

* [LLVM][XTHeadVector] Select segment store pseudos.

* [LLVM][XTHeadVector] Set load and store info for
segment store intrinsic.

* [LLVM][XTHeadVector] Add tests.
  • Loading branch information
@AinsleySnow @imkiva
AinsleySnow authored and imkiva committed Apr 1, 2024
1 parent 7be347c commit 6a4f6ff
Show file tree
Hide file tree
Showing 13 changed files with 19,739 additions and 10 deletions.
69 changes: 69 additions & 0 deletions llvm/include/llvm/IR/IntrinsicsRISCVXTHeadV.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,75 @@ let TargetPrefix = "riscv" in {

def int_riscv_th_vleff : XVUSLoadFF;
def int_riscv_th_vleff_mask : XVUSLoadFFMasked;

// 7.8.1 Vector Unit-Stride Segment Loads and Stores
// For unit stride segment load
class XVUSSegLoad<int nf>
: DefaultAttrsIntrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>,
!add(nf, -1))),
!listconcat(!listsplat(LLVMMatchType<0>, nf),
[llvm_ptr_ty, llvm_anyint_ty]),
[NoCapture<ArgIndex<nf>>, IntrReadMem]>, RISCVVIntrinsic {
let VLOperand = !add(nf, 1);
}
// For unit stride segment load with mask
class XVUSSegLoadMasked<int nf>
: DefaultAttrsIntrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>,
!add(nf, -1))),
!listconcat(!listsplat(LLVMMatchType<0>, nf),
[llvm_ptr_ty,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_anyint_ty]),
[NoCapture<ArgIndex<nf>>, IntrReadMem]>,
RISCVVIntrinsic {
let VLOperand = !add(nf, 2);
}

// For unit stride segment store
class XVUSSegStore<int nf>
: DefaultAttrsIntrinsic<[],
!listconcat([llvm_anyvector_ty],
!listsplat(LLVMMatchType<0>, !add(nf, -1)),
[llvm_ptr_ty, llvm_anyint_ty]),
[NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic {
let VLOperand = !add(nf, 1);
}
// For unit stride masked segment store
class XVUSSegStoreMasked<int nf>
: DefaultAttrsIntrinsic<[],
!listconcat([llvm_anyvector_ty],
!listsplat(LLVMMatchType<0>, !add(nf, -1)),
[llvm_ptr_ty,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_anyint_ty]),
[NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic {
let VLOperand = !add(nf, 2);
}

multiclass XVUSSegLoad<int nf> {
def "int_riscv_th_" # NAME : XVUSSegLoad<nf>;
def "int_riscv_th_" # NAME # "_mask" : XVUSSegLoadMasked<nf>;
}

multiclass XVUSSegStore<int nf> {
def "int_riscv_th_" # NAME : XVUSSegStore<nf>;
def "int_riscv_th_" # NAME # "_mask" : XVUSSegStoreMasked<nf>;
}

foreach nf = [2, 3, 4, 5, 6, 7, 8] in {
defm "vlseg" # nf # "b" : XVUSSegLoad<nf>;
defm "vlseg" # nf # "bu" : XVUSSegLoad<nf>;
defm "vlseg" # nf # "h" : XVUSSegLoad<nf>;
defm "vlseg" # nf # "hu" : XVUSSegLoad<nf>;
defm "vlseg" # nf # "w" : XVUSSegLoad<nf>;
defm "vlseg" # nf # "wu" : XVUSSegLoad<nf>;
defm "vlseg" # nf # "e" : XVUSSegLoad<nf>;

defm "vsseg" # nf # "b" : XVUSSegStore<nf>;
defm "vsseg" # nf # "h" : XVUSSegStore<nf>;
defm "vsseg" # nf # "w" : XVUSSegStore<nf>;
defm "vsseg" # nf # "e" : XVUSSegStore<nf>;
}
} // TargetPrefix = "riscv"

let TargetPrefix = "riscv" in {
Expand Down
199 changes: 195 additions & 4 deletions llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ namespace llvm::RISCV {
#define GET_RISCVMaskedPseudosTable_IMPL
#define GET_XTHeadVVLTable_IMPL
#define GET_XTHeadVVSTable_IMPL
#define GET_XTHeadVVLSEGTable_IMPL
#define GET_XTHeadVVSSEGTable_IMPL
#include "RISCVGenSearchableTables.inc"
} // namespace llvm::RISCV

Expand Down Expand Up @@ -422,8 +424,8 @@ void RISCVDAGToDAGISel::selectXVL(
}

unsigned Log2MEM = IsE ? Log2SEW : Tag2Log2MEM(IntNo);
const RISCV::XVLPseudo *P =
RISCV::getXVLPseudo(IsMasked, IsStrided, IsIndexed, IsFF, IsUnsigned, IsE,
const RISCV::TH_VLPseudo *P =
RISCV::getTH_VLPseudo(IsMasked, IsStrided, IsIndexed, IsFF, IsUnsigned, IsE,
Log2MEM, Log2SEW, static_cast<unsigned>(LMUL));
MachineSDNode *Load =
CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
Expand Down Expand Up @@ -488,8 +490,8 @@ void RISCVDAGToDAGISel::selectXVS(
}

unsigned Log2MEM = IsE ? Log2SEW : Tag2Log2MEM(IntNo);
const RISCV::XVSPseudo *P =
RISCV::getXVSPseudo(IsMasked, IsStrided, IsIndexed, IsE,
const RISCV::TH_VSPseudo *P =
RISCV::getTH_VSPseudo(IsMasked, IsStrided, IsIndexed, IsE,
Log2MEM, Log2SEW, static_cast<unsigned>(LMUL));
MachineSDNode *Store =
CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
Expand All @@ -501,6 +503,137 @@ void RISCVDAGToDAGISel::selectXVS(
return;
}

#define CASE_TAG_TO_THVLSEG_INTRINSIC(tag) \
case Intrinsic::riscv_th_vlseg2##tag: \
case Intrinsic::riscv_th_vlseg3##tag: \
case Intrinsic::riscv_th_vlseg4##tag: \
case Intrinsic::riscv_th_vlseg5##tag: \
case Intrinsic::riscv_th_vlseg6##tag: \
case Intrinsic::riscv_th_vlseg7##tag: \
case Intrinsic::riscv_th_vlseg8##tag:

void RISCVDAGToDAGISel::selectXVLSEG(SDNode *Node, unsigned IntNo, bool IsMasked,
bool IsUnsigned, bool IsE) {
auto Tag2Log2MEM = [] (unsigned no) {
switch (no) {
CASE_TAG_TO_THVLSEG_INTRINSIC(b)
CASE_TAG_TO_THVLSEG_INTRINSIC(bu)
CASE_TAG_TO_THVLSEG_INTRINSIC(b_mask)
CASE_TAG_TO_THVLSEG_INTRINSIC(bu_mask)
return 3;
CASE_TAG_TO_THVLSEG_INTRINSIC(h)
CASE_TAG_TO_THVLSEG_INTRINSIC(hu)
CASE_TAG_TO_THVLSEG_INTRINSIC(h_mask)
CASE_TAG_TO_THVLSEG_INTRINSIC(hu_mask)
return 4;
CASE_TAG_TO_THVLSEG_INTRINSIC(w)
CASE_TAG_TO_THVLSEG_INTRINSIC(wu)
CASE_TAG_TO_THVLSEG_INTRINSIC(w_mask)
CASE_TAG_TO_THVLSEG_INTRINSIC(wu_mask)
return 5;
}
llvm_unreachable("Impossible intrinsic tag");
};

SDLoc DL(Node);
unsigned NF = Node->getNumValues() - 1;

MVT VT = Node->getSimpleValueType(0);
unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);

unsigned CurOp = 2;
SmallVector<SDValue, 8> Operands;

SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
Node->op_begin() + CurOp + NF);
SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL);
Operands.push_back(Merge);
CurOp += NF;

addXTHeadVLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
/*IsStridedOrIndexed*/ false, Operands);

unsigned Log2MEM = IsE ? Log2SEW : Tag2Log2MEM(IntNo);
const RISCV::TH_VLSEGPseudo *P =
RISCV::getTH_VLSEGPseudo(NF, IsMasked, IsUnsigned, IsE,
Log2MEM, Log2SEW, static_cast<unsigned>(LMUL));

MachineSDNode *Load =
CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);

if (auto *MemOp = dyn_cast<MemSDNode>(Node))
CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});

SDValue SuperReg = SDValue(Load, 0);
for (unsigned I = 0; I < NF; ++I) {
unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
ReplaceUses(SDValue(Node, I),
CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
}

ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
CurDAG->RemoveDeadNode(Node);
}

#define CASE_TAG_TO_THVSSEG_INTRINSIC(tag) \
case Intrinsic::riscv_th_vsseg2##tag: \
case Intrinsic::riscv_th_vsseg3##tag: \
case Intrinsic::riscv_th_vsseg4##tag: \
case Intrinsic::riscv_th_vsseg5##tag: \
case Intrinsic::riscv_th_vsseg6##tag: \
case Intrinsic::riscv_th_vsseg7##tag: \
case Intrinsic::riscv_th_vsseg8##tag:

void RISCVDAGToDAGISel::selectXVSSEG(SDNode *Node, unsigned IntNo,
bool IsMasked, bool IsE) {
auto Tag2Log2MEM = [] (unsigned no) {
switch (no) {
CASE_TAG_TO_THVSSEG_INTRINSIC(b)
CASE_TAG_TO_THVSSEG_INTRINSIC(b_mask)
return 3;
CASE_TAG_TO_THVSSEG_INTRINSIC(h)
CASE_TAG_TO_THVSSEG_INTRINSIC(h_mask)
return 4;
CASE_TAG_TO_THVSSEG_INTRINSIC(w)
CASE_TAG_TO_THVSSEG_INTRINSIC(w_mask)
return 5;
}
llvm_unreachable("Impossible intrinsic tag");
};

SDLoc DL(Node);
unsigned NF = Node->getNumOperands() - 4;
if (IsMasked)
NF--;

MVT VT = Node->getOperand(2)->getSimpleValueType(0);

unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
unsigned Log2MEM = IsE ? Log2SEW : Tag2Log2MEM(IntNo);

RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);

SmallVector<SDValue, 8> Operands;
Operands.push_back(StoreVal);
unsigned CurOp = 2 + NF;

addXTHeadVLoadStoreOperands(Node, Log2SEW, DL, CurOp,
IsMasked, /*IsStrided*/ false, Operands);

const RISCV::TH_VSSEGPseudo *P = RISCV::getTH_VSSEGPseudo(
NF, IsMasked, IsE, Log2MEM, Log2SEW, static_cast<unsigned>(LMUL));
MachineSDNode *Store =
CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);

if (auto *MemOp = dyn_cast<MemSDNode>(Node))
CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});

ReplaceNode(Node, Store);
}

void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
bool IsStrided) {
SDLoc DL(Node);
Expand Down Expand Up @@ -1828,6 +1961,44 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
return;
}
CASE_TAG_TO_THVLSEG_INTRINSIC(b)
CASE_TAG_TO_THVLSEG_INTRINSIC(h)
CASE_TAG_TO_THVLSEG_INTRINSIC(w) {
selectXVLSEG(Node, IntNo, /*IsMask*/ false,
/*IsUnsigned*/ false, /*IsE*/ false);
return;
}
CASE_TAG_TO_THVLSEG_INTRINSIC(bu)
CASE_TAG_TO_THVLSEG_INTRINSIC(hu)
CASE_TAG_TO_THVLSEG_INTRINSIC(wu) {
selectXVLSEG(Node, IntNo, /*IsMask*/ false,
/*IsUnsigned*/ true, /*IsE*/ false);
return;
}
CASE_TAG_TO_THVLSEG_INTRINSIC(b_mask)
CASE_TAG_TO_THVLSEG_INTRINSIC(h_mask)
CASE_TAG_TO_THVLSEG_INTRINSIC(w_mask) {
selectXVLSEG(Node, IntNo, /*IsMask*/ true,
/*IsUnsigned*/ false, /*IsE*/ false);
return;
}
CASE_TAG_TO_THVLSEG_INTRINSIC(bu_mask)
CASE_TAG_TO_THVLSEG_INTRINSIC(hu_mask)
CASE_TAG_TO_THVLSEG_INTRINSIC(wu_mask) {
selectXVLSEG(Node, IntNo, /*IsMask*/ true,
/*IsUnsigned*/ true, /*IsE*/ false);
return;
}
CASE_TAG_TO_THVLSEG_INTRINSIC(e) {
selectXVLSEG(Node, IntNo, /*IsMask*/ false,
/*IsUnsigned*/ false, /*IsE*/ true);
return;
}
CASE_TAG_TO_THVLSEG_INTRINSIC(e_mask) {
selectXVLSEG(Node, IntNo, /*IsMask*/ true,
/*IsUnsigned*/ false, /*IsE*/ true);
return;
}
case Intrinsic::riscv_vlsseg2:
case Intrinsic::riscv_vlsseg3:
case Intrinsic::riscv_vlsseg4:
Expand Down Expand Up @@ -2182,6 +2353,26 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
return;
}
CASE_TAG_TO_THVSSEG_INTRINSIC(b)
CASE_TAG_TO_THVSSEG_INTRINSIC(h)
CASE_TAG_TO_THVSSEG_INTRINSIC(w) {
selectXVSSEG(Node, IntNo, /*IsMasked*/ false, /*IsE*/ false);
return;
}
CASE_TAG_TO_THVSSEG_INTRINSIC(e) {
selectXVSSEG(Node, IntNo, /*IsMasked*/ false, /*IsE*/ true);
return;
}
CASE_TAG_TO_THVSSEG_INTRINSIC(b_mask)
CASE_TAG_TO_THVSSEG_INTRINSIC(h_mask)
CASE_TAG_TO_THVSSEG_INTRINSIC(w_mask) {
selectXVSSEG(Node, IntNo, /*IsMasked*/ true, /*IsE*/ false);
return;
}
CASE_TAG_TO_THVSSEG_INTRINSIC(e_mask) {
selectXVSSEG(Node, IntNo, /*IsMasked*/ true, /*IsE*/ true);
return;
}
case Intrinsic::riscv_vssseg2:
case Intrinsic::riscv_vssseg3:
case Intrinsic::riscv_vssseg4:
Expand Down
29 changes: 27 additions & 2 deletions llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
bool IsMasked, bool IsStrided, bool IsIndexed, bool IsFF, bool IsE);
void selectXVS(SDNode *Node, const SDLoc& DL, unsigned IntNo,
bool IsMasked, bool IsStrided, bool IsIndexed, bool IsE);
void selectXVLSEG(SDNode *Node, unsigned IntNo, bool IsMasked, bool IsUnsigned, bool IsE);
void selectXVSSEG(SDNode *Node, unsigned IntNo, bool IsMasked, bool IsE);
void selectVLSEG(SDNode *Node, bool IsMasked, bool IsStrided);
void selectVLSEGFF(SDNode *Node, bool IsMasked);
void selectVLXSEG(SDNode *Node, bool IsMasked, bool IsOrdered);
Expand Down Expand Up @@ -227,6 +229,16 @@ struct VSSEGPseudo {
uint16_t Pseudo;
};

struct TH_VSSEGPseudo {
uint16_t NF : 4;
uint16_t Masked : 1;
uint16_t IsE : 1;
uint16_t Log2MEM : 3;
uint16_t Log2SEW : 3;
uint16_t LMUL : 3;
uint16_t Pseudo;
};

struct VSXSEGPseudo {
uint16_t NF : 4;
uint16_t Masked : 1;
Expand All @@ -237,6 +249,17 @@ struct VSXSEGPseudo {
uint16_t Pseudo;
};

struct TH_VLSEGPseudo {
uint16_t NF : 4;
uint16_t Masked : 1;
uint16_t Unsigned : 1;
uint16_t IsE : 1;
uint16_t Log2MEM : 3;
uint16_t Log2SEW : 3;
uint16_t LMUL : 3;
uint16_t Pseudo;
};

struct VLEPseudo {
uint16_t Masked : 1;
uint16_t Strided : 1;
Expand All @@ -246,7 +269,7 @@ struct VLEPseudo {
uint16_t Pseudo;
};

struct XVLPseudo {
struct TH_VLPseudo {
uint16_t Masked : 1;
uint16_t Strided : 1;
uint16_t Indexed : 1;
Expand All @@ -267,7 +290,7 @@ struct VSEPseudo {
uint16_t Pseudo;
};

struct XVSPseudo {
struct TH_VSPseudo {
uint16_t Masked : 1;
uint16_t Strided : 1;
uint16_t Indexed : 1;
Expand Down Expand Up @@ -305,6 +328,8 @@ struct RISCVMaskedPseudoInfo {

#define GET_XTHeadVVLTable_DECL
#define GET_XTHeadVVSTable_DECL
#define GET_XTHeadVVLSEGTable_DECL
#define GET_XTHeadVVSSEGTable_DECL
#include "RISCVGenSearchableTables.inc"
} // namespace RISCV

Expand Down
Loading

0 comments on commit 6a4f6ff

Please sign in to comment.