1395 lines · plain
1//===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file describes the BPF instructions in TableGen format.10//11//===----------------------------------------------------------------------===//12 13include "BPFInstrFormats.td"14 15// Instruction Operands and Patterns16 17// These are target-independent nodes, but have target-specific formats.18def SDT_BPFCallSeqStart : SDCallSeqStart<[SDTCisVT<0, iPTR>,19 SDTCisVT<1, iPTR>]>;20def SDT_BPFCallSeqEnd : SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>;21def SDT_BPFCall : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>;22def SDT_BPFSetFlag : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>]>;23def SDT_BPFSelectCC : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>,24 SDTCisSameAs<0, 4>,25 SDTCisSameAs<4, 5>]>;26def SDT_BPFBrCC : SDTypeProfile<0, 4, [SDTCisSameAs<0, 1>,27 SDTCisVT<3, OtherVT>]>;28def SDT_BPFWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,29 SDTCisPtrTy<0>]>;30def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>,31 SDTCisVT<1, i64>,32 SDTCisVT<2, i64>,33 SDTCisVT<3, i64>]>;34 35def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall,36 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,37 SDNPVariadic]>;38def BPFretglue : SDNode<"BPFISD::RET_GLUE", SDTNone,39 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;40def BPFcallseq_start: SDNode<"ISD::CALLSEQ_START", SDT_BPFCallSeqStart,41 [SDNPHasChain, SDNPOutGlue]>;42def BPFcallseq_end : SDNode<"ISD::CALLSEQ_END", SDT_BPFCallSeqEnd,43 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;44def BPFbrcc : SDNode<"BPFISD::BR_CC", SDT_BPFBrCC, [SDNPHasChain]>;45 46def BPFselectcc : SDNode<"BPFISD::SELECT_CC", SDT_BPFSelectCC>;47def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>;48def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY,49 [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;50def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">;51def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">;52def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">;53def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">;54def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">;55def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">;56def BPFHasBswap : Predicate<"Subtarget->hasBswap()">;57def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">;58def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">;59def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">;60def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">;61def BPFHasLoadAcqStoreRel : Predicate<"Subtarget->hasLoadAcqStoreRel()">;62def BPFHasGotox : Predicate<"Subtarget->hasGotox()">;63 64class ImmediateAsmOperand<string name> : AsmOperandClass {65 let Name = name;66 let RenderMethod = "addImmOperands";67 let DiagnosticType = !strconcat("Invalid", name);68}69 70def SImm16AsmOperand : ImmediateAsmOperand<"SImm16">;71 72def brtarget : Operand<OtherVT> {73 let PrintMethod = "printBrTargetOperand";74 let ParserMatchClass = ImmediateAsmOperand<"BrTarget">;75}76def calltarget : Operand<i64>;77 78def u64imm : Operand<i64> {79 let PrintMethod = "printImm64Operand";80}81 82def s16imm : Operand<i16> {83 let ParserMatchClass = SImm16AsmOperand;84}85 86def gpr_or_imm : Operand<i64>;87 88def i64immSExt32 : PatLeaf<(i64 imm),89 [{return isInt<32>(N->getSExtValue()); }]>;90def i32immSExt32 : PatLeaf<(i32 imm),91 [{return isInt<32>(N->getSExtValue()); }]>;92def i64immZExt32 : PatLeaf<(i64 imm),93 [{return isUInt<32>(N->getZExtValue()); }]>;94 95def imm_to_i64 : SDNodeXForm<timm, [{96 return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64);97}]>;98 99// Addressing modes.100def ADDRri : ComplexPattern<i64, 2, "SelectAddr", [], []>;101def FIri : ComplexPattern<i64, 2, "SelectFIAddr", [add, or], []>;102 103// Address operands104def MEMri : Operand<i64> {105 let PrintMethod = "printMemOperand";106 let EncoderMethod = "getMemoryOpValue";107 let DecoderMethod = "decodeMemoryOpValue";108 let MIOperandInfo = (ops GPR, s16imm);109}110 111// Conditional code predicates - used for pattern matching for jump instructions112def BPF_CC_EQ : PatLeaf<(i64 imm),113 [{return (N->getZExtValue() == ISD::SETEQ);}]>;114def BPF_CC_NE : PatLeaf<(i64 imm),115 [{return (N->getZExtValue() == ISD::SETNE);}]>;116def BPF_CC_GE : PatLeaf<(i64 imm),117 [{return (N->getZExtValue() == ISD::SETGE);}]>;118def BPF_CC_GT : PatLeaf<(i64 imm),119 [{return (N->getZExtValue() == ISD::SETGT);}]>;120def BPF_CC_GTU : PatLeaf<(i64 imm),121 [{return (N->getZExtValue() == ISD::SETUGT);}]>;122def BPF_CC_GEU : PatLeaf<(i64 imm),123 [{return (N->getZExtValue() == ISD::SETUGE);}]>;124def BPF_CC_LE : PatLeaf<(i64 imm),125 [{return (N->getZExtValue() == ISD::SETLE);}]>;126def BPF_CC_LT : PatLeaf<(i64 imm),127 [{return (N->getZExtValue() == ISD::SETLT);}]>;128def BPF_CC_LTU : PatLeaf<(i64 imm),129 [{return (N->getZExtValue() == ISD::SETULT);}]>;130def BPF_CC_LEU : PatLeaf<(i64 imm),131 [{return (N->getZExtValue() == ISD::SETULE);}]>;132def BPF_CC_EQ_32 : PatLeaf<(i32 imm),133 [{return (N->getZExtValue() == ISD::SETEQ);}]>;134def BPF_CC_NE_32 : PatLeaf<(i32 imm),135 [{return (N->getZExtValue() == ISD::SETNE);}]>;136def BPF_CC_GE_32 : PatLeaf<(i32 imm),137 [{return (N->getZExtValue() == ISD::SETGE);}]>;138def BPF_CC_GT_32 : PatLeaf<(i32 imm),139 [{return (N->getZExtValue() == ISD::SETGT);}]>;140def BPF_CC_GTU_32 : PatLeaf<(i32 imm),141 [{return (N->getZExtValue() == ISD::SETUGT);}]>;142def BPF_CC_GEU_32 : PatLeaf<(i32 imm),143 [{return (N->getZExtValue() == ISD::SETUGE);}]>;144def BPF_CC_LE_32 : PatLeaf<(i32 imm),145 [{return (N->getZExtValue() == ISD::SETLE);}]>;146def BPF_CC_LT_32 : PatLeaf<(i32 imm),147 [{return (N->getZExtValue() == ISD::SETLT);}]>;148def BPF_CC_LTU_32 : PatLeaf<(i32 imm),149 [{return (N->getZExtValue() == ISD::SETULT);}]>;150def BPF_CC_LEU_32 : PatLeaf<(i32 imm),151 [{return (N->getZExtValue() == ISD::SETULE);}]>;152def NoCond : PatLeaf<(vt)> {}153 154// For arithmetic and jump instructions the 8-bit 'code'155// field is divided into three parts:156//157// +----------------+--------+--------------------+158// | 4 bits | 1 bit | 3 bits |159// | operation code | source | instruction class |160// +----------------+--------+--------------------+161// (MSB) (LSB)162class TYPE_ALU_JMP<bits<4> op, bits<1> srctype,163 dag outs, dag ins, string asmstr, list<dag> pattern>164 : InstBPF<outs, ins, asmstr, pattern> {165 166 let Inst{63-60} = op;167 let Inst{59} = srctype;168}169 170//For load and store instructions the 8-bit 'code' field is divided as:171//172// +--------+--------+-------------------+173// | 3 bits | 2 bits | 3 bits |174// | mode | size | instruction class |175// +--------+--------+-------------------+176// (MSB) (LSB)177class TYPE_LD_ST<bits<3> mode, bits<2> size,178 dag outs, dag ins, string asmstr, list<dag> pattern>179 : InstBPF<outs, ins, asmstr, pattern> {180 181 let Inst{63-61} = mode;182 let Inst{60-59} = size;183}184 185// jump instructions186class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>187 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,188 (outs),189 (ins GPR:$dst, GPR:$src, brtarget:$BrDst),190 "if $dst "#OpcodeStr#" $src goto $BrDst",191 [(BPFbrcc i64:$dst, i64:$src, Cond, bb:$BrDst)]> {192 bits<4> dst;193 bits<4> src;194 bits<16> BrDst;195 196 let Inst{55-52} = src;197 let Inst{51-48} = dst;198 let Inst{47-32} = BrDst;199 let BPFClass = BPF_JMP;200}201 202class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>203 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,204 (outs),205 (ins GPR:$dst, i64imm:$imm, brtarget:$BrDst),206 "if $dst "#OpcodeStr#" $imm goto $BrDst",207 [(BPFbrcc i64:$dst, i64immSExt32:$imm, Cond, bb:$BrDst)]> {208 bits<4> dst;209 bits<16> BrDst;210 bits<32> imm;211 212 let Inst{51-48} = dst;213 let Inst{47-32} = BrDst;214 let Inst{31-0} = imm;215 let BPFClass = BPF_JMP;216}217 218class JMP_IND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>219 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,220 (outs),221 (ins GPR:$dst),222 !strconcat(OpcodeStr, " $dst"),223 Pattern> {224 bits<4> dst;225 226 let Inst{51-48} = dst;227 let BPFClass = BPF_JMP;228}229 230class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>231 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,232 (outs),233 (ins brtarget:$BrDst),234 !strconcat(OpcodeStr, " $BrDst"),235 Pattern> {236 bits<16> BrDst;237 238 let Inst{47-32} = BrDst;239 let BPFClass = BPF_JMP;240}241 242class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>243 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,244 (outs),245 (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst),246 "if $dst "#OpcodeStr#" $src goto $BrDst",247 [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> {248 bits<4> dst;249 bits<4> src;250 bits<16> BrDst;251 252 let Inst{55-52} = src;253 let Inst{51-48} = dst;254 let Inst{47-32} = BrDst;255 let BPFClass = BPF_JMP32;256}257 258class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>259 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,260 (outs),261 (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst),262 "if $dst "#OpcodeStr#" $imm goto $BrDst",263 [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> {264 bits<4> dst;265 bits<16> BrDst;266 bits<32> imm;267 268 let Inst{51-48} = dst;269 let Inst{47-32} = BrDst;270 let Inst{31-0} = imm;271 let BPFClass = BPF_JMP32;272}273 274multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> {275 def _rr : JMP_RR<Opc, OpcodeStr, Cond>;276 def _ri : JMP_RI<Opc, OpcodeStr, Cond>;277 def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>;278 def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>;279}280 281let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in {282// cmp+goto instructions283defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ, BPF_CC_EQ_32>;284defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>;285defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>;286defm JNE : J<BPF_JNE, "!=", BPF_CC_NE, BPF_CC_NE_32>;287defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>;288defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>;289defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>;290defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>;291defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;292defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;293defm JSET : J<BPF_JSET, "&", NoCond, NoCond>;294def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>;295 296let Predicates = [BPFHasGotox] in {297 let isIndirectBranch = 1, isBarrier = 1 in {298 def JX : JMP_IND<BPF_JA, "gotox", [(brind i64:$dst)]>;299 }300}301}302 303// ALU instructions304class ALU_RI<BPFOpClass Class, BPFArithOp Opc, int off,305 dag outs, dag ins, string asmstr, list<dag> pattern>306 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, outs, ins, asmstr, pattern> {307 bits<4> dst;308 bits<32> imm;309 310 let Inst{51-48} = dst;311 let Inst{47-32} = off;312 let Inst{31-0} = imm;313 let BPFClass = Class;314}315 316class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off,317 dag outs, dag ins, string asmstr, list<dag> pattern>318 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, outs, ins, asmstr, pattern> {319 bits<4> dst;320 bits<4> src;321 322 let Inst{55-52} = src;323 let Inst{51-48} = dst;324 let Inst{47-32} = off;325 let BPFClass = Class;326}327 328multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> {329 def _rr : ALU_RR<BPF_ALU64, Opc, off,330 (outs GPR:$dst),331 (ins GPR:$src2, GPR:$src),332 "$dst "#OpcodeStr#" $src",333 [(set GPR:$dst, (OpNode i64:$src2, i64:$src))]>;334 def _ri : ALU_RI<BPF_ALU64, Opc, off,335 (outs GPR:$dst),336 (ins GPR:$src2, i64imm:$imm),337 "$dst "#OpcodeStr#" $imm",338 [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>;339 def _rr_32 : ALU_RR<BPF_ALU, Opc, off,340 (outs GPR32:$dst),341 (ins GPR32:$src2, GPR32:$src),342 "$dst "#OpcodeStr#" $src",343 [(set GPR32:$dst, (OpNode i32:$src2, i32:$src))]>;344 def _ri_32 : ALU_RI<BPF_ALU, Opc, off,345 (outs GPR32:$dst),346 (ins GPR32:$src2, i32imm:$imm),347 "$dst "#OpcodeStr#" $imm",348 [(set GPR32:$dst, (OpNode GPR32:$src2, i32immSExt32:$imm))]>;349}350 351let Constraints = "$dst = $src2" in {352let isAsCheapAsAMove = 1 in {353 defm ADD : ALU<BPF_ADD, 0, "+=", add>;354 defm SUB : ALU<BPF_SUB, 0, "-=", sub>;355 defm OR : ALU<BPF_OR, 0, "|=", or>;356 defm AND : ALU<BPF_AND, 0, "&=", and>;357 defm SLL : ALU<BPF_LSH, 0, "<<=", shl>;358 defm SRL : ALU<BPF_RSH, 0, ">>=", srl>;359 defm XOR : ALU<BPF_XOR, 0, "^=", xor>;360 defm SRA : ALU<BPF_ARSH, 0, "s>>=", sra>;361}362 defm MUL : ALU<BPF_MUL, 0, "*=", mul>;363 defm DIV : ALU<BPF_DIV, 0, "/=", udiv>;364 defm MOD : ALU<BPF_MOD, 0, "%=", urem>;365 366 let Predicates = [BPFHasSdivSmod] in {367 defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>;368 defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>;369 }370}371 372class NEG_RR<BPFOpClass Class, BPFArithOp Opc,373 dag outs, dag ins, string asmstr, list<dag> pattern>374 : TYPE_ALU_JMP<Opc.Value, 0, outs, ins, asmstr, pattern> {375 bits<4> dst;376 377 let Inst{51-48} = dst;378 let BPFClass = Class;379}380 381let Constraints = "$dst = $src", isAsCheapAsAMove = 1 in {382 def NEG_64: NEG_RR<BPF_ALU64, BPF_NEG, (outs GPR:$dst), (ins GPR:$src),383 "$dst = -$src",384 [(set GPR:$dst, (ineg i64:$src))]>;385 def NEG_32: NEG_RR<BPF_ALU, BPF_NEG, (outs GPR32:$dst), (ins GPR32:$src),386 "$dst = -$src",387 [(set GPR32:$dst, (ineg i32:$src))]>;388}389 390class LD_IMM64<bits<4> Pseudo, string OpcodeStr>391 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,392 (outs GPR:$dst),393 (ins u64imm:$imm),394 "$dst "#OpcodeStr#" ${imm} ll",395 [(set GPR:$dst, (i64 imm:$imm))]> {396 397 bits<4> dst;398 bits<64> imm;399 400 let Inst{51-48} = dst;401 let Inst{55-52} = Pseudo;402 let Inst{47-32} = 0;403 let Inst{31-0} = imm{31-0};404 let BPFClass = BPF_LD;405}406 407let isReMaterializable = 1, isAsCheapAsAMove = 1 in {408def LD_imm64 : LD_IMM64<0, "=">;409def MOV_rr : ALU_RR<BPF_ALU64, BPF_MOV, 0,410 (outs GPR:$dst),411 (ins GPR:$src),412 "$dst = $src",413 []>;414def MOV_ri : ALU_RI<BPF_ALU64, BPF_MOV, 0,415 (outs GPR:$dst),416 (ins i64imm:$imm),417 "$dst = $imm",418 [(set GPR:$dst, (i64 i64immSExt32:$imm))]>;419def MOV_rr_32 : ALU_RR<BPF_ALU, BPF_MOV, 0,420 (outs GPR32:$dst),421 (ins GPR32:$src),422 "$dst = $src",423 []>;424def MOV_ri_32 : ALU_RI<BPF_ALU, BPF_MOV, 0,425 (outs GPR32:$dst),426 (ins i32imm:$imm),427 "$dst = $imm",428 [(set GPR32:$dst, (i32 i32immSExt32:$imm))]>;429 430let Predicates = [BPFHasMovsx] in {431 def MOVSX_rr_8 : ALU_RR<BPF_ALU64, BPF_MOV, 8,432 (outs GPR:$dst), (ins GPR:$src),433 "$dst = (s8)$src",434 [(set GPR:$dst, (sext_inreg GPR:$src, i8))]>;435 def MOVSX_rr_16 : ALU_RR<BPF_ALU64, BPF_MOV, 16,436 (outs GPR:$dst), (ins GPR:$src),437 "$dst = (s16)$src",438 [(set GPR:$dst, (sext_inreg GPR:$src, i16))]>;439 def MOVSX_rr_32 : ALU_RR<BPF_ALU64, BPF_MOV, 32,440 (outs GPR:$dst), (ins GPR:$src),441 "$dst = (s32)$src",442 [(set GPR:$dst, (sext_inreg GPR:$src, i32))]>;443 def MOVSX_rr_32_8 : ALU_RR<BPF_ALU, BPF_MOV, 8,444 (outs GPR32:$dst), (ins GPR32:$src),445 "$dst = (s8)$src",446 [(set GPR32:$dst, (sext_inreg GPR32:$src, i8))]>;447 def MOVSX_rr_32_16 : ALU_RR<BPF_ALU, BPF_MOV, 16,448 (outs GPR32:$dst), (ins GPR32:$src),449 "$dst = (s16)$src",450 [(set GPR32:$dst, (sext_inreg GPR32:$src, i16))]>;451}452}453 454def ADDR_SPACE_CAST455 : ALU_RR<BPF_ALU64, BPF_MOV, 64,456 (outs GPR:$dst),457 (ins GPR:$src, i64imm:$dst_as, i64imm:$src_as),458 "$dst = addr_space_cast($src, $dst_as, $src_as)",459 []> {460 bits<64> dst_as;461 bits<64> src_as;462 463 let Inst{47-32} = 1;464 let Inst{31-16} = dst_as{15-0};465 let Inst{15-0} = src_as{15-0};466}467 468def SrcAddrSpace : SDNodeXForm<addrspacecast, [{469 return CurDAG->getTargetConstant(470 cast<AddrSpaceCastSDNode>(N)->getSrcAddressSpace(),471 SDLoc(N), MVT::i64);472}]>;473 474def DstAddrSpace : SDNodeXForm<addrspacecast, [{475 return CurDAG->getTargetConstant(476 cast<AddrSpaceCastSDNode>(N)->getDestAddressSpace(),477 SDLoc(N), MVT::i64);478}]>;479 480def : Pat<(addrspacecast:$this GPR:$src),481 (ADDR_SPACE_CAST $src, (DstAddrSpace $this), (SrcAddrSpace $this))>;482 483def FI_ri484 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,485 (outs GPR:$dst),486 (ins MEMri:$addr),487 "lea\t$dst, $addr",488 [(set i64:$dst, FIri:$addr)]> {489 // This is a tentative instruction, and will be replaced490 // with MOV_rr and ADD_ri in PEI phase491 let Inst{51-48} = 0;492 let Inst{55-52} = 2;493 let Inst{47-32} = 0;494 let Inst{31-0} = 0;495 let BPFClass = BPF_LD;496 bit isPseudo = true;497}498 499def LD_pseudo500 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,501 (outs GPR:$dst),502 (ins i64imm:$pseudo, u64imm:$imm),503 "ld_pseudo\t$dst, $pseudo, $imm",504 [(set GPR:$dst, (int_bpf_pseudo imm:$pseudo, imm:$imm))]> {505 506 bits<4> dst;507 bits<64> imm;508 bits<4> pseudo;509 510 let Inst{51-48} = dst;511 let Inst{55-52} = pseudo;512 let Inst{47-32} = 0;513 let Inst{31-0} = imm{31-0};514 let BPFClass = BPF_LD;515}516 517// STORE instructions518class STORE<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern>519 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,520 (outs),521 (ins GPR:$src, MEMri:$addr),522 "*("#OpcodeStr#" *)($addr) = $src",523 Pattern> {524 bits<4> src;525 bits<20> addr;526 527 let Inst{51-48} = addr{19-16}; // base reg528 let Inst{55-52} = src;529 let Inst{47-32} = addr{15-0}; // offset530 let BPFClass = BPF_STX;531}532 533class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>534 : STORE<Opc, OpcodeStr, [(OpNode GPR:$src, ADDRri:$addr)]>;535 536let Predicates = [BPFNoALU32] in {537 def STW : STOREi64<BPF_W, "u32", truncstorei32>;538 def STH : STOREi64<BPF_H, "u16", truncstorei16>;539 def STB : STOREi64<BPF_B, "u8", truncstorei8>;540}541def STD : STOREi64<BPF_DW, "u64", store>;542 543class STORE_imm<BPFWidthModifer SizeOp,544 string OpcodeStr, dag Pattern>545 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,546 (outs),547 (ins i64imm:$imm, MEMri:$addr),548 "*("#OpcodeStr#" *)($addr) = $imm",549 [Pattern]> {550 bits<20> addr;551 bits<32> imm;552 553 let Inst{51-48} = addr{19-16}; // base reg554 let Inst{47-32} = addr{15-0}; // offset555 let Inst{31-0} = imm;556 let BPFClass = BPF_ST;557}558 559let Predicates = [BPFHasStoreImm] in {560 // Opcode (BPF_ST | BPF_MEM | BPF_DW) implies sign extension for561 // value stored to memory:562 // - it is fine to generate such write when immediate is -1563 // - it is incorrect to generate such write when immediate is564 // +0xffff_ffff.565 //566 // In the latter case two instructions would be generated instead of567 // one BPF_ST:568 // rA = 0xffffffff ll ; LD_imm64569 // *(u64 *)(rB + 0) = rA ; STX570 //571 // For BPF_{B,H,W} the size of value stored matches size of the immediate.572 def STD_imm : STORE_imm<BPF_DW, "u64", (store (i64 i64immSExt32:$imm), ADDRri:$addr)>;573 def STW_imm : STORE_imm<BPF_W, "u32", (truncstorei32 (i64 i64immZExt32:$imm), ADDRri:$addr)>;574 def STH_imm : STORE_imm<BPF_H, "u16", (truncstorei16 (i64 i64immZExt32:$imm), ADDRri:$addr)>;575 def STB_imm : STORE_imm<BPF_B, "u8", (truncstorei8 (i64 i64immZExt32:$imm), ADDRri:$addr)>;576}577 578let Predicates = [BPFHasALU32, BPFHasStoreImm] in {579 def : Pat<(store (i32 imm:$src), ADDRri:$dst),580 (STW_imm (imm_to_i64 $src), ADDRri:$dst)>;581 def : Pat<(truncstorei16 (i32 imm:$src), ADDRri:$dst),582 (STH_imm (imm_to_i64 imm:$src), ADDRri:$dst)>;583 def : Pat<(truncstorei8 (i32 imm:$src), ADDRri:$dst),584 (STB_imm (imm_to_i64 imm:$src), ADDRri:$dst)>;585}586 587class STORE_RELEASE<BPFWidthModifer SizeOp, string OpcodeStr, RegisterClass RegTp>588 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,589 (outs),590 (ins RegTp:$src, MEMri:$addr),591 "store_release(("#OpcodeStr#" *)($addr), $src)",592 []> {593 bits<4> src;594 bits<20> addr;595 596 let Inst{51-48} = addr{19-16}; // base reg597 let Inst{55-52} = src;598 let Inst{47-32} = addr{15-0}; // offset599 let Inst{8-4} = BPF_STORE_REL.Value;600 let BPFClass = BPF_STX;601}602 603class STORE_RELEASEi64<BPFWidthModifer Opc, string OpcodeStr>604 : STORE_RELEASE<Opc, OpcodeStr, GPR>;605 606class relaxed_store<PatFrag base>607 : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> {608 let IsAtomic = 1;609 let IsAtomicOrderingReleaseOrStronger = 0;610}611 612class releasing_store<PatFrag base>613 : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> {614 let IsAtomic = 1;615 let IsAtomicOrderingRelease = 1;616}617 618let Predicates = [BPFHasLoadAcqStoreRel] in {619 def STDREL : STORE_RELEASEi64<BPF_DW, "u64">;620 621 foreach P = [[relaxed_store<atomic_store_64>, STD],622 [releasing_store<atomic_store_64>, STDREL],623 ] in {624 def : Pat<(P[0] GPR:$val, ADDRri:$addr), (P[1] GPR:$val, ADDRri:$addr)>;625 }626}627 628// LOAD instructions629class LOAD<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern>630 : TYPE_LD_ST<ModOp.Value, SizeOp.Value,631 (outs GPR:$dst),632 (ins MEMri:$addr),633 "$dst = *("#OpcodeStr#" *)($addr)",634 Pattern> {635 bits<4> dst;636 bits<20> addr;637 638 let Inst{51-48} = dst;639 let Inst{55-52} = addr{19-16};640 let Inst{47-32} = addr{15-0};641 let BPFClass = BPF_LDX;642}643 644class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode>645 : LOAD<SizeOp, ModOp, OpcodeStr, [(set i64:$dst, (OpNode ADDRri:$addr))]>;646 647let isCodeGenOnly = 1 in {648 class CORE_LD<RegisterClass RegClass, string Sz>649 : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value,650 (outs RegClass:$dst),651 (ins u64imm:$opcode, GPR:$src, u64imm:$offset),652 "$dst = core_ld"#Sz#"($opcode, $src, $offset)",653 []>;654 def CORE_LD64 : CORE_LD<GPR, "64">;655 def CORE_LD32 : CORE_LD<GPR32, "32">;656 def CORE_ST : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value,657 (outs),658 (ins gpr_or_imm:$src, u64imm:$opcode, GPR:$ptr, u64imm:$offset),659 "core_st($src, $opcode, $ptr, $offset)",660 []>;661 let Constraints = "$dst = $src" in {662 def CORE_SHIFT : ALU_RR<BPF_ALU64, BPF_LSH, 0,663 (outs GPR:$dst),664 (ins u64imm:$opcode, GPR:$src, u64imm:$offset),665 "$dst = core_shift($opcode, $src, $offset)",666 []>;667 }668}669 670let Predicates = [BPFNoALU32] in {671 def LDW : LOADi64<BPF_W, BPF_MEM, "u32", zextloadi32>;672 def LDH : LOADi64<BPF_H, BPF_MEM, "u16", zextloadi16>;673 def LDB : LOADi64<BPF_B, BPF_MEM, "u8", zextloadi8>;674}675 676let Predicates = [BPFHasLdsx] in {677 def LDWSX : LOADi64<BPF_W, BPF_MEMSX, "s32", sextloadi32>;678 def LDHSX : LOADi64<BPF_H, BPF_MEMSX, "s16", sextloadi16>;679 def LDBSX : LOADi64<BPF_B, BPF_MEMSX, "s8", sextloadi8>;680}681 682def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>;683 684class LOAD_ACQUIRE<BPFWidthModifer SizeOp, string OpcodeStr, RegisterClass RegTp>685 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,686 (outs RegTp:$dst),687 (ins MEMri:$addr),688 "$dst = load_acquire(("#OpcodeStr#" *)($addr))",689 []> {690 bits<4> dst;691 bits<20> addr;692 693 let Inst{51-48} = dst;694 let Inst{55-52} = addr{19-16}; // base reg695 let Inst{47-32} = addr{15-0}; // offset696 let Inst{8-4} = BPF_LOAD_ACQ.Value;697 let BPFClass = BPF_STX;698}699 700class LOAD_ACQUIREi64<BPFWidthModifer SizeOp, string OpcodeStr>701 : LOAD_ACQUIRE<SizeOp, OpcodeStr, GPR>;702 703class relaxed_load<PatFrags base>704 : PatFrag<(ops node:$ptr), (base node:$ptr)> {705 let IsAtomic = 1;706 let IsAtomicOrderingAcquireOrStronger = 0;707}708 709class acquiring_load<PatFrags base>710 : PatFrag<(ops node:$ptr), (base node:$ptr)> {711 let IsAtomic = 1;712 let IsAtomicOrderingAcquire = 1;713}714 715let Predicates = [BPFHasLoadAcqStoreRel] in {716 def LDDACQ : LOAD_ACQUIREi64<BPF_DW, "u64">;717 718 foreach P = [[relaxed_load<atomic_load_nonext_64>, LDD],719 [acquiring_load<atomic_load_nonext_64>, LDDACQ],720 ] in {721 def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>;722 }723}724 725class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>726 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,727 (outs),728 (ins brtarget:$BrDst),729 !strconcat(OpcodeStr, " $BrDst"),730 Pattern> {731 bits<16> BrDst;732 733 let Inst{47-32} = BrDst;734 let BPFClass = BPF_JMP;735}736 737class BRANCH_LONG<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>738 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,739 (outs),740 (ins brtarget:$BrDst),741 !strconcat(OpcodeStr, " $BrDst"),742 Pattern> {743 bits<32> BrDst;744 745 let Inst{31-0} = BrDst;746 let BPFClass = BPF_JMP32;747}748 749class CALL<string OpcodeStr>750 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_K.Value,751 (outs),752 (ins calltarget:$BrDst),753 !strconcat(OpcodeStr, " $BrDst"),754 []> {755 bits<32> BrDst;756 757 let Inst{31-0} = BrDst;758 let BPFClass = BPF_JMP;759}760 761class CALLX<string OpcodeStr>762 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_X.Value,763 (outs),764 (ins GPR:$BrDst),765 !strconcat(OpcodeStr, " $BrDst"),766 []> {767 bits<4> BrDst;768 769 let Inst{51-48} = BrDst;770 let BPFClass = BPF_JMP;771}772 773// Jump always774let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in {775 def JMP : BRANCH<BPF_JA, "goto", [(br bb:$BrDst)]>;776 def JMPL : BRANCH_LONG<BPF_JA, "gotol", []>;777}778 779// Jump and link780let isCall=1, hasDelaySlot=0, Uses = [R11] in {781 def JAL : CALL<"call">;782 def JALX : CALLX<"callx">;783}784 785class NOP_I<string OpcodeStr>786 : TYPE_ALU_JMP<BPF_MOV.Value, BPF_X.Value,787 (outs),788 (ins i32imm:$imm),789 !strconcat(OpcodeStr, "\t$imm"),790 []> {791 // mov r0, r0 == nop792 let Inst{55-52} = 0;793 let Inst{51-48} = 0;794 let BPFClass = BPF_ALU64;795}796 797let hasSideEffects = 0, isCodeGenOnly = 1 in798 def NOP : NOP_I<"nop">;799 800class RET<string OpcodeStr>801 : TYPE_ALU_JMP<BPF_EXIT.Value, BPF_K.Value,802 (outs),803 (ins),804 !strconcat(OpcodeStr, ""),805 [(BPFretglue)]> {806 let Inst{31-0} = 0;807 let BPFClass = BPF_JMP;808}809 810let isReturn = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1,811 isNotDuplicable = 1 in {812 def RET : RET<"exit">;813}814 815// ADJCALLSTACKDOWN/UP pseudo insns816let Defs = [R11], Uses = [R11], isCodeGenOnly = 1 in {817def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2),818 "#ADJCALLSTACKDOWN $amt1 $amt2",819 [(BPFcallseq_start timm:$amt1, timm:$amt2)]>;820def ADJCALLSTACKUP : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2),821 "#ADJCALLSTACKUP $amt1 $amt2",822 [(BPFcallseq_end timm:$amt1, timm:$amt2)]>;823}824 825let usesCustomInserter = 1, isCodeGenOnly = 1 in {826 def Select : Pseudo<(outs GPR:$dst),827 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR:$src, GPR:$src2),828 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",829 [(set i64:$dst,830 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i64:$src, i64:$src2))]>;831 def Select_Ri : Pseudo<(outs GPR:$dst),832 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR:$src, GPR:$src2),833 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",834 [(set i64:$dst,835 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i64:$src, i64:$src2))]>;836 def Select_64_32 : Pseudo<(outs GPR32:$dst),837 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2),838 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",839 [(set i32:$dst,840 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i32:$src, i32:$src2))]>;841 def Select_Ri_64_32 : Pseudo<(outs GPR32:$dst),842 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2),843 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",844 [(set i32:$dst,845 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i32:$src, i32:$src2))]>;846 def Select_32 : Pseudo<(outs GPR32:$dst),847 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2),848 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",849 [(set i32:$dst,850 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i32:$src, i32:$src2))]>;851 def Select_Ri_32 : Pseudo<(outs GPR32:$dst),852 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2),853 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",854 [(set i32:$dst,855 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i32:$src, i32:$src2))]>;856 def Select_32_64 : Pseudo<(outs GPR:$dst),857 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR:$src, GPR:$src2),858 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",859 [(set i64:$dst,860 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i64:$src, i64:$src2))]>;861 def Select_Ri_32_64 : Pseudo<(outs GPR:$dst),862 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR:$src, GPR:$src2),863 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",864 [(set i64:$dst,865 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i64:$src, i64:$src2))]>;866}867 868// load 64-bit global addr into register869def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>;870def : Pat<(BPFWrapper tjumptable:$in), (LD_imm64 tjumptable:$in)>;871 872// 0xffffFFFF doesn't fit into simm32, optimize common case873def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)),874 (SRL_ri (SLL_ri (i64 GPR:$src), 32), 32)>;875 876// Calls877def : Pat<(BPFcall tglobaladdr:$dst), (JAL tglobaladdr:$dst)>;878def : Pat<(BPFcall texternalsym:$dst), (JAL texternalsym:$dst)>;879def : Pat<(BPFcall imm:$dst), (JAL imm:$dst)>;880def : Pat<(BPFcall GPR:$dst), (JALX GPR:$dst)>;881 882// Loads883let Predicates = [BPFNoALU32] in {884 def : Pat<(i64 (extloadi8 ADDRri:$src)), (i64 (LDB ADDRri:$src))>;885 def : Pat<(i64 (extloadi16 ADDRri:$src)), (i64 (LDH ADDRri:$src))>;886 def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>;887}888 889// Atomic add, and, or, xor890class ATOMIC_NOFETCH<BPFWidthModifer SizeOp, string OpType, RegisterClass RegTp,891 BPFArithOp Opc, string Opstr>892 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,893 (outs RegTp:$dst),894 (ins MEMri:$addr, RegTp:$val),895 "lock *(" #OpType# " *)($addr) " #Opstr# "= $val",896 []> {897 bits<4> dst;898 bits<20> addr;899 900 let Inst{51-48} = addr{19-16}; // base reg901 let Inst{55-52} = dst;902 let Inst{47-32} = addr{15-0}; // offset903 let Inst{7-4} = Opc.Value;904 let BPFClass = BPF_STX;905}906 907let Constraints = "$dst = $val" in {908 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {909 def XADDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_ADD, "+">;910 def XANDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_AND, "&">;911 def XORW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_OR, "|">;912 def XXORW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_XOR, "^">;913 }914 def XADDW : ATOMIC_NOFETCH<BPF_W, "u32", GPR, BPF_ADD, "+">;915 def XADDD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_ADD, "+">;916 def XANDD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_AND, "&">;917 def XORD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_OR, "|">;918 def XXORD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_XOR, "^">;919}920 921let Predicates = [BPFNoALU32] in {922 def : Pat<(atomic_load_add_i32 ADDRri:$addr, GPR:$val),923 (XADDW ADDRri:$addr, GPR:$val)>;924 def : Pat<(atomic_load_add_i64 ADDRri:$addr, GPR:$val),925 (XADDD ADDRri:$addr, GPR:$val)>;926}927 928// Atomic Fetch-and-<add, and, or, xor> operations929class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr>930 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,931 (outs GPR:$dst),932 (ins MEMri:$addr, GPR:$val),933 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",934 []> {935 bits<4> dst;936 bits<20> addr;937 938 let Inst{51-48} = addr{19-16}; // base reg939 let Inst{55-52} = dst;940 let Inst{47-32} = addr{15-0}; // offset941 let Inst{7-4} = Opc.Value;942 let Inst{3-0} = BPF_FETCH.Value;943 let BPFClass = BPF_STX;944}945 946class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr>947 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,948 (outs GPR32:$dst),949 (ins MEMri:$addr, GPR32:$val),950 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",951 []> {952 bits<4> dst;953 bits<20> addr;954 955 let Inst{51-48} = addr{19-16}; // base reg956 let Inst{55-52} = dst;957 let Inst{47-32} = addr{15-0}; // offset958 let Inst{7-4} = Opc.Value;959 let Inst{3-0} = BPF_FETCH.Value;960 let BPFClass = BPF_STX;961}962 963let Constraints = "$dst = $val" in {964 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {965 def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add">;966 def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and">;967 def XFORW32 : XFALU32<BPF_W, BPF_OR, "u32", "or">;968 def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor">;969 }970 971 let Predicates = [BPFHasALU32] in {972 def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add">;973 }974 def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and">;975 def XFORD : XFALU64<BPF_DW, BPF_OR, "u64", "or">;976 def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor">;977}978 979let Predicates = [BPFHasALU32] in {980 foreach P = [// add981 [atomic_load_add_i32_monotonic, XADDW32],982 [atomic_load_add_i32_acquire, XFADDW32],983 [atomic_load_add_i32_release, XFADDW32],984 [atomic_load_add_i32_acq_rel, XFADDW32],985 [atomic_load_add_i32_seq_cst, XFADDW32],986 // and987 [atomic_load_and_i32_monotonic, XANDW32],988 [atomic_load_and_i32_acquire, XFANDW32],989 [atomic_load_and_i32_release, XFANDW32],990 [atomic_load_and_i32_acq_rel, XFANDW32],991 [atomic_load_and_i32_seq_cst, XFANDW32],992 // or993 [atomic_load_or_i32_monotonic, XORW32],994 [atomic_load_or_i32_acquire, XFORW32],995 [atomic_load_or_i32_release, XFORW32],996 [atomic_load_or_i32_acq_rel, XFORW32],997 [atomic_load_or_i32_seq_cst, XFORW32],998 // xor999 [atomic_load_xor_i32_monotonic, XXORW32],1000 [atomic_load_xor_i32_acquire, XFXORW32],1001 [atomic_load_xor_i32_release, XFXORW32],1002 [atomic_load_xor_i32_acq_rel, XFXORW32],1003 [atomic_load_xor_i32_seq_cst, XFXORW32],1004 ] in {1005 def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1] ADDRri:$addr, GPR32:$val)>;1006 }1007 1008 // atomic_load_sub can be represented as a neg followed1009 // by an atomic_load_add.1010 foreach P = [[atomic_load_sub_i32_monotonic, XADDW32],1011 [atomic_load_sub_i32_acquire, XFADDW32],1012 [atomic_load_sub_i32_release, XFADDW32],1013 [atomic_load_sub_i32_acq_rel, XFADDW32],1014 [atomic_load_sub_i32_seq_cst, XFADDW32],1015 ] in {1016 def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1] ADDRri:$addr, (NEG_32 GPR32:$val))>;1017 }1018 1019 foreach P = [// add1020 [atomic_load_add_i64_monotonic, XADDD],1021 [atomic_load_add_i64_acquire, XFADDD],1022 [atomic_load_add_i64_release, XFADDD],1023 [atomic_load_add_i64_acq_rel, XFADDD],1024 [atomic_load_add_i64_seq_cst, XFADDD],1025 ] in {1026 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, GPR:$val)>;1027 }1028}1029 1030foreach P = [[atomic_load_sub_i64_monotonic, XADDD],1031 [atomic_load_sub_i64_acquire, XFADDD],1032 [atomic_load_sub_i64_release, XFADDD],1033 [atomic_load_sub_i64_acq_rel, XFADDD],1034 [atomic_load_sub_i64_seq_cst, XFADDD],1035 ] in {1036 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, (NEG_64 GPR:$val))>;1037}1038 1039// Borrow the idea from X86InstrFragments.td1040class binop_no_use<SDPatternOperator operator>1041 : PatFrag<(ops node:$A, node:$B),1042 (operator node:$A, node:$B),1043 [{ return SDValue(N, 0).use_empty(); }]>;1044 1045class binop_has_use<SDPatternOperator operator>1046 : PatFrag<(ops node:$A, node:$B),1047 (operator node:$A, node:$B),1048 [{ return !SDValue(N, 0).use_empty(); }]>;1049 1050foreach op = [add, and, or, xor] in {1051def atomic_load_ # op # _i64_monotonic_nu:1052 binop_no_use <!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>;1053def atomic_load_ # op # _i64_monotonic_hu:1054 binop_has_use<!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>;1055}1056 1057foreach P = [// and1058 [atomic_load_and_i64_monotonic_nu, XANDD],1059 [atomic_load_and_i64_monotonic_hu, XFANDD],1060 [atomic_load_and_i64_acquire, XFANDD],1061 [atomic_load_and_i64_release, XFANDD],1062 [atomic_load_and_i64_acq_rel, XFANDD],1063 [atomic_load_and_i64_seq_cst, XFANDD],1064 // or1065 [atomic_load_or_i64_monotonic_nu, XORD],1066 [atomic_load_or_i64_monotonic_hu, XFORD],1067 [atomic_load_or_i64_acquire, XFORD],1068 [atomic_load_or_i64_release, XFORD],1069 [atomic_load_or_i64_acq_rel, XFORD],1070 [atomic_load_or_i64_seq_cst, XFORD],1071 // xor1072 [atomic_load_xor_i64_monotonic_nu, XXORD],1073 [atomic_load_xor_i64_monotonic_hu, XFXORD],1074 [atomic_load_xor_i64_acquire, XFXORD],1075 [atomic_load_xor_i64_release, XFXORD],1076 [atomic_load_xor_i64_acq_rel, XFXORD],1077 [atomic_load_xor_i64_seq_cst, XFXORD],1078 ] in {1079 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, GPR:$val)>;1080}1081 1082// Atomic Exchange1083class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>1084 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,1085 (outs GPR:$dst),1086 (ins MEMri:$addr, GPR:$val),1087 "$dst = xchg_"#OpcodeStr#"($addr, $val)",1088 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> {1089 bits<4> dst;1090 bits<20> addr;1091 1092 let Inst{51-48} = addr{19-16}; // base reg1093 let Inst{55-52} = dst;1094 let Inst{47-32} = addr{15-0}; // offset1095 let Inst{7-4} = BPF_XCHG.Value;1096 let Inst{3-0} = BPF_FETCH.Value;1097 let BPFClass = BPF_STX;1098}1099 1100class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>1101 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,1102 (outs GPR32:$dst),1103 (ins MEMri:$addr, GPR32:$val),1104 "$dst = xchg32_"#OpcodeStr#"($addr, $val)",1105 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> {1106 bits<4> dst;1107 bits<20> addr;1108 1109 let Inst{51-48} = addr{19-16}; // base reg1110 let Inst{55-52} = dst;1111 let Inst{47-32} = addr{15-0}; // offset1112 let Inst{7-4} = BPF_XCHG.Value;1113 let Inst{3-0} = BPF_FETCH.Value;1114 let BPFClass = BPF_STX;1115}1116 1117let Constraints = "$dst = $val" in {1118 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {1119 def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_i32>;1120 }1121 1122 def XCHGD : XCHG<BPF_DW, "64", atomic_swap_i64>;1123}1124 1125// Compare-And-Exchange1126class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>1127 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,1128 (outs),1129 (ins MEMri:$addr, GPR:$new),1130 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)",1131 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> {1132 bits<4> new;1133 bits<20> addr;1134 1135 let Inst{51-48} = addr{19-16}; // base reg1136 let Inst{55-52} = new;1137 let Inst{47-32} = addr{15-0}; // offset1138 let Inst{7-4} = BPF_CMPXCHG.Value;1139 let Inst{3-0} = BPF_FETCH.Value;1140 let BPFClass = BPF_STX;1141}1142 1143class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>1144 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,1145 (outs),1146 (ins MEMri:$addr, GPR32:$new),1147 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)",1148 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> {1149 bits<4> new;1150 bits<20> addr;1151 1152 let Inst{51-48} = addr{19-16}; // base reg1153 let Inst{55-52} = new;1154 let Inst{47-32} = addr{15-0}; // offset1155 let Inst{7-4} = BPF_CMPXCHG.Value;1156 let Inst{3-0} = BPF_FETCH.Value;1157 let BPFClass = BPF_STX;1158}1159 1160let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0],1161 DecoderNamespace = "BPFALU32" in {1162 def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_i32>;1163}1164 1165let Defs = [R0], Uses = [R0] in {1166 def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_i64>;1167}1168 1169// bswap16, bswap32, bswap641170class BSWAP<BPFOpClass Class, bits<32> SizeOp, string OpcodeStr, BPFSrcType SrcType, list<dag> Pattern>1171 : TYPE_ALU_JMP<BPF_END.Value, SrcType.Value,1172 (outs GPR:$dst),1173 (ins GPR:$src),1174 "$dst = "#OpcodeStr#" $src",1175 Pattern> {1176 bits<4> dst;1177 1178 let Inst{51-48} = dst;1179 let Inst{31-0} = SizeOp;1180 let BPFClass = Class;1181}1182 1183 1184let Constraints = "$dst = $src" in {1185 let Predicates = [BPFHasBswap] in {1186 def BSWAP16 : BSWAP<BPF_ALU64, 16, "bswap16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;1187 def BSWAP32 : BSWAP<BPF_ALU64, 32, "bswap32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;1188 def BSWAP64 : BSWAP<BPF_ALU64, 64, "bswap64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>;1189 }1190 1191 let Predicates = [BPFNoBswap] in {1192 let Predicates = [BPFIsLittleEndian] in {1193 def BE16 : BSWAP<BPF_ALU, 16, "be16", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;1194 def BE32 : BSWAP<BPF_ALU, 32, "be32", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;1195 def BE64 : BSWAP<BPF_ALU, 64, "be64", BPF_TO_BE, [(set GPR:$dst, (bswap GPR:$src))]>;1196 }1197 let Predicates = [BPFIsBigEndian] in {1198 def LE16 : BSWAP<BPF_ALU, 16, "le16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;1199 def LE32 : BSWAP<BPF_ALU, 32, "le32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;1200 def LE64 : BSWAP<BPF_ALU, 64, "le64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>;1201 }1202 }1203}1204 1205let Defs = [R0, R1, R2, R3, R4, R5], Uses = [R6], hasSideEffects = 1,1206 hasExtraDefRegAllocReq = 1, hasExtraSrcRegAllocReq = 1, mayLoad = 1 in {1207class LOAD_ABS<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode>1208 : TYPE_LD_ST<BPF_ABS.Value, SizeOp.Value,1209 (outs), (ins i64imm:$imm),1210 "r0 = *("#OpcodeStr#" *)skb[$imm]",1211 [(set R0, (OpNode R6, i64immSExt32:$imm))]> {1212 bits<32> imm;1213 1214 let Inst{31-0} = imm;1215 let BPFClass = BPF_LD;1216}1217 1218class LOAD_IND<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode>1219 : TYPE_LD_ST<BPF_IND.Value, SizeOp.Value,1220 (outs), (ins GPR:$val),1221 "r0 = *("#OpcodeStr#" *)skb[$val]",1222 [(set R0, (OpNode R6, GPR:$val))]> {1223 bits<4> val;1224 1225 let Inst{55-52} = val;1226 let BPFClass = BPF_LD;1227}1228}1229 1230def LD_ABS_B : LOAD_ABS<BPF_B, "u8", int_bpf_load_byte>;1231def LD_ABS_H : LOAD_ABS<BPF_H, "u16", int_bpf_load_half>;1232def LD_ABS_W : LOAD_ABS<BPF_W, "u32", int_bpf_load_word>;1233 1234def LD_IND_B : LOAD_IND<BPF_B, "u8", int_bpf_load_byte>;1235def LD_IND_H : LOAD_IND<BPF_H, "u16", int_bpf_load_half>;1236def LD_IND_W : LOAD_IND<BPF_W, "u32", int_bpf_load_word>;1237 1238let isCodeGenOnly = 1 in {1239 def MOV_32_64 : ALU_RR<BPF_ALU, BPF_MOV, 0,1240 (outs GPR:$dst), (ins GPR32:$src),1241 "$dst = $src", []>;1242}1243 1244let Predicates = [BPFNoMovsx] in {1245 def : Pat<(i64 (sext GPR32:$src)),1246 (SRA_ri (SLL_ri (MOV_32_64 GPR32:$src), 32), 32)>;1247}1248 1249let Predicates = [BPFHasMovsx] in {1250 def : Pat<(i64 (sext GPR32:$src)),1251 (MOVSX_rr_32 (MOV_32_64 GPR32:$src))>;1252}1253 1254def : Pat<(i64 (zext GPR32:$src)), (MOV_32_64 GPR32:$src)>;1255 1256// For i64 -> i32 truncation, use the 32-bit subregister directly.1257def : Pat<(i32 (trunc GPR:$src)),1258 (i32 (EXTRACT_SUBREG GPR:$src, sub_32))>;1259 1260// For i32 -> i64 anyext, we don't care about the high bits.1261def : Pat<(i64 (anyext GPR32:$src)),1262 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>;1263 1264class STORE32<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern>1265 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,1266 (outs),1267 (ins GPR32:$src, MEMri:$addr),1268 "*("#OpcodeStr#" *)($addr) = $src",1269 Pattern> {1270 bits<4> src;1271 bits<20> addr;1272 1273 let Inst{51-48} = addr{19-16}; // base reg1274 let Inst{55-52} = src;1275 let Inst{47-32} = addr{15-0}; // offset1276 let BPFClass = BPF_STX;1277}1278 1279class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>1280 : STORE32<Opc, OpcodeStr, [(OpNode GPR32:$src, ADDRri:$addr)]>;1281 1282class STORE_RELEASEi32<BPFWidthModifer Opc, string OpcodeStr>1283 : STORE_RELEASE<Opc, OpcodeStr, GPR32>;1284 1285let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {1286 def STW32 : STOREi32<BPF_W, "u32", store>;1287 def STH32 : STOREi32<BPF_H, "u16", truncstorei16>;1288 def STB32 : STOREi32<BPF_B, "u8", truncstorei8>;1289 1290 let Predicates = [BPFHasLoadAcqStoreRel] in {1291 def STWREL32 : STORE_RELEASEi32<BPF_W, "u32">;1292 def STHREL32 : STORE_RELEASEi32<BPF_H, "u16">;1293 def STBREL32 : STORE_RELEASEi32<BPF_B, "u8">;1294 }1295}1296 1297class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern>1298 : TYPE_LD_ST<ModOp.Value, SizeOp.Value,1299 (outs GPR32:$dst),1300 (ins MEMri:$addr),1301 "$dst = *("#OpcodeStr#" *)($addr)",1302 Pattern> {1303 bits<4> dst;1304 bits<20> addr;1305 1306 let Inst{51-48} = dst;1307 let Inst{55-52} = addr{19-16};1308 let Inst{47-32} = addr{15-0};1309 let BPFClass = BPF_LDX;1310}1311 1312class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode>1313 : LOAD32<SizeOp, ModOp, OpcodeStr, [(set i32:$dst, (OpNode ADDRri:$addr))]>;1314 1315class LOAD_ACQUIREi32<BPFWidthModifer SizeOp, string OpcodeStr>1316 : LOAD_ACQUIRE<SizeOp, OpcodeStr, GPR32>;1317 1318let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {1319 def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>;1320 def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>;1321 def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>;1322 1323 let Predicates = [BPFHasLoadAcqStoreRel] in {1324 def LDWACQ32 : LOAD_ACQUIREi32<BPF_W, "u32">;1325 def LDHACQ32 : LOAD_ACQUIREi32<BPF_H, "u16">;1326 def LDBACQ32 : LOAD_ACQUIREi32<BPF_B, "u8">;1327 }1328}1329 1330let Predicates = [BPFHasALU32] in {1331 def : Pat<(truncstorei8 GPR:$src, ADDRri:$dst),1332 (STB32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;1333 def : Pat<(truncstorei16 GPR:$src, ADDRri:$dst),1334 (STH32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;1335 def : Pat<(truncstorei32 GPR:$src, ADDRri:$dst),1336 (STW32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;1337 def : Pat<(i32 (extloadi8 ADDRri:$src)), (i32 (LDB32 ADDRri:$src))>;1338 def : Pat<(i32 (extloadi16 ADDRri:$src)), (i32 (LDH32 ADDRri:$src))>;1339 1340 let Predicates = [BPFHasLdsx] in {1341 def : Pat<(i32 (sextloadi8 ADDRri:$src)), (EXTRACT_SUBREG (LDBSX ADDRri:$src), sub_32)>;1342 def : Pat<(i32 (sextloadi16 ADDRri:$src)), (EXTRACT_SUBREG (LDHSX ADDRri:$src), sub_32)>;1343 }1344 1345 def : Pat<(i64 (zextloadi8 ADDRri:$src)),1346 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>;1347 def : Pat<(i64 (zextloadi16 ADDRri:$src)),1348 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>;1349 def : Pat<(i64 (zextloadi32 ADDRri:$src)),1350 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>;1351 def : Pat<(i64 (extloadi8 ADDRri:$src)),1352 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>;1353 def : Pat<(i64 (extloadi16 ADDRri:$src)),1354 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>;1355 def : Pat<(i64 (extloadi32 ADDRri:$src)),1356 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>;1357 1358 let Predicates = [BPFHasLoadAcqStoreRel] in {1359 foreach P = [[relaxed_load<atomic_load_nonext_32>, LDW32],1360 [relaxed_load<atomic_load_azext_16>, LDH32],1361 [relaxed_load<atomic_load_azext_8>, LDB32],1362 [acquiring_load<atomic_load_nonext_32>, LDWACQ32],1363 [acquiring_load<atomic_load_azext_16>, LDHACQ32],1364 [acquiring_load<atomic_load_azext_8>, LDBACQ32],1365 ] in {1366 def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>;1367 }1368 }1369 1370 let Predicates = [BPFHasLoadAcqStoreRel] in {1371 foreach P = [[relaxed_store<atomic_store_32>, STW32],1372 [relaxed_store<atomic_store_16>, STH32],1373 [relaxed_store<atomic_store_8>, STB32],1374 [releasing_store<atomic_store_32>, STWREL32],1375 [releasing_store<atomic_store_16>, STHREL32],1376 [releasing_store<atomic_store_8>, STBREL32],1377 ] in {1378 def : Pat<(P[0] GPR32:$val, ADDRri:$addr), (P[1] GPR32:$val, ADDRri:$addr)>;1379 }1380 }1381}1382 1383let usesCustomInserter = 1, isCodeGenOnly = 1 in {1384 def MEMCPY : Pseudo<1385 (outs),1386 (ins GPR:$dst, GPR:$src, i64imm:$len, i64imm:$align, variable_ops),1387 "#memcpy dst: $dst, src: $src, len: $len, align: $align",1388 [(BPFmemcpy GPR:$dst, GPR:$src, imm:$len, imm:$align)]>;1389}1390 1391// For GlobalValue and BlockAddress.1392let usesCustomInserter = 1, isCodeGenOnly = 1 in {1393 def LDIMM64 : Pseudo<(outs GPR:$dst), (ins i64imm:$addr), "", []>;1394}1395