3008 lines · cpp
1//===-- PPCIntrinsicCall.cpp ----------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Helper routines for constructing the FIR dialect of MLIR for PowerPC10// intrinsics. Extensive use of MLIR interfaces and MLIR's coding style11// (https://mlir.llvm.org/getting_started/DeveloperGuide/) is used in this12// module.13//14//===----------------------------------------------------------------------===//15 16#include "flang/Optimizer/Builder/PPCIntrinsicCall.h"17#include "flang/Evaluate/common.h"18#include "flang/Optimizer/Builder/FIRBuilder.h"19#include "flang/Optimizer/Builder/MutableBox.h"20#include "mlir/Dialect/Index/IR/IndexOps.h"21#include "mlir/Dialect/Vector/IR/VectorOps.h"22 23namespace fir {24 25using PI = PPCIntrinsicLibrary;26 27// PPC specific intrinsic handlers.28static constexpr IntrinsicHandler ppcHandlers[]{29 {"__ppc_mma_assemble_acc",30 static_cast<IntrinsicLibrary::SubroutineGenerator>(31 &PI::genMmaIntr<MMAOp::AssembleAcc, MMAHandlerOp::SubToFunc>),32 {{{"acc", asAddr},33 {"arg1", asValue},34 {"arg2", asValue},35 {"arg3", asValue},36 {"arg4", asValue}}},37 /*isElemental=*/true},38 {"__ppc_mma_assemble_pair",39 static_cast<IntrinsicLibrary::SubroutineGenerator>(40 &PI::genMmaIntr<MMAOp::AssemblePair, MMAHandlerOp::SubToFunc>),41 {{{"pair", asAddr}, {"arg1", asValue}, {"arg2", asValue}}},42 /*isElemental=*/true},43 {"__ppc_mma_build_acc",44 static_cast<IntrinsicLibrary::SubroutineGenerator>(45 &PI::genMmaIntr<MMAOp::AssembleAcc,46 MMAHandlerOp::SubToFuncReverseArgOnLE>),47 {{{"acc", asAddr},48 {"arg1", asValue},49 {"arg2", asValue},50 {"arg3", asValue},51 {"arg4", asValue}}},52 /*isElemental=*/true},53 {"__ppc_mma_disassemble_acc",54 static_cast<IntrinsicLibrary::SubroutineGenerator>(55 &PI::genMmaIntr<MMAOp::DisassembleAcc, MMAHandlerOp::SubToFunc>),56 {{{"data", asAddr}, {"acc", asValue}}},57 /*isElemental=*/true},58 {"__ppc_mma_disassemble_pair",59 static_cast<IntrinsicLibrary::SubroutineGenerator>(60 &PI::genMmaIntr<MMAOp::DisassemblePair, MMAHandlerOp::SubToFunc>),61 {{{"data", asAddr}, {"pair", asValue}}},62 /*isElemental=*/true},63 {"__ppc_mma_pmxvbf16ger2_",64 static_cast<IntrinsicLibrary::SubroutineGenerator>(65 &PI::genMmaIntr<MMAOp::Pmxvbf16ger2, MMAHandlerOp::SubToFunc>),66 {{{"acc", asAddr},67 {"a", asValue},68 {"b", asValue},69 {"xmask", asValue},70 {"ymask", asValue},71 {"pmask", asValue}}},72 /*isElemental=*/true},73 {"__ppc_mma_pmxvbf16ger2nn",74 static_cast<IntrinsicLibrary::SubroutineGenerator>(75 &PI::genMmaIntr<MMAOp::Pmxvbf16ger2nn,76 MMAHandlerOp::FirstArgIsResult>),77 {{{"acc", asAddr},78 {"a", asValue},79 {"b", asValue},80 {"xmask", asValue},81 {"ymask", asValue},82 {"pmask", asValue}}},83 /*isElemental=*/true},84 {"__ppc_mma_pmxvbf16ger2np",85 static_cast<IntrinsicLibrary::SubroutineGenerator>(86 &PI::genMmaIntr<MMAOp::Pmxvbf16ger2np,87 MMAHandlerOp::FirstArgIsResult>),88 {{{"acc", asAddr},89 {"a", asValue},90 {"b", asValue},91 {"xmask", asValue},92 {"ymask", asValue},93 {"pmask", asValue}}},94 /*isElemental=*/true},95 {"__ppc_mma_pmxvbf16ger2pn",96 static_cast<IntrinsicLibrary::SubroutineGenerator>(97 &PI::genMmaIntr<MMAOp::Pmxvbf16ger2pn,98 MMAHandlerOp::FirstArgIsResult>),99 {{{"acc", asAddr},100 {"a", asValue},101 {"b", asValue},102 {"xmask", asValue},103 {"ymask", asValue},104 {"pmask", asValue}}},105 /*isElemental=*/true},106 {"__ppc_mma_pmxvbf16ger2pp",107 static_cast<IntrinsicLibrary::SubroutineGenerator>(108 &PI::genMmaIntr<MMAOp::Pmxvbf16ger2pp,109 MMAHandlerOp::FirstArgIsResult>),110 {{{"acc", asAddr},111 {"a", asValue},112 {"b", asValue},113 {"xmask", asValue},114 {"ymask", asValue},115 {"pmask", asValue}}},116 /*isElemental=*/true},117 {"__ppc_mma_pmxvf16ger2_",118 static_cast<IntrinsicLibrary::SubroutineGenerator>(119 &PI::genMmaIntr<MMAOp::Pmxvf16ger2, MMAHandlerOp::SubToFunc>),120 {{{"acc", asAddr},121 {"a", asValue},122 {"b", asValue},123 {"xmask", asValue},124 {"ymask", asValue},125 {"pmask", asValue}}},126 /*isElemental=*/true},127 {"__ppc_mma_pmxvf16ger2nn",128 static_cast<IntrinsicLibrary::SubroutineGenerator>(129 &PI::genMmaIntr<MMAOp::Pmxvf16ger2nn, MMAHandlerOp::FirstArgIsResult>),130 {{{"acc", asAddr},131 {"a", asValue},132 {"b", asValue},133 {"xmask", asValue},134 {"ymask", asValue},135 {"pmask", asValue}}},136 /*isElemental=*/true},137 {"__ppc_mma_pmxvf16ger2np",138 static_cast<IntrinsicLibrary::SubroutineGenerator>(139 &PI::genMmaIntr<MMAOp::Pmxvf16ger2np, MMAHandlerOp::FirstArgIsResult>),140 {{{"acc", asAddr},141 {"a", asValue},142 {"b", asValue},143 {"xmask", asValue},144 {"ymask", asValue},145 {"pmask", asValue}}},146 /*isElemental=*/true},147 {"__ppc_mma_pmxvf16ger2pn",148 static_cast<IntrinsicLibrary::SubroutineGenerator>(149 &PI::genMmaIntr<MMAOp::Pmxvf16ger2pn, MMAHandlerOp::FirstArgIsResult>),150 {{{"acc", asAddr},151 {"a", asValue},152 {"b", asValue},153 {"xmask", asValue},154 {"ymask", asValue},155 {"pmask", asValue}}},156 /*isElemental=*/true},157 {"__ppc_mma_pmxvf16ger2pp",158 static_cast<IntrinsicLibrary::SubroutineGenerator>(159 &PI::genMmaIntr<MMAOp::Pmxvf16ger2pp, MMAHandlerOp::FirstArgIsResult>),160 {{{"acc", asAddr},161 {"a", asValue},162 {"b", asValue},163 {"xmask", asValue},164 {"ymask", asValue},165 {"pmask", asValue}}},166 /*isElemental=*/true},167 {"__ppc_mma_pmxvf32ger",168 static_cast<IntrinsicLibrary::SubroutineGenerator>(169 &PI::genMmaIntr<MMAOp::Pmxvf32ger, MMAHandlerOp::SubToFunc>),170 {{{"acc", asAddr},171 {"a", asValue},172 {"b", asValue},173 {"xmask", asValue},174 {"ymask", asValue}}},175 /*isElemental=*/true},176 {"__ppc_mma_pmxvf32gernn",177 static_cast<IntrinsicLibrary::SubroutineGenerator>(178 &PI::genMmaIntr<MMAOp::Pmxvf32gernn, MMAHandlerOp::FirstArgIsResult>),179 {{{"acc", asAddr},180 {"a", asValue},181 {"b", asValue},182 {"xmask", asValue},183 {"ymask", asValue}}},184 /*isElemental=*/true},185 {"__ppc_mma_pmxvf32gernp",186 static_cast<IntrinsicLibrary::SubroutineGenerator>(187 &PI::genMmaIntr<MMAOp::Pmxvf32gernp, MMAHandlerOp::FirstArgIsResult>),188 {{{"acc", asAddr},189 {"a", asValue},190 {"b", asValue},191 {"xmask", asValue},192 {"ymask", asValue}}},193 /*isElemental=*/true},194 {"__ppc_mma_pmxvf32gerpn",195 static_cast<IntrinsicLibrary::SubroutineGenerator>(196 &PI::genMmaIntr<MMAOp::Pmxvf32gerpn, MMAHandlerOp::FirstArgIsResult>),197 {{{"acc", asAddr},198 {"a", asValue},199 {"b", asValue},200 {"xmask", asValue},201 {"ymask", asValue}}},202 /*isElemental=*/true},203 {"__ppc_mma_pmxvf32gerpp",204 static_cast<IntrinsicLibrary::SubroutineGenerator>(205 &PI::genMmaIntr<MMAOp::Pmxvf32gerpp, MMAHandlerOp::FirstArgIsResult>),206 {{{"acc", asAddr},207 {"a", asValue},208 {"b", asValue},209 {"xmask", asValue},210 {"ymask", asValue}}},211 /*isElemental=*/true},212 {"__ppc_mma_pmxvf64ger",213 static_cast<IntrinsicLibrary::SubroutineGenerator>(214 &PI::genMmaIntr<MMAOp::Pmxvf64ger, MMAHandlerOp::SubToFunc>),215 {{{"acc", asAddr},216 {"a", asValue},217 {"b", asValue},218 {"xmask", asValue},219 {"ymask", asValue}}},220 /*isElemental=*/true},221 {"__ppc_mma_pmxvf64gernn",222 static_cast<IntrinsicLibrary::SubroutineGenerator>(223 &PI::genMmaIntr<MMAOp::Pmxvf64gernn, MMAHandlerOp::FirstArgIsResult>),224 {{{"acc", asAddr},225 {"a", asValue},226 {"b", asValue},227 {"xmask", asValue},228 {"ymask", asValue}}},229 /*isElemental=*/true},230 {"__ppc_mma_pmxvf64gernp",231 static_cast<IntrinsicLibrary::SubroutineGenerator>(232 &PI::genMmaIntr<MMAOp::Pmxvf64gernp, MMAHandlerOp::FirstArgIsResult>),233 {{{"acc", asAddr},234 {"a", asValue},235 {"b", asValue},236 {"xmask", asValue},237 {"ymask", asValue}}},238 /*isElemental=*/true},239 {"__ppc_mma_pmxvf64gerpn",240 static_cast<IntrinsicLibrary::SubroutineGenerator>(241 &PI::genMmaIntr<MMAOp::Pmxvf64gerpn, MMAHandlerOp::FirstArgIsResult>),242 {{{"acc", asAddr},243 {"a", asValue},244 {"b", asValue},245 {"xmask", asValue},246 {"ymask", asValue}}},247 /*isElemental=*/true},248 {"__ppc_mma_pmxvf64gerpp",249 static_cast<IntrinsicLibrary::SubroutineGenerator>(250 &PI::genMmaIntr<MMAOp::Pmxvf64gerpp, MMAHandlerOp::FirstArgIsResult>),251 {{{"acc", asAddr},252 {"a", asValue},253 {"b", asValue},254 {"xmask", asValue},255 {"ymask", asValue}}},256 /*isElemental=*/true},257 {"__ppc_mma_pmxvi16ger2_",258 static_cast<IntrinsicLibrary::SubroutineGenerator>(259 &PI::genMmaIntr<MMAOp::Pmxvi16ger2, MMAHandlerOp::SubToFunc>),260 {{{"acc", asAddr},261 {"a", asValue},262 {"b", asValue},263 {"xmask", asValue},264 {"ymask", asValue},265 {"pmask", asValue}}},266 /*isElemental=*/true},267 {"__ppc_mma_pmxvi16ger2pp",268 static_cast<IntrinsicLibrary::SubroutineGenerator>(269 &PI::genMmaIntr<MMAOp::Pmxvi16ger2pp, MMAHandlerOp::FirstArgIsResult>),270 {{{"acc", asAddr},271 {"a", asValue},272 {"b", asValue},273 {"xmask", asValue},274 {"ymask", asValue},275 {"pmask", asValue}}},276 /*isElemental=*/true},277 {"__ppc_mma_pmxvi16ger2s",278 static_cast<IntrinsicLibrary::SubroutineGenerator>(279 &PI::genMmaIntr<MMAOp::Pmxvi16ger2s, MMAHandlerOp::SubToFunc>),280 {{{"acc", asAddr},281 {"a", asValue},282 {"b", asValue},283 {"xmask", asValue},284 {"ymask", asValue},285 {"pmask", asValue}}},286 /*isElemental=*/true},287 {"__ppc_mma_pmxvi16ger2spp",288 static_cast<IntrinsicLibrary::SubroutineGenerator>(289 &PI::genMmaIntr<MMAOp::Pmxvi16ger2spp,290 MMAHandlerOp::FirstArgIsResult>),291 {{{"acc", asAddr},292 {"a", asValue},293 {"b", asValue},294 {"xmask", asValue},295 {"ymask", asValue},296 {"pmask", asValue}}},297 /*isElemental=*/true},298 {"__ppc_mma_pmxvi4ger8_",299 static_cast<IntrinsicLibrary::SubroutineGenerator>(300 &PI::genMmaIntr<MMAOp::Pmxvi4ger8, MMAHandlerOp::SubToFunc>),301 {{{"acc", asAddr},302 {"a", asValue},303 {"b", asValue},304 {"xmask", asValue},305 {"ymask", asValue},306 {"pmask", asValue}}},307 /*isElemental=*/true},308 {"__ppc_mma_pmxvi4ger8pp",309 static_cast<IntrinsicLibrary::SubroutineGenerator>(310 &PI::genMmaIntr<MMAOp::Pmxvi4ger8pp, MMAHandlerOp::FirstArgIsResult>),311 {{{"acc", asAddr},312 {"a", asValue},313 {"b", asValue},314 {"xmask", asValue},315 {"ymask", asValue},316 {"pmask", asValue}}},317 /*isElemental=*/true},318 {"__ppc_mma_pmxvi8ger4_",319 static_cast<IntrinsicLibrary::SubroutineGenerator>(320 &PI::genMmaIntr<MMAOp::Pmxvi8ger4, MMAHandlerOp::SubToFunc>),321 {{{"acc", asAddr},322 {"a", asValue},323 {"b", asValue},324 {"xmask", asValue},325 {"ymask", asValue},326 {"pmask", asValue}}},327 /*isElemental=*/true},328 {"__ppc_mma_pmxvi8ger4pp",329 static_cast<IntrinsicLibrary::SubroutineGenerator>(330 &PI::genMmaIntr<MMAOp::Pmxvi8ger4pp, MMAHandlerOp::FirstArgIsResult>),331 {{{"acc", asAddr},332 {"a", asValue},333 {"b", asValue},334 {"xmask", asValue},335 {"ymask", asValue},336 {"pmask", asValue}}},337 /*isElemental=*/true},338 {"__ppc_mma_pmxvi8ger4spp",339 static_cast<IntrinsicLibrary::SubroutineGenerator>(340 &PI::genMmaIntr<MMAOp::Pmxvi8ger4spp, MMAHandlerOp::FirstArgIsResult>),341 {{{"acc", asAddr},342 {"a", asValue},343 {"b", asValue},344 {"xmask", asValue},345 {"ymask", asValue},346 {"pmask", asValue}}},347 /*isElemental=*/true},348 {"__ppc_mma_xvbf16ger2_",349 static_cast<IntrinsicLibrary::SubroutineGenerator>(350 &PI::genMmaIntr<MMAOp::Xvbf16ger2, MMAHandlerOp::SubToFunc>),351 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},352 /*isElemental=*/true},353 {"__ppc_mma_xvbf16ger2nn",354 static_cast<IntrinsicLibrary::SubroutineGenerator>(355 &PI::genMmaIntr<MMAOp::Xvbf16ger2nn, MMAHandlerOp::FirstArgIsResult>),356 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},357 /*isElemental=*/true},358 {"__ppc_mma_xvbf16ger2np",359 static_cast<IntrinsicLibrary::SubroutineGenerator>(360 &PI::genMmaIntr<MMAOp::Xvbf16ger2np, MMAHandlerOp::FirstArgIsResult>),361 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},362 /*isElemental=*/true},363 {"__ppc_mma_xvbf16ger2pn",364 static_cast<IntrinsicLibrary::SubroutineGenerator>(365 &PI::genMmaIntr<MMAOp::Xvbf16ger2pn, MMAHandlerOp::FirstArgIsResult>),366 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},367 /*isElemental=*/true},368 {"__ppc_mma_xvbf16ger2pp",369 static_cast<IntrinsicLibrary::SubroutineGenerator>(370 &PI::genMmaIntr<MMAOp::Xvbf16ger2pp, MMAHandlerOp::FirstArgIsResult>),371 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},372 /*isElemental=*/true},373 {"__ppc_mma_xvf16ger2_",374 static_cast<IntrinsicLibrary::SubroutineGenerator>(375 &PI::genMmaIntr<MMAOp::Xvf16ger2, MMAHandlerOp::SubToFunc>),376 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},377 /*isElemental=*/true},378 {"__ppc_mma_xvf16ger2nn",379 static_cast<IntrinsicLibrary::SubroutineGenerator>(380 &PI::genMmaIntr<MMAOp::Xvf16ger2nn, MMAHandlerOp::FirstArgIsResult>),381 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},382 /*isElemental=*/true},383 {"__ppc_mma_xvf16ger2np",384 static_cast<IntrinsicLibrary::SubroutineGenerator>(385 &PI::genMmaIntr<MMAOp::Xvf16ger2np, MMAHandlerOp::FirstArgIsResult>),386 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},387 /*isElemental=*/true},388 {"__ppc_mma_xvf16ger2pn",389 static_cast<IntrinsicLibrary::SubroutineGenerator>(390 &PI::genMmaIntr<MMAOp::Xvf16ger2pn, MMAHandlerOp::FirstArgIsResult>),391 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},392 /*isElemental=*/true},393 {"__ppc_mma_xvf16ger2pp",394 static_cast<IntrinsicLibrary::SubroutineGenerator>(395 &PI::genMmaIntr<MMAOp::Xvf16ger2pp, MMAHandlerOp::FirstArgIsResult>),396 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},397 /*isElemental=*/true},398 {"__ppc_mma_xvf32ger",399 static_cast<IntrinsicLibrary::SubroutineGenerator>(400 &PI::genMmaIntr<MMAOp::Xvf32ger, MMAHandlerOp::SubToFunc>),401 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},402 /*isElemental=*/true},403 {"__ppc_mma_xvf32gernn",404 static_cast<IntrinsicLibrary::SubroutineGenerator>(405 &PI::genMmaIntr<MMAOp::Xvf32gernn, MMAHandlerOp::FirstArgIsResult>),406 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},407 /*isElemental=*/true},408 {"__ppc_mma_xvf32gernp",409 static_cast<IntrinsicLibrary::SubroutineGenerator>(410 &PI::genMmaIntr<MMAOp::Xvf32gernp, MMAHandlerOp::FirstArgIsResult>),411 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},412 /*isElemental=*/true},413 {"__ppc_mma_xvf32gerpn",414 static_cast<IntrinsicLibrary::SubroutineGenerator>(415 &PI::genMmaIntr<MMAOp::Xvf32gerpn, MMAHandlerOp::FirstArgIsResult>),416 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},417 /*isElemental=*/true},418 {"__ppc_mma_xvf32gerpp",419 static_cast<IntrinsicLibrary::SubroutineGenerator>(420 &PI::genMmaIntr<MMAOp::Xvf32gerpp, MMAHandlerOp::FirstArgIsResult>),421 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},422 /*isElemental=*/true},423 {"__ppc_mma_xvf64ger",424 static_cast<IntrinsicLibrary::SubroutineGenerator>(425 &PI::genMmaIntr<MMAOp::Xvf64ger, MMAHandlerOp::SubToFunc>),426 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},427 /*isElemental=*/true},428 {"__ppc_mma_xvf64gernn",429 static_cast<IntrinsicLibrary::SubroutineGenerator>(430 &PI::genMmaIntr<MMAOp::Xvf64gernn, MMAHandlerOp::FirstArgIsResult>),431 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},432 /*isElemental=*/true},433 {"__ppc_mma_xvf64gernp",434 static_cast<IntrinsicLibrary::SubroutineGenerator>(435 &PI::genMmaIntr<MMAOp::Xvf64gernp, MMAHandlerOp::FirstArgIsResult>),436 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},437 /*isElemental=*/true},438 {"__ppc_mma_xvf64gerpn",439 static_cast<IntrinsicLibrary::SubroutineGenerator>(440 &PI::genMmaIntr<MMAOp::Xvf64gerpn, MMAHandlerOp::FirstArgIsResult>),441 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},442 /*isElemental=*/true},443 {"__ppc_mma_xvf64gerpp",444 static_cast<IntrinsicLibrary::SubroutineGenerator>(445 &PI::genMmaIntr<MMAOp::Xvf64gerpp, MMAHandlerOp::FirstArgIsResult>),446 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},447 /*isElemental=*/true},448 {"__ppc_mma_xvi16ger2_",449 static_cast<IntrinsicLibrary::SubroutineGenerator>(450 &PI::genMmaIntr<MMAOp::Xvi16ger2, MMAHandlerOp::SubToFunc>),451 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},452 /*isElemental=*/true},453 {"__ppc_mma_xvi16ger2pp",454 static_cast<IntrinsicLibrary::SubroutineGenerator>(455 &PI::genMmaIntr<MMAOp::Xvi16ger2pp, MMAHandlerOp::FirstArgIsResult>),456 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},457 /*isElemental=*/true},458 {"__ppc_mma_xvi16ger2s",459 static_cast<IntrinsicLibrary::SubroutineGenerator>(460 &PI::genMmaIntr<MMAOp::Xvi16ger2s, MMAHandlerOp::SubToFunc>),461 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},462 /*isElemental=*/true},463 {"__ppc_mma_xvi16ger2spp",464 static_cast<IntrinsicLibrary::SubroutineGenerator>(465 &PI::genMmaIntr<MMAOp::Xvi16ger2spp, MMAHandlerOp::FirstArgIsResult>),466 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},467 /*isElemental=*/true},468 {"__ppc_mma_xvi4ger8_",469 static_cast<IntrinsicLibrary::SubroutineGenerator>(470 &PI::genMmaIntr<MMAOp::Xvi4ger8, MMAHandlerOp::SubToFunc>),471 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},472 /*isElemental=*/true},473 {"__ppc_mma_xvi4ger8pp",474 static_cast<IntrinsicLibrary::SubroutineGenerator>(475 &PI::genMmaIntr<MMAOp::Xvi4ger8pp, MMAHandlerOp::FirstArgIsResult>),476 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},477 /*isElemental=*/true},478 {"__ppc_mma_xvi8ger4_",479 static_cast<IntrinsicLibrary::SubroutineGenerator>(480 &PI::genMmaIntr<MMAOp::Xvi8ger4, MMAHandlerOp::SubToFunc>),481 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},482 /*isElemental=*/true},483 {"__ppc_mma_xvi8ger4pp",484 static_cast<IntrinsicLibrary::SubroutineGenerator>(485 &PI::genMmaIntr<MMAOp::Xvi8ger4pp, MMAHandlerOp::FirstArgIsResult>),486 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},487 /*isElemental=*/true},488 {"__ppc_mma_xvi8ger4spp",489 static_cast<IntrinsicLibrary::SubroutineGenerator>(490 &PI::genMmaIntr<MMAOp::Xvi8ger4spp, MMAHandlerOp::FirstArgIsResult>),491 {{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},492 /*isElemental=*/true},493 {"__ppc_mma_xxmfacc",494 static_cast<IntrinsicLibrary::SubroutineGenerator>(495 &PI::genMmaIntr<MMAOp::Xxmfacc, MMAHandlerOp::FirstArgIsResult>),496 {{{"acc", asAddr}}},497 /*isElemental=*/true},498 {"__ppc_mma_xxmtacc",499 static_cast<IntrinsicLibrary::SubroutineGenerator>(500 &PI::genMmaIntr<MMAOp::Xxmtacc, MMAHandlerOp::FirstArgIsResult>),501 {{{"acc", asAddr}}},502 /*isElemental=*/true},503 {"__ppc_mma_xxsetaccz",504 static_cast<IntrinsicLibrary::SubroutineGenerator>(505 &PI::genMmaIntr<MMAOp::Xxsetaccz, MMAHandlerOp::SubToFunc>),506 {{{"acc", asAddr}}},507 /*isElemental=*/true},508 {"__ppc_mtfsf",509 static_cast<IntrinsicLibrary::SubroutineGenerator>(&PI::genMtfsf<false>),510 {{{"mask", asValue}, {"r", asValue}}},511 /*isElemental=*/false},512 {"__ppc_mtfsfi",513 static_cast<IntrinsicLibrary::SubroutineGenerator>(&PI::genMtfsf<true>),514 {{{"bf", asValue}, {"i", asValue}}},515 /*isElemental=*/false},516 {"__ppc_vec_abs",517 static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecAbs),518 {{{"arg1", asValue}}},519 /*isElemental=*/true},520 {"__ppc_vec_add",521 static_cast<IntrinsicLibrary::ExtendedGenerator>(522 &PI::genVecAddAndMulSubXor<VecOp::Add>),523 {{{"arg1", asValue}, {"arg2", asValue}}},524 /*isElemental=*/true},525 {"__ppc_vec_and",526 static_cast<IntrinsicLibrary::ExtendedGenerator>(527 &PI::genVecAddAndMulSubXor<VecOp::And>),528 {{{"arg1", asValue}, {"arg2", asValue}}},529 /*isElemental=*/true},530 {"__ppc_vec_any_ge",531 static_cast<IntrinsicLibrary::ExtendedGenerator>(532 &PI::genVecAnyCompare<VecOp::Anyge>),533 {{{"arg1", asValue}, {"arg2", asValue}}},534 /*isElemental=*/true},535 {"__ppc_vec_cmpge",536 static_cast<IntrinsicLibrary::ExtendedGenerator>(537 &PI::genVecCmp<VecOp::Cmpge>),538 {{{"arg1", asValue}, {"arg2", asValue}}},539 /*isElemental=*/true},540 {"__ppc_vec_cmpgt",541 static_cast<IntrinsicLibrary::ExtendedGenerator>(542 &PI::genVecCmp<VecOp::Cmpgt>),543 {{{"arg1", asValue}, {"arg2", asValue}}},544 /*isElemental=*/true},545 {"__ppc_vec_cmple",546 static_cast<IntrinsicLibrary::ExtendedGenerator>(547 &PI::genVecCmp<VecOp::Cmple>),548 {{{"arg1", asValue}, {"arg2", asValue}}},549 /*isElemental=*/true},550 {"__ppc_vec_cmplt",551 static_cast<IntrinsicLibrary::ExtendedGenerator>(552 &PI::genVecCmp<VecOp::Cmplt>),553 {{{"arg1", asValue}, {"arg2", asValue}}},554 /*isElemental=*/true},555 {"__ppc_vec_convert",556 static_cast<IntrinsicLibrary::ExtendedGenerator>(557 &PI::genVecConvert<VecOp::Convert>),558 {{{"v", asValue}, {"mold", asValue}}},559 /*isElemental=*/false},560 {"__ppc_vec_ctf",561 static_cast<IntrinsicLibrary::ExtendedGenerator>(562 &PI::genVecConvert<VecOp::Ctf>),563 {{{"arg1", asValue}, {"arg2", asValue}}},564 /*isElemental=*/true},565 {"__ppc_vec_cvf",566 static_cast<IntrinsicLibrary::ExtendedGenerator>(567 &PI::genVecConvert<VecOp::Cvf>),568 {{{"arg1", asValue}}},569 /*isElemental=*/true},570 {"__ppc_vec_extract",571 static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecExtract),572 {{{"arg1", asValue}, {"arg2", asValue}}},573 /*isElemental=*/true},574 {"__ppc_vec_insert",575 static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecInsert),576 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},577 /*isElemental=*/true},578 {"__ppc_vec_ld",579 static_cast<IntrinsicLibrary::ExtendedGenerator>(580 &PI::genVecLdCallGrp<VecOp::Ld>),581 {{{"arg1", asValue}, {"arg2", asAddr}}},582 /*isElemental=*/false},583 {"__ppc_vec_lde",584 static_cast<IntrinsicLibrary::ExtendedGenerator>(585 &PI::genVecLdCallGrp<VecOp::Lde>),586 {{{"arg1", asValue}, {"arg2", asAddr}}},587 /*isElemental=*/false},588 {"__ppc_vec_ldl",589 static_cast<IntrinsicLibrary::ExtendedGenerator>(590 &PI::genVecLdCallGrp<VecOp::Ldl>),591 {{{"arg1", asValue}, {"arg2", asAddr}}},592 /*isElemental=*/false},593 {"__ppc_vec_lvsl",594 static_cast<IntrinsicLibrary::ExtendedGenerator>(595 &PI::genVecLvsGrp<VecOp::Lvsl>),596 {{{"arg1", asValue}, {"arg2", asAddr}}},597 /*isElemental=*/false},598 {"__ppc_vec_lvsr",599 static_cast<IntrinsicLibrary::ExtendedGenerator>(600 &PI::genVecLvsGrp<VecOp::Lvsr>),601 {{{"arg1", asValue}, {"arg2", asAddr}}},602 /*isElemental=*/false},603 {"__ppc_vec_lxv",604 static_cast<IntrinsicLibrary::ExtendedGenerator>(605 &PI::genVecLdNoCallGrp<VecOp::Lxv>),606 {{{"arg1", asValue}, {"arg2", asAddr}}},607 /*isElemental=*/false},608 {"__ppc_vec_lxvp",609 static_cast<IntrinsicLibrary::ExtendedGenerator>(610 &PI::genVecLdCallGrp<VecOp::Lxvp>),611 {{{"arg1", asValue}, {"arg2", asAddr}}},612 /*isElemental=*/false},613 {"__ppc_vec_mergeh",614 static_cast<IntrinsicLibrary::ExtendedGenerator>(615 &PI::genVecMerge<VecOp::Mergeh>),616 {{{"arg1", asValue}, {"arg2", asValue}}},617 /*isElemental=*/true},618 {"__ppc_vec_mergel",619 static_cast<IntrinsicLibrary::ExtendedGenerator>(620 &PI::genVecMerge<VecOp::Mergel>),621 {{{"arg1", asValue}, {"arg2", asValue}}},622 /*isElemental=*/true},623 {"__ppc_vec_msub",624 static_cast<IntrinsicLibrary::ExtendedGenerator>(625 &PI::genVecNmaddMsub<VecOp::Msub>),626 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},627 /*isElemental=*/true},628 {"__ppc_vec_mul",629 static_cast<IntrinsicLibrary::ExtendedGenerator>(630 &PI::genVecAddAndMulSubXor<VecOp::Mul>),631 {{{"arg1", asValue}, {"arg2", asValue}}},632 /*isElemental=*/true},633 {"__ppc_vec_nmadd",634 static_cast<IntrinsicLibrary::ExtendedGenerator>(635 &PI::genVecNmaddMsub<VecOp::Nmadd>),636 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},637 /*isElemental=*/true},638 {"__ppc_vec_perm",639 static_cast<IntrinsicLibrary::ExtendedGenerator>(640 &PI::genVecPerm<VecOp::Perm>),641 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},642 /*isElemental=*/true},643 {"__ppc_vec_permi",644 static_cast<IntrinsicLibrary::ExtendedGenerator>(645 &PI::genVecPerm<VecOp::Permi>),646 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},647 /*isElemental=*/true},648 {"__ppc_vec_sel",649 static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecSel),650 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},651 /*isElemental=*/true},652 {"__ppc_vec_sl",653 static_cast<IntrinsicLibrary::ExtendedGenerator>(654 &PI::genVecShift<VecOp::Sl>),655 {{{"arg1", asValue}, {"arg2", asValue}}},656 /*isElemental=*/true},657 {"__ppc_vec_sld",658 static_cast<IntrinsicLibrary::ExtendedGenerator>(659 &PI::genVecShift<VecOp::Sld>),660 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},661 /*isElemental=*/true},662 {"__ppc_vec_sldw",663 static_cast<IntrinsicLibrary::ExtendedGenerator>(664 &PI::genVecShift<VecOp::Sldw>),665 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},666 /*isElemental=*/true},667 {"__ppc_vec_sll",668 static_cast<IntrinsicLibrary::ExtendedGenerator>(669 &PI::genVecShift<VecOp::Sll>),670 {{{"arg1", asValue}, {"arg2", asValue}}},671 /*isElemental=*/true},672 {"__ppc_vec_slo",673 static_cast<IntrinsicLibrary::ExtendedGenerator>(674 &PI::genVecShift<VecOp::Slo>),675 {{{"arg1", asValue}, {"arg2", asValue}}},676 /*isElemental=*/true},677 {"__ppc_vec_splat",678 static_cast<IntrinsicLibrary::ExtendedGenerator>(679 &PI::genVecSplat<VecOp::Splat>),680 {{{"arg1", asValue}, {"arg2", asValue}}},681 /*isElemental=*/true},682 {"__ppc_vec_splat_s32_",683 static_cast<IntrinsicLibrary::ExtendedGenerator>(684 &PI::genVecSplat<VecOp::Splat_s32>),685 {{{"arg1", asValue}}},686 /*isElemental=*/true},687 {"__ppc_vec_splats",688 static_cast<IntrinsicLibrary::ExtendedGenerator>(689 &PI::genVecSplat<VecOp::Splats>),690 {{{"arg1", asValue}}},691 /*isElemental=*/true},692 {"__ppc_vec_sr",693 static_cast<IntrinsicLibrary::ExtendedGenerator>(694 &PI::genVecShift<VecOp::Sr>),695 {{{"arg1", asValue}, {"arg2", asValue}}},696 /*isElemental=*/true},697 {"__ppc_vec_srl",698 static_cast<IntrinsicLibrary::ExtendedGenerator>(699 &PI::genVecShift<VecOp::Srl>),700 {{{"arg1", asValue}, {"arg2", asValue}}},701 /*isElemental=*/true},702 {"__ppc_vec_sro",703 static_cast<IntrinsicLibrary::ExtendedGenerator>(704 &PI::genVecShift<VecOp::Sro>),705 {{{"arg1", asValue}, {"arg2", asValue}}},706 /*isElemental=*/true},707 {"__ppc_vec_st",708 static_cast<IntrinsicLibrary::SubroutineGenerator>(709 &PI::genVecStore<VecOp::St>),710 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},711 /*isElemental=*/false},712 {"__ppc_vec_ste",713 static_cast<IntrinsicLibrary::SubroutineGenerator>(714 &PI::genVecStore<VecOp::Ste>),715 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},716 /*isElemental=*/false},717 {"__ppc_vec_stxv",718 static_cast<IntrinsicLibrary::SubroutineGenerator>(719 &PI::genVecXStore<VecOp::Stxv>),720 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},721 /*isElemental=*/false},722 {"__ppc_vec_stxvp",723 static_cast<IntrinsicLibrary::SubroutineGenerator>(724 &PI::genVecStore<VecOp::Stxvp>),725 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},726 /*isElemental=*/false},727 {"__ppc_vec_sub",728 static_cast<IntrinsicLibrary::ExtendedGenerator>(729 &PI::genVecAddAndMulSubXor<VecOp::Sub>),730 {{{"arg1", asValue}, {"arg2", asValue}}},731 /*isElemental=*/true},732 {"__ppc_vec_xl",733 static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecXlGrp),734 {{{"arg1", asValue}, {"arg2", asAddr}}},735 /*isElemental=*/false},736 {"__ppc_vec_xl_be",737 static_cast<IntrinsicLibrary::ExtendedGenerator>(738 &PI::genVecLdNoCallGrp<VecOp::Xlbe>),739 {{{"arg1", asValue}, {"arg2", asAddr}}},740 /*isElemental=*/false},741 {"__ppc_vec_xld2_",742 static_cast<IntrinsicLibrary::ExtendedGenerator>(743 &PI::genVecLdCallGrp<VecOp::Xld2>),744 {{{"arg1", asValue}, {"arg2", asAddr}}},745 /*isElemental=*/false},746 {"__ppc_vec_xlds",747 static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecXlds),748 {{{"arg1", asValue}, {"arg2", asAddr}}},749 /*isElemental=*/false},750 {"__ppc_vec_xlw4_",751 static_cast<IntrinsicLibrary::ExtendedGenerator>(752 &PI::genVecLdCallGrp<VecOp::Xlw4>),753 {{{"arg1", asValue}, {"arg2", asAddr}}},754 /*isElemental=*/false},755 {"__ppc_vec_xor",756 static_cast<IntrinsicLibrary::ExtendedGenerator>(757 &PI::genVecAddAndMulSubXor<VecOp::Xor>),758 {{{"arg1", asValue}, {"arg2", asValue}}},759 /*isElemental=*/true},760 {"__ppc_vec_xst",761 static_cast<IntrinsicLibrary::SubroutineGenerator>(762 &PI::genVecXStore<VecOp::Xst>),763 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},764 /*isElemental=*/false},765 {"__ppc_vec_xst_be",766 static_cast<IntrinsicLibrary::SubroutineGenerator>(767 &PI::genVecXStore<VecOp::Xst_be>),768 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},769 /*isElemental=*/false},770 {"__ppc_vec_xstd2_",771 static_cast<IntrinsicLibrary::SubroutineGenerator>(772 &PI::genVecXStore<VecOp::Xstd2>),773 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},774 /*isElemental=*/false},775 {"__ppc_vec_xstw4_",776 static_cast<IntrinsicLibrary::SubroutineGenerator>(777 &PI::genVecXStore<VecOp::Xstw4>),778 {{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},779 /*isElemental=*/false},780};781 782static constexpr MathOperation ppcMathOperations[] = {783 // fcfi is just another name for fcfid, there is no llvm.ppc.fcfi.784 {"__ppc_fcfi", "llvm.ppc.fcfid", genFuncType<Ty::Real<8>, Ty::Real<8>>,785 genLibCall},786 {"__ppc_fcfid", "llvm.ppc.fcfid", genFuncType<Ty::Real<8>, Ty::Real<8>>,787 genLibCall},788 {"__ppc_fcfud", "llvm.ppc.fcfud", genFuncType<Ty::Real<8>, Ty::Real<8>>,789 genLibCall},790 {"__ppc_fctid", "llvm.ppc.fctid", genFuncType<Ty::Real<8>, Ty::Real<8>>,791 genLibCall},792 {"__ppc_fctidz", "llvm.ppc.fctidz", genFuncType<Ty::Real<8>, Ty::Real<8>>,793 genLibCall},794 {"__ppc_fctiw", "llvm.ppc.fctiw", genFuncType<Ty::Real<8>, Ty::Real<8>>,795 genLibCall},796 {"__ppc_fctiwz", "llvm.ppc.fctiwz", genFuncType<Ty::Real<8>, Ty::Real<8>>,797 genLibCall},798 {"__ppc_fctudz", "llvm.ppc.fctudz", genFuncType<Ty::Real<8>, Ty::Real<8>>,799 genLibCall},800 {"__ppc_fctuwz", "llvm.ppc.fctuwz", genFuncType<Ty::Real<8>, Ty::Real<8>>,801 genLibCall},802 {"__ppc_fmadd", "llvm.fma.f32",803 genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,804 genMathOp<mlir::math::FmaOp>},805 {"__ppc_fmadd", "llvm.fma.f64",806 genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,807 genMathOp<mlir::math::FmaOp>},808 {"__ppc_fmsub", "llvm.ppc.fmsubs",809 genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,810 genLibCall},811 {"__ppc_fmsub", "llvm.ppc.fmsub",812 genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,813 genLibCall},814 {"__ppc_fnabs", "llvm.ppc.fnabss", genFuncType<Ty::Real<4>, Ty::Real<4>>,815 genLibCall},816 {"__ppc_fnabs", "llvm.ppc.fnabs", genFuncType<Ty::Real<8>, Ty::Real<8>>,817 genLibCall},818 {"__ppc_fnmadd", "llvm.ppc.fnmadds",819 genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,820 genLibCall},821 {"__ppc_fnmadd", "llvm.ppc.fnmadd",822 genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,823 genLibCall},824 {"__ppc_fnmsub", "llvm.ppc.fnmsub.f32",825 genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,826 genLibCall},827 {"__ppc_fnmsub", "llvm.ppc.fnmsub.f64",828 genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,829 genLibCall},830 {"__ppc_fre", "llvm.ppc.fre", genFuncType<Ty::Real<8>, Ty::Real<8>>,831 genLibCall},832 {"__ppc_fres", "llvm.ppc.fres", genFuncType<Ty::Real<4>, Ty::Real<4>>,833 genLibCall},834 {"__ppc_frsqrte", "llvm.ppc.frsqrte", genFuncType<Ty::Real<8>, Ty::Real<8>>,835 genLibCall},836 {"__ppc_frsqrtes", "llvm.ppc.frsqrtes",837 genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},838 {"__ppc_vec_cvbf16spn", "llvm.ppc.vsx.xvcvbf16spn",839 genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>, genLibCall},840 {"__ppc_vec_cvspbf16_", "llvm.ppc.vsx.xvcvspbf16",841 genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>, genLibCall},842 {"__ppc_vec_madd", "llvm.fma.v4f32",843 genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>,844 Ty::RealVector<4>>,845 genLibCall},846 {"__ppc_vec_madd", "llvm.fma.v2f64",847 genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>,848 Ty::RealVector<8>>,849 genLibCall},850 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsb",851 genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>,852 Ty::IntegerVector<1>>,853 genLibCall},854 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsh",855 genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>,856 Ty::IntegerVector<2>>,857 genLibCall},858 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsw",859 genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,860 Ty::IntegerVector<4>>,861 genLibCall},862 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxsd",863 genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>,864 Ty::IntegerVector<8>>,865 genLibCall},866 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxub",867 genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>,868 Ty::UnsignedVector<1>>,869 genLibCall},870 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxuh",871 genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>,872 Ty::UnsignedVector<2>>,873 genLibCall},874 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxuw",875 genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>,876 Ty::UnsignedVector<4>>,877 genLibCall},878 {"__ppc_vec_max", "llvm.ppc.altivec.vmaxud",879 genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>,880 Ty::UnsignedVector<8>>,881 genLibCall},882 {"__ppc_vec_max", "llvm.ppc.vsx.xvmaxsp",883 genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>,884 genLibCall},885 {"__ppc_vec_max", "llvm.ppc.vsx.xvmaxdp",886 genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>,887 genLibCall},888 {"__ppc_vec_min", "llvm.ppc.altivec.vminsb",889 genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>,890 Ty::IntegerVector<1>>,891 genLibCall},892 {"__ppc_vec_min", "llvm.ppc.altivec.vminsh",893 genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>,894 Ty::IntegerVector<2>>,895 genLibCall},896 {"__ppc_vec_min", "llvm.ppc.altivec.vminsw",897 genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,898 Ty::IntegerVector<4>>,899 genLibCall},900 {"__ppc_vec_min", "llvm.ppc.altivec.vminsd",901 genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>,902 Ty::IntegerVector<8>>,903 genLibCall},904 {"__ppc_vec_min", "llvm.ppc.altivec.vminub",905 genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>,906 Ty::UnsignedVector<1>>,907 genLibCall},908 {"__ppc_vec_min", "llvm.ppc.altivec.vminuh",909 genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>,910 Ty::UnsignedVector<2>>,911 genLibCall},912 {"__ppc_vec_min", "llvm.ppc.altivec.vminuw",913 genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>,914 Ty::UnsignedVector<4>>,915 genLibCall},916 {"__ppc_vec_min", "llvm.ppc.altivec.vminud",917 genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>,918 Ty::UnsignedVector<8>>,919 genLibCall},920 {"__ppc_vec_min", "llvm.ppc.vsx.xvminsp",921 genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>,922 genLibCall},923 {"__ppc_vec_min", "llvm.ppc.vsx.xvmindp",924 genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>,925 genLibCall},926 {"__ppc_vec_nmsub", "llvm.ppc.fnmsub.v4f32",927 genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>,928 Ty::RealVector<4>>,929 genLibCall},930 {"__ppc_vec_nmsub", "llvm.ppc.fnmsub.v2f64",931 genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>,932 Ty::RealVector<8>>,933 genLibCall},934};935 936const IntrinsicHandler *findPPCIntrinsicHandler(llvm::StringRef name) {937 auto compare = [](const IntrinsicHandler &ppcHandler, llvm::StringRef name) {938 return name.compare(ppcHandler.name) > 0;939 };940 auto result = llvm::lower_bound(ppcHandlers, name, compare);941 return result != std::end(ppcHandlers) && result->name == name ? result942 : nullptr;943}944 945using RtMap = Fortran::common::StaticMultimapView<MathOperation>;946static constexpr RtMap ppcMathOps(ppcMathOperations);947static_assert(ppcMathOps.Verify() && "map must be sorted");948 949std::pair<const MathOperation *, const MathOperation *>950checkPPCMathOperationsRange(llvm::StringRef name) {951 return ppcMathOps.equal_range(name);952}953 954// Helper functions for vector element ordering.955bool PPCIntrinsicLibrary::isBEVecElemOrderOnLE() {956 const auto triple{fir::getTargetTriple(builder.getModule())};957 return (triple.isLittleEndian() &&958 converter->getLoweringOptions().getNoPPCNativeVecElemOrder());959}960bool PPCIntrinsicLibrary::isNativeVecElemOrderOnLE() {961 const auto triple{fir::getTargetTriple(builder.getModule())};962 return (triple.isLittleEndian() &&963 !converter->getLoweringOptions().getNoPPCNativeVecElemOrder());964}965bool PPCIntrinsicLibrary::changeVecElemOrder() {966 const auto triple{fir::getTargetTriple(builder.getModule())};967 return (triple.isLittleEndian() !=968 converter->getLoweringOptions().getNoPPCNativeVecElemOrder());969}970 971static mlir::FunctionType genMmaVpFuncType(mlir::MLIRContext *context,972 int quadCnt, int pairCnt, int vecCnt,973 int intCnt = 0,974 int vecElemBitSize = 8,975 int intBitSize = 32) {976 // Constructs a function type with the following signature:977 // Result type: __vector_pair978 // Arguments:979 // quadCnt: number of arguments that has __vector_quad type, followed by980 // pairCnt: number of arguments that has __vector_pair type, followed by981 // vecCnt: number of arguments that has vector(integer) type, followed by982 // intCnt: number of arguments that has integer type983 // vecElemBitSize: specifies the size of vector elements in bits984 // intBitSize: specifies the size of integer arguments in bits985 auto vType{mlir::VectorType::get(986 128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))};987 auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))};988 auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))};989 auto iType{mlir::IntegerType::get(context, intBitSize)};990 llvm::SmallVector<mlir::Type> argTypes;991 for (int i = 0; i < quadCnt; ++i) {992 argTypes.push_back(vqType);993 }994 for (int i = 0; i < pairCnt; ++i) {995 argTypes.push_back(vpType);996 }997 for (int i = 0; i < vecCnt; ++i) {998 argTypes.push_back(vType);999 }1000 for (int i = 0; i < intCnt; ++i) {1001 argTypes.push_back(iType);1002 }1003 1004 return mlir::FunctionType::get(context, argTypes, {vpType});1005}1006 1007static mlir::FunctionType genMmaVqFuncType(mlir::MLIRContext *context,1008 int quadCnt, int pairCnt, int vecCnt,1009 int intCnt = 0,1010 int vecElemBitSize = 8,1011 int intBitSize = 32) {1012 // Constructs a function type with the following signature:1013 // Result type: __vector_quad1014 // Arguments:1015 // quadCnt: number of arguments that has __vector_quad type, followed by1016 // pairCnt: number of arguments that has __vector_pair type, followed by1017 // vecCnt: number of arguments that has vector(integer) type, followed by1018 // intCnt: number of arguments that has integer type1019 // vecElemBitSize: specifies the size of vector elements in bits1020 // intBitSize: specifies the size of integer arguments in bits1021 auto vType{mlir::VectorType::get(1022 128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))};1023 auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))};1024 auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))};1025 auto iType{mlir::IntegerType::get(context, intBitSize)};1026 llvm::SmallVector<mlir::Type> argTypes;1027 for (int i = 0; i < quadCnt; ++i) {1028 argTypes.push_back(vqType);1029 }1030 for (int i = 0; i < pairCnt; ++i) {1031 argTypes.push_back(vpType);1032 }1033 for (int i = 0; i < vecCnt; ++i) {1034 argTypes.push_back(vType);1035 }1036 for (int i = 0; i < intCnt; ++i) {1037 argTypes.push_back(iType);1038 }1039 1040 return mlir::FunctionType::get(context, argTypes, {vqType});1041}1042 1043mlir::FunctionType genMmaDisassembleFuncType(mlir::MLIRContext *context,1044 MMAOp mmaOp) {1045 auto vType{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))};1046 llvm::SmallVector<mlir::Type> members;1047 1048 if (mmaOp == MMAOp::DisassembleAcc) {1049 auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))};1050 members.push_back(vType);1051 members.push_back(vType);1052 members.push_back(vType);1053 members.push_back(vType);1054 auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)};1055 return mlir::FunctionType::get(context, {vqType}, {resType});1056 } else if (mmaOp == MMAOp::DisassemblePair) {1057 auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))};1058 members.push_back(vType);1059 members.push_back(vType);1060 auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)};1061 return mlir::FunctionType::get(context, {vpType}, {resType});1062 } else {1063 llvm_unreachable(1064 "Unsupported intrinsic code for function signature generator");1065 }1066}1067 1068//===----------------------------------------------------------------------===//1069// PowerPC specific intrinsic handlers.1070//===----------------------------------------------------------------------===//1071 1072// MTFSF, MTFSFI1073template <bool isImm>1074void PPCIntrinsicLibrary::genMtfsf(llvm::ArrayRef<fir::ExtendedValue> args) {1075 assert(args.size() == 2);1076 llvm::SmallVector<mlir::Value> scalarArgs;1077 for (const fir::ExtendedValue &arg : args)1078 if (arg.getUnboxed())1079 scalarArgs.emplace_back(fir::getBase(arg));1080 else1081 mlir::emitError(loc, "nonscalar intrinsic argument");1082 1083 mlir::FunctionType libFuncType;1084 mlir::func::FuncOp funcOp;1085 if (isImm) {1086 libFuncType = genFuncType<Ty::Void, Ty::Integer<4>, Ty::Integer<4>>(1087 builder.getContext(), builder);1088 funcOp = builder.createFunction(loc, "llvm.ppc.mtfsfi", libFuncType);1089 } else {1090 libFuncType = genFuncType<Ty::Void, Ty::Integer<4>, Ty::Real<8>>(1091 builder.getContext(), builder);1092 funcOp = builder.createFunction(loc, "llvm.ppc.mtfsf", libFuncType);1093 }1094 fir::CallOp::create(builder, loc, funcOp, scalarArgs);1095}1096 1097// VEC_ABS1098fir::ExtendedValue1099PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType,1100 llvm::ArrayRef<fir::ExtendedValue> args) {1101 assert(args.size() == 1);1102 auto context{builder.getContext()};1103 auto argBases{getBasesForArgs(args)};1104 auto vTypeInfo{getVecTypeFromFir(argBases[0])};1105 1106 mlir::func::FuncOp funcOp{nullptr};1107 mlir::FunctionType ftype;1108 llvm::StringRef fname{};1109 if (vTypeInfo.isFloat()) {1110 if (vTypeInfo.isFloat32()) {1111 fname = "llvm.fabs.v4f32";1112 ftype =1113 genFuncType<Ty::RealVector<4>, Ty::RealVector<4>>(context, builder);1114 } else if (vTypeInfo.isFloat64()) {1115 fname = "llvm.fabs.v2f64";1116 ftype =1117 genFuncType<Ty::RealVector<8>, Ty::RealVector<8>>(context, builder);1118 }1119 1120 funcOp = builder.createFunction(loc, fname, ftype);1121 auto callOp{fir::CallOp::create(builder, loc, funcOp, argBases[0])};1122 return callOp.getResult(0);1123 } else if (auto eleTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) {1124 // vec_abs(arg1) = max(0 - arg1, arg1)1125 1126 auto newVecTy{mlir::VectorType::get(vTypeInfo.len, eleTy)};1127 auto varg1{builder.createConvert(loc, newVecTy, argBases[0])};1128 // construct vector(0,..)1129 auto zeroVal{builder.createIntegerConstant(loc, eleTy, 0)};1130 auto vZero{1131 mlir::vector::BroadcastOp::create(builder, loc, newVecTy, zeroVal)};1132 auto zeroSubVarg1{mlir::arith::SubIOp::create(builder, loc, vZero, varg1)};1133 1134 mlir::func::FuncOp funcOp{nullptr};1135 switch (eleTy.getWidth()) {1136 case 8:1137 fname = "llvm.ppc.altivec.vmaxsb";1138 ftype = genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>,1139 Ty::IntegerVector<1>>(context, builder);1140 break;1141 case 16:1142 fname = "llvm.ppc.altivec.vmaxsh";1143 ftype = genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>,1144 Ty::IntegerVector<2>>(context, builder);1145 break;1146 case 32:1147 fname = "llvm.ppc.altivec.vmaxsw";1148 ftype = genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,1149 Ty::IntegerVector<4>>(context, builder);1150 break;1151 case 64:1152 fname = "llvm.ppc.altivec.vmaxsd";1153 ftype = genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>,1154 Ty::IntegerVector<8>>(context, builder);1155 break;1156 default:1157 llvm_unreachable("invalid integer size");1158 }1159 funcOp = builder.createFunction(loc, fname, ftype);1160 1161 mlir::Value args[] = {zeroSubVarg1, varg1};1162 auto callOp{fir::CallOp::create(builder, loc, funcOp, args)};1163 return builder.createConvert(loc, argBases[0].getType(),1164 callOp.getResult(0));1165 }1166 1167 llvm_unreachable("unknown vector type");1168}1169 1170// VEC_ADD, VEC_AND, VEC_SUB, VEC_MUL, VEC_XOR1171template <VecOp vop>1172fir::ExtendedValue PPCIntrinsicLibrary::genVecAddAndMulSubXor(1173 mlir::Type resultType, llvm::ArrayRef<fir::ExtendedValue> args) {1174 assert(args.size() == 2);1175 auto argBases{getBasesForArgs(args)};1176 auto argsTy{getTypesForArgs(argBases)};1177 assert(mlir::isa<fir::VectorType>(argsTy[0]) &&1178 mlir::isa<fir::VectorType>(argsTy[1]));1179 1180 auto vecTyInfo{getVecTypeFromFir(argBases[0])};1181 1182 const auto isInteger{mlir::isa<mlir::IntegerType>(vecTyInfo.eleTy)};1183 const auto isFloat{mlir::isa<mlir::FloatType>(vecTyInfo.eleTy)};1184 assert((isInteger || isFloat) && "unknown vector type");1185 1186 auto vargs{convertVecArgs(builder, loc, vecTyInfo, argBases)};1187 1188 mlir::Value r{nullptr};1189 switch (vop) {1190 case VecOp::Add:1191 if (isInteger)1192 r = mlir::arith::AddIOp::create(builder, loc, vargs[0], vargs[1]);1193 else if (isFloat)1194 r = mlir::arith::AddFOp::create(builder, loc, vargs[0], vargs[1]);1195 break;1196 case VecOp::Mul:1197 if (isInteger)1198 r = mlir::arith::MulIOp::create(builder, loc, vargs[0], vargs[1]);1199 else if (isFloat)1200 r = mlir::arith::MulFOp::create(builder, loc, vargs[0], vargs[1]);1201 break;1202 case VecOp::Sub:1203 if (isInteger)1204 r = mlir::arith::SubIOp::create(builder, loc, vargs[0], vargs[1]);1205 else if (isFloat)1206 r = mlir::arith::SubFOp::create(builder, loc, vargs[0], vargs[1]);1207 break;1208 case VecOp::And:1209 case VecOp::Xor: {1210 mlir::Value arg1{nullptr};1211 mlir::Value arg2{nullptr};1212 if (isInteger) {1213 arg1 = vargs[0];1214 arg2 = vargs[1];1215 } else if (isFloat) {1216 // bitcast the arguments to integer1217 auto wd{mlir::dyn_cast<mlir::FloatType>(vecTyInfo.eleTy).getWidth()};1218 auto ftype{builder.getIntegerType(wd)};1219 auto bcVecTy{mlir::VectorType::get(vecTyInfo.len, ftype)};1220 arg1 = mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[0]);1221 arg2 = mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[1]);1222 }1223 if (vop == VecOp::And)1224 r = mlir::arith::AndIOp::create(builder, loc, arg1, arg2);1225 else if (vop == VecOp::Xor)1226 r = mlir::arith::XOrIOp::create(builder, loc, arg1, arg2);1227 1228 if (isFloat)1229 r = mlir::vector::BitCastOp::create(builder, loc, vargs[0].getType(), r);1230 1231 break;1232 }1233 }1234 1235 return builder.createConvert(loc, argsTy[0], r);1236}1237 1238// VEC_ANY_GE1239template <VecOp vop>1240fir::ExtendedValue1241PPCIntrinsicLibrary::genVecAnyCompare(mlir::Type resultType,1242 llvm::ArrayRef<fir::ExtendedValue> args) {1243 assert(args.size() == 2);1244 assert(vop == VecOp::Anyge && "unknown vector compare operation");1245 auto argBases{getBasesForArgs(args)};1246 VecTypeInfo vTypeInfo{getVecTypeFromFir(argBases[0])};1247 [[maybe_unused]] const auto isSupportedTy{1248 mlir::isa<mlir::Float32Type, mlir::Float64Type, mlir::IntegerType>(1249 vTypeInfo.eleTy)};1250 assert(isSupportedTy && "unsupported vector type");1251 1252 // Constants for mapping CR6 bits to predicate result1253 enum { CR6_EQ_REV = 1, CR6_LT_REV = 3 };1254 1255 auto context{builder.getContext()};1256 1257 static std::map<std::pair<ParamTypeId, unsigned>,1258 std::pair<llvm::StringRef, mlir::FunctionType>>1259 uiBuiltin{1260 {std::make_pair(ParamTypeId::IntegerVector, 8),1261 std::make_pair(1262 "llvm.ppc.altivec.vcmpgtsb.p",1263 genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<1>,1264 Ty::IntegerVector<1>>(context, builder))},1265 {std::make_pair(ParamTypeId::IntegerVector, 16),1266 std::make_pair(1267 "llvm.ppc.altivec.vcmpgtsh.p",1268 genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<2>,1269 Ty::IntegerVector<2>>(context, builder))},1270 {std::make_pair(ParamTypeId::IntegerVector, 32),1271 std::make_pair(1272 "llvm.ppc.altivec.vcmpgtsw.p",1273 genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<4>,1274 Ty::IntegerVector<4>>(context, builder))},1275 {std::make_pair(ParamTypeId::IntegerVector, 64),1276 std::make_pair(1277 "llvm.ppc.altivec.vcmpgtsd.p",1278 genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<8>,1279 Ty::IntegerVector<8>>(context, builder))},1280 {std::make_pair(ParamTypeId::UnsignedVector, 8),1281 std::make_pair(1282 "llvm.ppc.altivec.vcmpgtub.p",1283 genFuncType<Ty::Integer<4>, Ty::Integer<4>,1284 Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>(1285 context, builder))},1286 {std::make_pair(ParamTypeId::UnsignedVector, 16),1287 std::make_pair(1288 "llvm.ppc.altivec.vcmpgtuh.p",1289 genFuncType<Ty::Integer<4>, Ty::Integer<4>,1290 Ty::UnsignedVector<2>, Ty::UnsignedVector<2>>(1291 context, builder))},1292 {std::make_pair(ParamTypeId::UnsignedVector, 32),1293 std::make_pair(1294 "llvm.ppc.altivec.vcmpgtuw.p",1295 genFuncType<Ty::Integer<4>, Ty::Integer<4>,1296 Ty::UnsignedVector<4>, Ty::UnsignedVector<4>>(1297 context, builder))},1298 {std::make_pair(ParamTypeId::UnsignedVector, 64),1299 std::make_pair(1300 "llvm.ppc.altivec.vcmpgtud.p",1301 genFuncType<Ty::Integer<4>, Ty::Integer<4>,1302 Ty::UnsignedVector<8>, Ty::UnsignedVector<8>>(1303 context, builder))},1304 };1305 1306 mlir::FunctionType ftype{nullptr};1307 llvm::StringRef fname;1308 const auto i32Ty{mlir::IntegerType::get(context, 32)};1309 llvm::SmallVector<mlir::Value> cmpArgs;1310 mlir::Value op{nullptr};1311 const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()};1312 1313 if (auto elementTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) {1314 std::pair<llvm::StringRef, mlir::FunctionType> bi;1315 bi = (elementTy.isUnsignedInteger())1316 ? uiBuiltin[std::pair(ParamTypeId::UnsignedVector, width)]1317 : uiBuiltin[std::pair(ParamTypeId::IntegerVector, width)];1318 1319 fname = std::get<0>(bi);1320 ftype = std::get<1>(bi);1321 1322 op = builder.createIntegerConstant(loc, i32Ty, CR6_LT_REV);1323 cmpArgs.emplace_back(op);1324 // reverse the argument order1325 cmpArgs.emplace_back(argBases[1]);1326 cmpArgs.emplace_back(argBases[0]);1327 } else if (vTypeInfo.isFloat()) {1328 if (vTypeInfo.isFloat32()) {1329 fname = "llvm.ppc.vsx.xvcmpgesp.p";1330 ftype = genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::RealVector<4>,1331 Ty::RealVector<4>>(context, builder);1332 } else {1333 fname = "llvm.ppc.vsx.xvcmpgedp.p";1334 ftype = genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::RealVector<8>,1335 Ty::RealVector<8>>(context, builder);1336 }1337 op = builder.createIntegerConstant(loc, i32Ty, CR6_EQ_REV);1338 cmpArgs.emplace_back(op);1339 cmpArgs.emplace_back(argBases[0]);1340 cmpArgs.emplace_back(argBases[1]);1341 }1342 assert((!fname.empty() && ftype) && "invalid type");1343 1344 mlir::func::FuncOp funcOp{builder.createFunction(loc, fname, ftype)};1345 auto callOp{fir::CallOp::create(builder, loc, funcOp, cmpArgs)};1346 return callOp.getResult(0);1347}1348 1349static std::pair<llvm::StringRef, mlir::FunctionType>1350getVecCmpFuncTypeAndName(VecTypeInfo &vTypeInfo, VecOp vop,1351 fir::FirOpBuilder &builder) {1352 auto context{builder.getContext()};1353 static std::map<std::pair<ParamTypeId, unsigned>,1354 std::pair<llvm::StringRef, mlir::FunctionType>>1355 iuBuiltinName{1356 {std::make_pair(ParamTypeId::IntegerVector, 8),1357 std::make_pair(1358 "llvm.ppc.altivec.vcmpgtsb",1359 genFuncType<Ty::UnsignedVector<1>, Ty::IntegerVector<1>,1360 Ty::IntegerVector<1>>(context, builder))},1361 {std::make_pair(ParamTypeId::IntegerVector, 16),1362 std::make_pair(1363 "llvm.ppc.altivec.vcmpgtsh",1364 genFuncType<Ty::UnsignedVector<2>, Ty::IntegerVector<2>,1365 Ty::IntegerVector<2>>(context, builder))},1366 {std::make_pair(ParamTypeId::IntegerVector, 32),1367 std::make_pair(1368 "llvm.ppc.altivec.vcmpgtsw",1369 genFuncType<Ty::UnsignedVector<4>, Ty::IntegerVector<4>,1370 Ty::IntegerVector<4>>(context, builder))},1371 {std::make_pair(ParamTypeId::IntegerVector, 64),1372 std::make_pair(1373 "llvm.ppc.altivec.vcmpgtsd",1374 genFuncType<Ty::UnsignedVector<8>, Ty::IntegerVector<8>,1375 Ty::IntegerVector<8>>(context, builder))},1376 {std::make_pair(ParamTypeId::UnsignedVector, 8),1377 std::make_pair(1378 "llvm.ppc.altivec.vcmpgtub",1379 genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>,1380 Ty::UnsignedVector<1>>(context, builder))},1381 {std::make_pair(ParamTypeId::UnsignedVector, 16),1382 std::make_pair(1383 "llvm.ppc.altivec.vcmpgtuh",1384 genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>,1385 Ty::UnsignedVector<2>>(context, builder))},1386 {std::make_pair(ParamTypeId::UnsignedVector, 32),1387 std::make_pair(1388 "llvm.ppc.altivec.vcmpgtuw",1389 genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>,1390 Ty::UnsignedVector<4>>(context, builder))},1391 {std::make_pair(ParamTypeId::UnsignedVector, 64),1392 std::make_pair(1393 "llvm.ppc.altivec.vcmpgtud",1394 genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>,1395 Ty::UnsignedVector<8>>(context, builder))}};1396 1397 // VSX only defines GE and GT builtins. Cmple and Cmplt use GE and GT with1398 // arguments revsered.1399 enum class Cmp { gtOrLt, geOrLe };1400 static std::map<std::pair<Cmp, int>,1401 std::pair<llvm::StringRef, mlir::FunctionType>>1402 rGBI{{std::make_pair(Cmp::geOrLe, 32),1403 std::make_pair("llvm.ppc.vsx.xvcmpgesp",1404 genFuncType<Ty::UnsignedVector<4>, Ty::RealVector<4>,1405 Ty::RealVector<4>>(context, builder))},1406 {std::make_pair(Cmp::geOrLe, 64),1407 std::make_pair("llvm.ppc.vsx.xvcmpgedp",1408 genFuncType<Ty::UnsignedVector<8>, Ty::RealVector<8>,1409 Ty::RealVector<8>>(context, builder))},1410 {std::make_pair(Cmp::gtOrLt, 32),1411 std::make_pair("llvm.ppc.vsx.xvcmpgtsp",1412 genFuncType<Ty::UnsignedVector<4>, Ty::RealVector<4>,1413 Ty::RealVector<4>>(context, builder))},1414 {std::make_pair(Cmp::gtOrLt, 64),1415 std::make_pair("llvm.ppc.vsx.xvcmpgtdp",1416 genFuncType<Ty::UnsignedVector<8>, Ty::RealVector<8>,1417 Ty::RealVector<8>>(context, builder))}};1418 1419 const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()};1420 std::pair<llvm::StringRef, mlir::FunctionType> specFunc;1421 if (auto elementTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy))1422 specFunc =1423 (elementTy.isUnsignedInteger())1424 ? iuBuiltinName[std::make_pair(ParamTypeId::UnsignedVector, width)]1425 : iuBuiltinName[std::make_pair(ParamTypeId::IntegerVector, width)];1426 else if (vTypeInfo.isFloat())1427 specFunc = (vop == VecOp::Cmpge || vop == VecOp::Cmple)1428 ? rGBI[std::make_pair(Cmp::geOrLe, width)]1429 : rGBI[std::make_pair(Cmp::gtOrLt, width)];1430 1431 assert(!std::get<0>(specFunc).empty() && "unknown builtin name");1432 assert(std::get<1>(specFunc) && "unknown function type");1433 return specFunc;1434}1435 1436// VEC_CMPGE, VEC_CMPGT, VEC_CMPLE, VEC_CMPLT1437template <VecOp vop>1438fir::ExtendedValue1439PPCIntrinsicLibrary::genVecCmp(mlir::Type resultType,1440 llvm::ArrayRef<fir::ExtendedValue> args) {1441 assert(args.size() == 2);1442 auto context{builder.getContext()};1443 auto argBases{getBasesForArgs(args)};1444 VecTypeInfo vecTyInfo{getVecTypeFromFir(argBases[0])};1445 auto varg{convertVecArgs(builder, loc, vecTyInfo, argBases)};1446 1447 std::pair<llvm::StringRef, mlir::FunctionType> funcTyNam{1448 getVecCmpFuncTypeAndName(vecTyInfo, vop, builder)};1449 1450 mlir::func::FuncOp funcOp = builder.createFunction(1451 loc, std::get<0>(funcTyNam), std::get<1>(funcTyNam));1452 1453 mlir::Value res{nullptr};1454 1455 if (auto eTy = mlir::dyn_cast<mlir::IntegerType>(vecTyInfo.eleTy)) {1456 constexpr int firstArg{0};1457 constexpr int secondArg{1};1458 std::map<VecOp, std::array<int, 2>> argOrder{1459 {VecOp::Cmpge, {secondArg, firstArg}},1460 {VecOp::Cmple, {firstArg, secondArg}},1461 {VecOp::Cmpgt, {firstArg, secondArg}},1462 {VecOp::Cmplt, {secondArg, firstArg}}};1463 1464 // Construct the function return type, unsigned vector, for conversion.1465 auto itype = mlir::IntegerType::get(context, eTy.getWidth(),1466 mlir::IntegerType::Unsigned);1467 auto returnType = fir::VectorType::get(vecTyInfo.len, itype);1468 1469 switch (vop) {1470 case VecOp::Cmpgt:1471 case VecOp::Cmplt: {1472 // arg1 > arg2 --> vcmpgt(arg1, arg2)1473 // arg1 < arg2 --> vcmpgt(arg2, arg1)1474 mlir::Value vargs[]{argBases[argOrder[vop][0]],1475 argBases[argOrder[vop][1]]};1476 auto callOp{fir::CallOp::create(builder, loc, funcOp, vargs)};1477 res = callOp.getResult(0);1478 break;1479 }1480 case VecOp::Cmpge:1481 case VecOp::Cmple: {1482 // arg1 >= arg2 --> vcmpge(arg2, arg1) xor vector(-1)1483 // arg1 <= arg2 --> vcmpge(arg1, arg2) xor vector(-1)1484 mlir::Value vargs[]{argBases[argOrder[vop][0]],1485 argBases[argOrder[vop][1]]};1486 1487 // Construct a constant vector(-1)1488 auto negOneVal{builder.createIntegerConstant(1489 loc, getConvertedElementType(context, eTy), -1)};1490 auto vNegOne{mlir::vector::BroadcastOp::create(1491 builder, loc, vecTyInfo.toMlirVectorType(context), negOneVal)};1492 1493 auto callOp{fir::CallOp::create(builder, loc, funcOp, vargs)};1494 mlir::Value callRes{callOp.getResult(0)};1495 auto vargs2{1496 convertVecArgs(builder, loc, vecTyInfo, mlir::ValueRange{callRes})};1497 auto xorRes{1498 mlir::arith::XOrIOp::create(builder, loc, vargs2[0], vNegOne)};1499 1500 res = builder.createConvert(loc, returnType, xorRes);1501 break;1502 }1503 default:1504 llvm_unreachable("Invalid vector operation for generator");1505 }1506 } else if (vecTyInfo.isFloat()) {1507 mlir::Value vargs[2];1508 switch (vop) {1509 case VecOp::Cmpge:1510 case VecOp::Cmpgt:1511 vargs[0] = argBases[0];1512 vargs[1] = argBases[1];1513 break;1514 case VecOp::Cmple:1515 case VecOp::Cmplt:1516 // Swap the arguments as xvcmpg[et] is used1517 vargs[0] = argBases[1];1518 vargs[1] = argBases[0];1519 break;1520 default:1521 llvm_unreachable("Invalid vector operation for generator");1522 }1523 auto callOp{fir::CallOp::create(builder, loc, funcOp, vargs)};1524 res = callOp.getResult(0);1525 } else1526 llvm_unreachable("invalid vector type");1527 1528 return res;1529}1530 1531static inline mlir::Value swapVectorWordPairs(fir::FirOpBuilder &builder,1532 mlir::Location loc,1533 mlir::Value arg) {1534 auto ty = arg.getType();1535 auto context{builder.getContext()};1536 auto vtype{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))};1537 1538 if (ty != vtype)1539 arg = mlir::LLVM::BitcastOp::create(builder, loc, vtype, arg).getResult();1540 1541 llvm::SmallVector<int64_t, 16> mask{4, 5, 6, 7, 0, 1, 2, 3,1542 12, 13, 14, 15, 8, 9, 10, 11};1543 arg = mlir::vector::ShuffleOp::create(builder, loc, arg, arg, mask);1544 if (ty != vtype)1545 arg = mlir::LLVM::BitcastOp::create(builder, loc, ty, arg);1546 return arg;1547}1548 1549// VEC_CONVERT, VEC_CTF, VEC_CVF1550template <VecOp vop>1551fir::ExtendedValue1552PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,1553 llvm::ArrayRef<fir::ExtendedValue> args) {1554 auto context{builder.getContext()};1555 auto argBases{getBasesForArgs(args)};1556 auto vecTyInfo{getVecTypeFromFir(argBases[0])};1557 auto mlirTy{vecTyInfo.toMlirVectorType(context)};1558 auto vArg1{builder.createConvert(loc, mlirTy, argBases[0])};1559 const auto i32Ty{mlir::IntegerType::get(context, 32)};1560 1561 switch (vop) {1562 case VecOp::Ctf: {1563 assert(args.size() == 2);1564 auto convArg{builder.createConvert(loc, i32Ty, argBases[1])};1565 auto eTy{mlir::dyn_cast<mlir::IntegerType>(vecTyInfo.eleTy)};1566 assert(eTy && "Unsupported vector type");1567 const auto isUnsigned{eTy.isUnsignedInteger()};1568 const auto width{eTy.getWidth()};1569 1570 if (width == 32) {1571 auto ftype{(isUnsigned)1572 ? genFuncType<Ty::RealVector<4>, Ty::UnsignedVector<4>,1573 Ty::Integer<4>>(context, builder)1574 : genFuncType<Ty::RealVector<4>, Ty::IntegerVector<4>,1575 Ty::Integer<4>>(context, builder)};1576 const llvm::StringRef fname{(isUnsigned) ? "llvm.ppc.altivec.vcfux"1577 : "llvm.ppc.altivec.vcfsx"};1578 auto funcOp{builder.createFunction(loc, fname, ftype)};1579 mlir::Value newArgs[] = {argBases[0], convArg};1580 auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};1581 1582 return callOp.getResult(0);1583 } else if (width == 64) {1584 auto fTy{mlir::Float64Type::get(context)};1585 auto ty{mlir::VectorType::get(2, fTy)};1586 1587 // vec_vtf(arg1, arg2) = fmul(1.0 / (1 << arg2), llvm.sitofp(arg1))1588 auto convOp{(isUnsigned)1589 ? mlir::LLVM::UIToFPOp::create(builder, loc, ty, vArg1)1590 : mlir::LLVM::SIToFPOp::create(builder, loc, ty, vArg1)};1591 1592 // construct vector<1./(1<<arg1), 1.0/(1<<arg1)>1593 auto constInt{mlir::dyn_cast_or_null<mlir::IntegerAttr>(1594 mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[1].getDefiningOp())1595 .getValue())};1596 assert(constInt && "expected integer constant argument");1597 double f{1.0 / (1 << constInt.getInt())};1598 llvm::SmallVector<double> vals{f, f};1599 auto constOp{mlir::arith::ConstantOp::create(1600 builder, loc, ty, builder.getF64VectorAttr(vals))};1601 1602 auto mulOp{mlir::LLVM::FMulOp::create(builder, loc, ty,1603 convOp->getResult(0), constOp)};1604 1605 return builder.createConvert(loc, fir::VectorType::get(2, fTy), mulOp);1606 }1607 llvm_unreachable("invalid element integer kind");1608 }1609 case VecOp::Convert: {1610 assert(args.size() == 2);1611 // resultType has mold type (if scalar) or element type (if array)1612 auto resTyInfo{getVecTypeFromFirType(resultType)};1613 auto moldTy{resTyInfo.toMlirVectorType(context)};1614 auto firTy{resTyInfo.toFirVectorType()};1615 1616 // vec_convert(v, mold) = bitcast v to "type of mold"1617 auto conv{mlir::LLVM::BitcastOp::create(builder, loc, moldTy, vArg1)};1618 1619 return builder.createConvert(loc, firTy, conv);1620 }1621 case VecOp::Cvf: {1622 assert(args.size() == 1);1623 1624 mlir::Value newArgs[]{vArg1};1625 if (vecTyInfo.isFloat32()) {1626 if (changeVecElemOrder())1627 newArgs[0] = swapVectorWordPairs(builder, loc, newArgs[0]);1628 1629 const llvm::StringRef fname{"llvm.ppc.vsx.xvcvspdp"};1630 auto ftype{1631 genFuncType<Ty::RealVector<8>, Ty::RealVector<4>>(context, builder)};1632 auto funcOp{builder.createFunction(loc, fname, ftype)};1633 auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};1634 1635 return callOp.getResult(0);1636 } else if (vecTyInfo.isFloat64()) {1637 const llvm::StringRef fname{"llvm.ppc.vsx.xvcvdpsp"};1638 auto ftype{1639 genFuncType<Ty::RealVector<4>, Ty::RealVector<8>>(context, builder)};1640 auto funcOp{builder.createFunction(loc, fname, ftype)};1641 newArgs[0] =1642 fir::CallOp::create(builder, loc, funcOp, newArgs).getResult(0);1643 auto fvf32Ty{newArgs[0].getType()};1644 auto f32type{mlir::Float32Type::get(context)};1645 auto mvf32Ty{mlir::VectorType::get(4, f32type)};1646 newArgs[0] = builder.createConvert(loc, mvf32Ty, newArgs[0]);1647 1648 if (changeVecElemOrder())1649 newArgs[0] = swapVectorWordPairs(builder, loc, newArgs[0]);1650 1651 return builder.createConvert(loc, fvf32Ty, newArgs[0]);1652 }1653 llvm_unreachable("invalid element integer kind");1654 }1655 default:1656 llvm_unreachable("Invalid vector operation for generator");1657 }1658}1659 1660static mlir::Value convertVectorElementOrder(fir::FirOpBuilder &builder,1661 mlir::Location loc,1662 VecTypeInfo vecInfo,1663 mlir::Value idx) {1664 mlir::Value numSub1{1665 builder.createIntegerConstant(loc, idx.getType(), vecInfo.len - 1)};1666 return mlir::LLVM::SubOp::create(builder, loc, idx.getType(), numSub1, idx);1667}1668 1669// VEC_EXTRACT1670fir::ExtendedValue1671PPCIntrinsicLibrary::genVecExtract(mlir::Type resultType,1672 llvm::ArrayRef<fir::ExtendedValue> args) {1673 assert(args.size() == 2);1674 auto argBases{getBasesForArgs(args)};1675 auto argTypes{getTypesForArgs(argBases)};1676 auto vecTyInfo{getVecTypeFromFir(argBases[0])};1677 1678 auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};1679 auto varg0{builder.createConvert(loc, mlirTy, argBases[0])};1680 1681 // arg2 modulo the number of elements in arg1 to determine the element1682 // position1683 auto numEle{builder.createIntegerConstant(loc, argTypes[1], vecTyInfo.len)};1684 mlir::Value uremOp{1685 mlir::LLVM::URemOp::create(builder, loc, argBases[1], numEle)};1686 1687 if (!isNativeVecElemOrderOnLE())1688 uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);1689 1690 mlir::Value index = builder.createOrFold<mlir::index::CastUOp>(1691 loc, builder.getIndexType(), uremOp);1692 return mlir::vector::ExtractOp::create(builder, loc, varg0, index);1693}1694 1695// VEC_INSERT1696fir::ExtendedValue1697PPCIntrinsicLibrary::genVecInsert(mlir::Type resultType,1698 llvm::ArrayRef<fir::ExtendedValue> args) {1699 assert(args.size() == 3);1700 auto argBases{getBasesForArgs(args)};1701 auto argTypes{getTypesForArgs(argBases)};1702 auto vecTyInfo{getVecTypeFromFir(argBases[1])};1703 auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};1704 auto varg1{builder.createConvert(loc, mlirTy, argBases[1])};1705 1706 auto numEle{builder.createIntegerConstant(loc, argTypes[2], vecTyInfo.len)};1707 mlir::Value uremOp{1708 mlir::LLVM::URemOp::create(builder, loc, argBases[2], numEle)};1709 1710 if (!isNativeVecElemOrderOnLE())1711 uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);1712 1713 mlir::Value index = builder.createOrFold<mlir::index::CastUOp>(1714 loc, builder.getIndexType(), uremOp);1715 mlir::Value res =1716 mlir::vector::InsertOp::create(builder, loc, argBases[0], varg1, index);1717 return fir::ConvertOp::create(builder, loc, vecTyInfo.toFirVectorType(), res);1718}1719 1720// VEC_MERGEH, VEC_MERGEL1721template <VecOp vop>1722fir::ExtendedValue1723PPCIntrinsicLibrary::genVecMerge(mlir::Type resultType,1724 llvm::ArrayRef<fir::ExtendedValue> args) {1725 assert(args.size() == 2);1726 auto argBases{getBasesForArgs(args)};1727 auto vecTyInfo{getVecTypeFromFir(argBases[0])};1728 llvm::SmallVector<int64_t, 16> mMask; // native vector element order mask1729 llvm::SmallVector<int64_t, 16> rMask; // non-native vector element order mask1730 1731 switch (vop) {1732 case VecOp::Mergeh: {1733 switch (vecTyInfo.len) {1734 case 2: {1735 enum { V1 = 0, V2 = 2 };1736 mMask = {V1 + 0, V2 + 0};1737 rMask = {V2 + 1, V1 + 1};1738 break;1739 }1740 case 4: {1741 enum { V1 = 0, V2 = 4 };1742 mMask = {V1 + 0, V2 + 0, V1 + 1, V2 + 1};1743 rMask = {V2 + 2, V1 + 2, V2 + 3, V1 + 3};1744 break;1745 }1746 case 8: {1747 enum { V1 = 0, V2 = 8 };1748 mMask = {V1 + 0, V2 + 0, V1 + 1, V2 + 1, V1 + 2, V2 + 2, V1 + 3, V2 + 3};1749 rMask = {V2 + 4, V1 + 4, V2 + 5, V1 + 5, V2 + 6, V1 + 6, V2 + 7, V1 + 7};1750 break;1751 }1752 case 16:1753 mMask = {0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13,1754 0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17};1755 rMask = {0x18, 0x08, 0x19, 0x09, 0x1A, 0x0A, 0x1B, 0x0B,1756 0x1C, 0x0C, 0x1D, 0x0D, 0x1E, 0x0E, 0x1F, 0x0F};1757 break;1758 default:1759 llvm_unreachable("unexpected vector length");1760 }1761 break;1762 }1763 case VecOp::Mergel: {1764 switch (vecTyInfo.len) {1765 case 2: {1766 enum { V1 = 0, V2 = 2 };1767 mMask = {V1 + 1, V2 + 1};1768 rMask = {V2 + 0, V1 + 0};1769 break;1770 }1771 case 4: {1772 enum { V1 = 0, V2 = 4 };1773 mMask = {V1 + 2, V2 + 2, V1 + 3, V2 + 3};1774 rMask = {V2 + 0, V1 + 0, V2 + 1, V1 + 1};1775 break;1776 }1777 case 8: {1778 enum { V1 = 0, V2 = 8 };1779 mMask = {V1 + 4, V2 + 4, V1 + 5, V2 + 5, V1 + 6, V2 + 6, V1 + 7, V2 + 7};1780 rMask = {V2 + 0, V1 + 0, V2 + 1, V1 + 1, V2 + 2, V1 + 2, V2 + 3, V1 + 3};1781 break;1782 }1783 case 16:1784 mMask = {0x08, 0x18, 0x09, 0x19, 0x0A, 0x1A, 0x0B, 0x1B,1785 0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F};1786 rMask = {0x10, 0x00, 0x11, 0x01, 0x12, 0x02, 0x13, 0x03,1787 0x14, 0x04, 0x15, 0x05, 0x16, 0x06, 0x17, 0x07};1788 break;1789 default:1790 llvm_unreachable("unexpected vector length");1791 }1792 break;1793 }1794 default:1795 llvm_unreachable("invalid vector operation for generator");1796 }1797 1798 auto vargs{convertVecArgs(builder, loc, vecTyInfo, argBases)};1799 1800 llvm::SmallVector<int64_t, 16> &mergeMask =1801 (isBEVecElemOrderOnLE()) ? rMask : mMask;1802 1803 auto callOp{mlir::vector::ShuffleOp::create(builder, loc, vargs[0], vargs[1],1804 mergeMask)};1805 return builder.createConvert(loc, resultType, callOp);1806}1807 1808static mlir::Value addOffsetToAddress(fir::FirOpBuilder &builder,1809 mlir::Location loc, mlir::Value baseAddr,1810 mlir::Value offset) {1811 auto typeExtent{fir::SequenceType::getUnknownExtent()};1812 // Construct an !fir.ref<!ref.array<?xi8>> type1813 auto arrRefTy{builder.getRefType(fir::SequenceType::get(1814 {typeExtent}, mlir::IntegerType::get(builder.getContext(), 8)))};1815 // Convert arg to !fir.ref<!ref.array<?xi8>>1816 auto resAddr{fir::ConvertOp::create(builder, loc, arrRefTy, baseAddr)};1817 1818 return fir::CoordinateOp::create(builder, loc, arrRefTy, resAddr, offset);1819}1820 1821static mlir::Value reverseVectorElements(fir::FirOpBuilder &builder,1822 mlir::Location loc, mlir::Value v,1823 int64_t len) {1824 assert(mlir::isa<mlir::VectorType>(v.getType()));1825 assert(len > 0);1826 llvm::SmallVector<int64_t, 16> mask;1827 for (int64_t i = 0; i < len; ++i) {1828 mask.push_back(len - 1 - i);1829 }1830 auto undefVec{fir::UndefOp::create(builder, loc, v.getType())};1831 return mlir::vector::ShuffleOp::create(builder, loc, v, undefVec, mask);1832}1833 1834static mlir::NamedAttribute getAlignmentAttr(fir::FirOpBuilder &builder,1835 const int val) {1836 auto i64ty{mlir::IntegerType::get(builder.getContext(), 64)};1837 auto alignAttr{mlir::IntegerAttr::get(i64ty, val)};1838 return builder.getNamedAttr("alignment", alignAttr);1839}1840 1841fir::ExtendedValue1842PPCIntrinsicLibrary::genVecXlGrp(mlir::Type resultType,1843 llvm::ArrayRef<fir::ExtendedValue> args) {1844 VecTypeInfo vecTyInfo{getVecTypeFromFirType(resultType)};1845 switch (vecTyInfo.eleTy.getIntOrFloatBitWidth()) {1846 case 8:1847 // vec_xlb11848 return genVecLdNoCallGrp<VecOp::Xl>(resultType, args);1849 case 16:1850 // vec_xlh81851 return genVecLdNoCallGrp<VecOp::Xl>(resultType, args);1852 case 32:1853 // vec_xlw41854 return genVecLdCallGrp<VecOp::Xlw4>(resultType, args);1855 case 64:1856 // vec_xld21857 return genVecLdCallGrp<VecOp::Xld2>(resultType, args);1858 default:1859 llvm_unreachable("invalid kind");1860 }1861 llvm_unreachable("invalid vector operation for generator");1862}1863 1864template <VecOp vop>1865fir::ExtendedValue PPCIntrinsicLibrary::genVecLdNoCallGrp(1866 mlir::Type resultType, llvm::ArrayRef<fir::ExtendedValue> args) {1867 assert(args.size() == 2);1868 auto arg0{getBase(args[0])};1869 auto arg1{getBase(args[1])};1870 1871 auto vecTyInfo{getVecTypeFromFirType(resultType)};1872 auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};1873 auto firTy{vecTyInfo.toFirVectorType()};1874 1875 // Add the %val of arg0 to %addr of arg11876 auto addr{addOffsetToAddress(builder, loc, arg1, arg0)};1877 1878 const auto triple{fir::getTargetTriple(builder.getModule())};1879 // Need to get align 1.1880 auto result{fir::LoadOp::create(builder, loc, mlirTy, addr,1881 getAlignmentAttr(builder, 1))};1882 if ((vop == VecOp::Xl && isBEVecElemOrderOnLE()) ||1883 (vop == VecOp::Xlbe && triple.isLittleEndian()))1884 return builder.createConvert(1885 loc, firTy, reverseVectorElements(builder, loc, result, vecTyInfo.len));1886 1887 return builder.createConvert(loc, firTy, result);1888}1889 1890// VEC_LD, VEC_LDE, VEC_LDL, VEC_LXVP, VEC_XLD2, VEC_XLW41891template <VecOp vop>1892fir::ExtendedValue1893PPCIntrinsicLibrary::genVecLdCallGrp(mlir::Type resultType,1894 llvm::ArrayRef<fir::ExtendedValue> args) {1895 assert(args.size() == 2);1896 auto context{builder.getContext()};1897 auto arg0{getBase(args[0])};1898 auto arg1{getBase(args[1])};1899 1900 // Prepare the return type in FIR.1901 auto vecResTyInfo{getVecTypeFromFirType(resultType)};1902 auto mlirTy{vecResTyInfo.toMlirVectorType(context)};1903 auto firTy{vecResTyInfo.toFirVectorType()};1904 1905 // llvm.ppc.altivec.lvx* returns <4xi32>1906 // Others, like "llvm.ppc.altivec.lvebx" too if arg2 is not of Integer type1907 const auto i32Ty{mlir::IntegerType::get(builder.getContext(), 32)};1908 const auto mVecI32Ty{mlir::VectorType::get(4, i32Ty)};1909 1910 // For vec_ld, need to convert arg0 from i64 to i321911 if (vop == VecOp::Ld && arg0.getType().getIntOrFloatBitWidth() == 64)1912 arg0 = builder.createConvert(loc, i32Ty, arg0);1913 1914 // Add the %val of arg0 to %addr of arg11915 auto addr{addOffsetToAddress(builder, loc, arg1, arg0)};1916 llvm::SmallVector<mlir::Value, 4> parsedArgs{addr};1917 1918 mlir::Type intrinResTy{nullptr};1919 llvm::StringRef fname{};1920 switch (vop) {1921 case VecOp::Ld:1922 fname = "llvm.ppc.altivec.lvx";1923 intrinResTy = mVecI32Ty;1924 break;1925 case VecOp::Lde:1926 switch (vecResTyInfo.eleTy.getIntOrFloatBitWidth()) {1927 case 8:1928 fname = "llvm.ppc.altivec.lvebx";1929 intrinResTy = mlirTy;1930 break;1931 case 16:1932 fname = "llvm.ppc.altivec.lvehx";1933 intrinResTy = mlirTy;1934 break;1935 case 32:1936 fname = "llvm.ppc.altivec.lvewx";1937 if (mlir::isa<mlir::IntegerType>(vecResTyInfo.eleTy))1938 intrinResTy = mlirTy;1939 else1940 intrinResTy = mVecI32Ty;1941 break;1942 default:1943 llvm_unreachable("invalid vector for vec_lde");1944 }1945 break;1946 case VecOp::Ldl:1947 fname = "llvm.ppc.altivec.lvxl";1948 intrinResTy = mVecI32Ty;1949 break;1950 case VecOp::Lxvp:1951 fname = "llvm.ppc.vsx.lxvp";1952 intrinResTy = fir::VectorType::get(256, mlir::IntegerType::get(context, 1));1953 break;1954 case VecOp::Xld2: {1955 fname = isBEVecElemOrderOnLE() ? "llvm.ppc.vsx.lxvd2x.be"1956 : "llvm.ppc.vsx.lxvd2x";1957 // llvm.ppc.altivec.lxvd2x* returns <2 x double>1958 intrinResTy = mlir::VectorType::get(2, mlir::Float64Type::get(context));1959 } break;1960 case VecOp::Xlw4:1961 fname = isBEVecElemOrderOnLE() ? "llvm.ppc.vsx.lxvw4x.be"1962 : "llvm.ppc.vsx.lxvw4x";1963 // llvm.ppc.altivec.lxvw4x* returns <4xi32>1964 intrinResTy = mVecI32Ty;1965 break;1966 default:1967 llvm_unreachable("invalid vector operation for generator");1968 }1969 1970 auto funcType{1971 mlir::FunctionType::get(context, {addr.getType()}, {intrinResTy})};1972 auto funcOp{builder.createFunction(loc, fname, funcType)};1973 auto result{1974 fir::CallOp::create(builder, loc, funcOp, parsedArgs).getResult(0)};1975 1976 if (vop == VecOp::Lxvp)1977 return result;1978 1979 if (intrinResTy != mlirTy)1980 result = mlir::vector::BitCastOp::create(builder, loc, mlirTy, result);1981 1982 if (vop != VecOp::Xld2 && vop != VecOp::Xlw4 && isBEVecElemOrderOnLE())1983 return builder.createConvert(1984 loc, firTy,1985 reverseVectorElements(builder, loc, result, vecResTyInfo.len));1986 1987 return builder.createConvert(loc, firTy, result);1988}1989 1990// VEC_LVSL, VEC_LVSR1991template <VecOp vop>1992fir::ExtendedValue1993PPCIntrinsicLibrary::genVecLvsGrp(mlir::Type resultType,1994 llvm::ArrayRef<fir::ExtendedValue> args) {1995 assert(args.size() == 2);1996 auto context{builder.getContext()};1997 auto arg0{getBase(args[0])};1998 auto arg1{getBase(args[1])};1999 2000 auto vecTyInfo{getVecTypeFromFirType(resultType)};2001 auto mlirTy{vecTyInfo.toMlirVectorType(context)};2002 auto firTy{vecTyInfo.toFirVectorType()};2003 2004 // Convert arg0 to i64 type if needed2005 auto i64ty{mlir::IntegerType::get(context, 64)};2006 if (arg0.getType() != i64ty)2007 arg0 = fir::ConvertOp::create(builder, loc, i64ty, arg0);2008 2009 // offset is modulo 16, so shift left 56 bits and then right 56 bits to clear2010 // upper 56 bit while preserving sign2011 auto shiftVal{builder.createIntegerConstant(loc, i64ty, 56)};2012 auto offset{mlir::arith::ShLIOp::create(builder, loc, arg0, shiftVal)};2013 auto offset2{mlir::arith::ShRSIOp::create(builder, loc, offset, shiftVal)};2014 2015 // Add the offsetArg to %addr of arg12016 auto addr{addOffsetToAddress(builder, loc, arg1, offset2)};2017 llvm::SmallVector<mlir::Value, 4> parsedArgs{addr};2018 2019 llvm::StringRef fname{};2020 switch (vop) {2021 case VecOp::Lvsl:2022 fname = "llvm.ppc.altivec.lvsl";2023 break;2024 case VecOp::Lvsr:2025 fname = "llvm.ppc.altivec.lvsr";2026 break;2027 default:2028 llvm_unreachable("invalid vector operation for generator");2029 }2030 auto funcType{mlir::FunctionType::get(context, {addr.getType()}, {mlirTy})};2031 auto funcOp{builder.createFunction(loc, fname, funcType)};2032 auto result{2033 fir::CallOp::create(builder, loc, funcOp, parsedArgs).getResult(0)};2034 2035 if (isNativeVecElemOrderOnLE())2036 return builder.createConvert(2037 loc, firTy, reverseVectorElements(builder, loc, result, vecTyInfo.len));2038 2039 return builder.createConvert(loc, firTy, result);2040}2041 2042// VEC_NMADD, VEC_MSUB2043template <VecOp vop>2044fir::ExtendedValue2045PPCIntrinsicLibrary::genVecNmaddMsub(mlir::Type resultType,2046 llvm::ArrayRef<fir::ExtendedValue> args) {2047 assert(args.size() == 3);2048 auto context{builder.getContext()};2049 auto argBases{getBasesForArgs(args)};2050 auto vTypeInfo{getVecTypeFromFir(argBases[0])};2051 auto newArgs{convertVecArgs(builder, loc, vTypeInfo, argBases)};2052 const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()};2053 2054 static std::map<int, std::pair<llvm::StringRef, mlir::FunctionType>> fmaMap{2055 {32,2056 std::make_pair(2057 "llvm.fma.v4f32",2058 genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>(2059 context, builder))},2060 {64,2061 std::make_pair(2062 "llvm.fma.v2f64",2063 genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>(2064 context, builder))}};2065 2066 auto funcOp{builder.createFunction(loc, std::get<0>(fmaMap[width]),2067 std::get<1>(fmaMap[width]))};2068 if (vop == VecOp::Nmadd) {2069 // vec_nmadd(arg1, arg2, arg3) = -fma(arg1, arg2, arg3)2070 auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};2071 2072 // We need to convert fir.vector to MLIR vector to use fneg and then back2073 // to fir.vector to store.2074 auto vCall{builder.createConvert(loc, vTypeInfo.toMlirVectorType(context),2075 callOp.getResult(0))};2076 auto neg{mlir::arith::NegFOp::create(builder, loc, vCall)};2077 return builder.createConvert(loc, vTypeInfo.toFirVectorType(), neg);2078 } else if (vop == VecOp::Msub) {2079 // vec_msub(arg1, arg2, arg3) = fma(arg1, arg2, -arg3)2080 newArgs[2] = mlir::arith::NegFOp::create(builder, loc, newArgs[2]);2081 2082 auto callOp{fir::CallOp::create(builder, loc, funcOp, newArgs)};2083 return callOp.getResult(0);2084 }2085 llvm_unreachable("Invalid vector operation for generator");2086}2087 2088// VEC_PERM, VEC_PERMI2089template <VecOp vop>2090fir::ExtendedValue2091PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType,2092 llvm::ArrayRef<fir::ExtendedValue> args) {2093 assert(args.size() == 3);2094 auto context{builder.getContext()};2095 auto argBases{getBasesForArgs(args)};2096 auto argTypes{getTypesForArgs(argBases)};2097 auto vecTyInfo{getVecTypeFromFir(argBases[0])};2098 auto mlirTy{vecTyInfo.toMlirVectorType(context)};2099 2100 auto vi32Ty{mlir::VectorType::get(4, mlir::IntegerType::get(context, 32))};2101 auto vf64Ty{mlir::VectorType::get(2, mlir::Float64Type::get(context))};2102 2103 auto mArg0{builder.createConvert(loc, mlirTy, argBases[0])};2104 auto mArg1{builder.createConvert(loc, mlirTy, argBases[1])};2105 2106 switch (vop) {2107 case VecOp::Perm: {2108 VecTypeInfo maskVecTyInfo{getVecTypeFromFir(argBases[2])};2109 auto mlirMaskTy{maskVecTyInfo.toMlirVectorType(context)};2110 auto mMask{builder.createConvert(loc, mlirMaskTy, argBases[2])};2111 2112 if (mlirTy != vi32Ty) {2113 mArg0 = mlir::LLVM::BitcastOp::create(builder, loc, vi32Ty, mArg0)2114 .getResult();2115 mArg1 = mlir::LLVM::BitcastOp::create(builder, loc, vi32Ty, mArg1)2116 .getResult();2117 }2118 2119 auto funcOp{builder.createFunction(2120 loc, "llvm.ppc.altivec.vperm",2121 genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,2122 Ty::IntegerVector<4>, Ty::IntegerVector<1>>(context,2123 builder))};2124 2125 llvm::SmallVector<mlir::Value> newArgs;2126 if (isNativeVecElemOrderOnLE()) {2127 auto i8Ty{mlir::IntegerType::get(context, 8)};2128 auto v8Ty{mlir::VectorType::get(16, i8Ty)};2129 auto negOne{builder.createMinusOneInteger(loc, i8Ty)};2130 auto vNegOne{2131 mlir::vector::BroadcastOp::create(builder, loc, v8Ty, negOne)};2132 2133 mMask = mlir::arith::XOrIOp::create(builder, loc, mMask, vNegOne);2134 newArgs = {mArg1, mArg0, mMask};2135 } else {2136 newArgs = {mArg0, mArg1, mMask};2137 }2138 2139 auto res{fir::CallOp::create(builder, loc, funcOp, newArgs).getResult(0)};2140 2141 if (res.getType() != argTypes[0]) {2142 // fir.call llvm.ppc.altivec.vperm returns !fir.vector<i4:32>2143 // convert the result back to the original type2144 res = builder.createConvert(loc, vi32Ty, res);2145 if (mlirTy != vi32Ty)2146 res = mlir::LLVM::BitcastOp::create(builder, loc, mlirTy, res)2147 .getResult();2148 }2149 return builder.createConvert(loc, resultType, res);2150 }2151 case VecOp::Permi: {2152 // arg3 is a constant2153 auto constIntOp{mlir::dyn_cast_or_null<mlir::IntegerAttr>(2154 mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[2].getDefiningOp())2155 .getValue())};2156 assert(constIntOp && "expected integer constant argument");2157 auto constInt{constIntOp.getInt()};2158 // arg1, arg2, and result type share same VecTypeInfo2159 if (vecTyInfo.isFloat()) {2160 mArg0 = mlir::LLVM::BitcastOp::create(builder, loc, vf64Ty, mArg0)2161 .getResult();2162 mArg1 = mlir::LLVM::BitcastOp::create(builder, loc, vf64Ty, mArg1)2163 .getResult();2164 }2165 2166 llvm::SmallVector<int64_t, 2> nMask; // native vector element order mask2167 llvm::SmallVector<int64_t, 2> rMask; // non-native vector element order mask2168 enum { V1 = 0, V2 = 2 };2169 switch (constInt) {2170 case 0:2171 nMask = {V1 + 0, V2 + 0};2172 rMask = {V2 + 1, V1 + 1};2173 break;2174 case 1:2175 nMask = {V1 + 0, V2 + 1};2176 rMask = {V2 + 0, V1 + 1};2177 break;2178 case 2:2179 nMask = {V1 + 1, V2 + 0};2180 rMask = {V2 + 1, V1 + 0};2181 break;2182 case 3:2183 nMask = {V1 + 1, V2 + 1};2184 rMask = {V2 + 0, V1 + 0};2185 break;2186 default:2187 llvm_unreachable("unexpected arg3 value for vec_permi");2188 }2189 2190 llvm::SmallVector<int64_t, 2> mask =2191 (isBEVecElemOrderOnLE()) ? rMask : nMask;2192 auto res{mlir::vector::ShuffleOp::create(builder, loc, mArg0, mArg1, mask)};2193 if (res.getType() != mlirTy) {2194 auto cast{mlir::LLVM::BitcastOp::create(builder, loc, mlirTy, res)};2195 return builder.createConvert(loc, resultType, cast);2196 }2197 return builder.createConvert(loc, resultType, res);2198 }2199 default:2200 llvm_unreachable("invalid vector operation for generator");2201 }2202}2203 2204// VEC_SEL2205fir::ExtendedValue2206PPCIntrinsicLibrary::genVecSel(mlir::Type resultType,2207 llvm::ArrayRef<fir::ExtendedValue> args) {2208 assert(args.size() == 3);2209 auto argBases{getBasesForArgs(args)};2210 llvm::SmallVector<VecTypeInfo, 4> vecTyInfos;2211 for (size_t i = 0; i < argBases.size(); i++) {2212 vecTyInfos.push_back(getVecTypeFromFir(argBases[i]));2213 }2214 auto vargs{convertVecArgs(builder, loc, vecTyInfos, argBases)};2215 2216 auto i8Ty{mlir::IntegerType::get(builder.getContext(), 8)};2217 auto negOne{builder.createMinusOneInteger(loc, i8Ty)};2218 2219 // construct a constant <16 x i8> vector with value -1 for bitcast2220 auto bcVecTy{mlir::VectorType::get(16, i8Ty)};2221 auto vNegOne{2222 mlir::vector::BroadcastOp::create(builder, loc, bcVecTy, negOne)};2223 2224 // bitcast arguments to bcVecTy2225 auto arg1{mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[0])};2226 auto arg2{mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[1])};2227 auto arg3{mlir::vector::BitCastOp::create(builder, loc, bcVecTy, vargs[2])};2228 2229 // vec_sel(arg1, arg2, arg3) =2230 // (arg2 and arg3) or (arg1 and (arg3 xor vector(-1,...)))2231 auto comp{mlir::arith::XOrIOp::create(builder, loc, arg3, vNegOne)};2232 auto a1AndComp{mlir::arith::AndIOp::create(builder, loc, arg1, comp)};2233 auto a1OrA2{mlir::arith::AndIOp::create(builder, loc, arg2, arg3)};2234 auto res{mlir::arith::OrIOp::create(builder, loc, a1AndComp, a1OrA2)};2235 2236 auto bcRes{2237 mlir::vector::BitCastOp::create(builder, loc, vargs[0].getType(), res)};2238 2239 return builder.createConvert(loc, vecTyInfos[0].toFirVectorType(), bcRes);2240}2241 2242// VEC_SL, VEC_SLD, VEC_SLDW, VEC_SLL, VEC_SLO, VEC_SR, VEC_SRL, VEC_SRO2243template <VecOp vop>2244fir::ExtendedValue2245PPCIntrinsicLibrary::genVecShift(mlir::Type resultType,2246 llvm::ArrayRef<fir::ExtendedValue> args) {2247 auto context{builder.getContext()};2248 auto argBases{getBasesForArgs(args)};2249 auto argTypes{getTypesForArgs(argBases)};2250 2251 llvm::SmallVector<VecTypeInfo, 2> vecTyInfoArgs;2252 vecTyInfoArgs.push_back(getVecTypeFromFir(argBases[0]));2253 vecTyInfoArgs.push_back(getVecTypeFromFir(argBases[1]));2254 2255 // Convert the first two arguments to MLIR vectors2256 llvm::SmallVector<mlir::Type, 2> mlirTyArgs;2257 mlirTyArgs.push_back(vecTyInfoArgs[0].toMlirVectorType(context));2258 mlirTyArgs.push_back(vecTyInfoArgs[1].toMlirVectorType(context));2259 2260 llvm::SmallVector<mlir::Value, 2> mlirVecArgs;2261 mlirVecArgs.push_back(builder.createConvert(loc, mlirTyArgs[0], argBases[0]));2262 mlirVecArgs.push_back(builder.createConvert(loc, mlirTyArgs[1], argBases[1]));2263 2264 mlir::Value shftRes{nullptr};2265 2266 if (vop == VecOp::Sl || vop == VecOp::Sr) {2267 assert(args.size() == 2);2268 // Construct the mask2269 auto width{2270 mlir::dyn_cast<mlir::IntegerType>(vecTyInfoArgs[1].eleTy).getWidth()};2271 auto vecVal{builder.createIntegerConstant(2272 loc, getConvertedElementType(context, vecTyInfoArgs[0].eleTy), width)};2273 auto mask{2274 mlir::vector::BroadcastOp::create(builder, loc, mlirTyArgs[1], vecVal)};2275 auto shft{mlir::arith::RemUIOp::create(builder, loc, mlirVecArgs[1], mask)};2276 2277 mlir::Value res{nullptr};2278 if (vop == VecOp::Sr)2279 res = mlir::arith::ShRUIOp::create(builder, loc, mlirVecArgs[0], shft);2280 else if (vop == VecOp::Sl)2281 res = mlir::arith::ShLIOp::create(builder, loc, mlirVecArgs[0], shft);2282 2283 shftRes = builder.createConvert(loc, argTypes[0], res);2284 } else if (vop == VecOp::Sll || vop == VecOp::Slo || vop == VecOp::Srl ||2285 vop == VecOp::Sro) {2286 assert(args.size() == 2);2287 2288 // Bitcast to vector<4xi32>2289 auto bcVecTy{mlir::VectorType::get(4, builder.getIntegerType(32))};2290 if (mlirTyArgs[0] != bcVecTy)2291 mlirVecArgs[0] = mlir::vector::BitCastOp::create(builder, loc, bcVecTy,2292 mlirVecArgs[0]);2293 if (mlirTyArgs[1] != bcVecTy)2294 mlirVecArgs[1] = mlir::vector::BitCastOp::create(builder, loc, bcVecTy,2295 mlirVecArgs[1]);2296 2297 llvm::StringRef funcName;2298 switch (vop) {2299 case VecOp::Srl:2300 funcName = "llvm.ppc.altivec.vsr";2301 break;2302 case VecOp::Sro:2303 funcName = "llvm.ppc.altivec.vsro";2304 break;2305 case VecOp::Sll:2306 funcName = "llvm.ppc.altivec.vsl";2307 break;2308 case VecOp::Slo:2309 funcName = "llvm.ppc.altivec.vslo";2310 break;2311 default:2312 llvm_unreachable("unknown vector shift operation");2313 }2314 auto funcTy{genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,2315 Ty::IntegerVector<4>>(context, builder)};2316 mlir::func::FuncOp funcOp{builder.createFunction(loc, funcName, funcTy)};2317 auto callOp{fir::CallOp::create(builder, loc, funcOp, mlirVecArgs)};2318 2319 // If the result vector type is different from the original type, need2320 // to convert to mlir vector, bitcast and then convert back to fir vector.2321 if (callOp.getResult(0).getType() != argTypes[0]) {2322 auto res = builder.createConvert(loc, bcVecTy, callOp.getResult(0));2323 res = mlir::vector::BitCastOp::create(builder, loc, mlirTyArgs[0], res);2324 shftRes = builder.createConvert(loc, argTypes[0], res);2325 } else {2326 shftRes = callOp.getResult(0);2327 }2328 } else if (vop == VecOp::Sld || vop == VecOp::Sldw) {2329 assert(args.size() == 3);2330 auto constIntOp = mlir::dyn_cast_or_null<mlir::IntegerAttr>(2331 mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[2].getDefiningOp())2332 .getValue());2333 assert(constIntOp && "expected integer constant argument");2334 2335 // Bitcast to vector<16xi8>2336 auto vi8Ty{mlir::VectorType::get(16, builder.getIntegerType(8))};2337 if (mlirTyArgs[0] != vi8Ty) {2338 mlirVecArgs[0] =2339 mlir::LLVM::BitcastOp::create(builder, loc, vi8Ty, mlirVecArgs[0])2340 .getResult();2341 mlirVecArgs[1] =2342 mlir::LLVM::BitcastOp::create(builder, loc, vi8Ty, mlirVecArgs[1])2343 .getResult();2344 }2345 2346 // Construct the mask for shuffling2347 auto shiftVal{constIntOp.getInt()};2348 if (vop == VecOp::Sldw)2349 shiftVal = shiftVal << 2;2350 shiftVal &= 0xF;2351 llvm::SmallVector<int64_t, 16> mask;2352 // Shuffle with mask based on the endianness2353 const auto triple{fir::getTargetTriple(builder.getModule())};2354 if (triple.isLittleEndian()) {2355 for (int i = 16; i < 32; ++i)2356 mask.push_back(i - shiftVal);2357 shftRes = mlir::vector::ShuffleOp::create(builder, loc, mlirVecArgs[1],2358 mlirVecArgs[0], mask);2359 } else {2360 for (int i = 0; i < 16; ++i)2361 mask.push_back(i + shiftVal);2362 shftRes = mlir::vector::ShuffleOp::create(builder, loc, mlirVecArgs[0],2363 mlirVecArgs[1], mask);2364 }2365 2366 // Bitcast to the original type2367 if (shftRes.getType() != mlirTyArgs[0])2368 shftRes =2369 mlir::LLVM::BitcastOp::create(builder, loc, mlirTyArgs[0], shftRes);2370 2371 return builder.createConvert(loc, resultType, shftRes);2372 } else2373 llvm_unreachable("Invalid vector operation for generator");2374 2375 return shftRes;2376}2377 2378// VEC_SPLAT, VEC_SPLATS, VEC_SPLAT_S322379template <VecOp vop>2380fir::ExtendedValue2381PPCIntrinsicLibrary::genVecSplat(mlir::Type resultType,2382 llvm::ArrayRef<fir::ExtendedValue> args) {2383 auto context{builder.getContext()};2384 auto argBases{getBasesForArgs(args)};2385 2386 mlir::vector::BroadcastOp splatOp{nullptr};2387 mlir::Type retTy{nullptr};2388 switch (vop) {2389 case VecOp::Splat: {2390 assert(args.size() == 2);2391 auto vecTyInfo{getVecTypeFromFir(argBases[0])};2392 2393 auto extractOp{genVecExtract(resultType, args)};2394 splatOp = mlir::vector::BroadcastOp::create(2395 builder, loc, vecTyInfo.toMlirVectorType(context),2396 *(extractOp.getUnboxed()));2397 retTy = vecTyInfo.toFirVectorType();2398 break;2399 }2400 case VecOp::Splats: {2401 assert(args.size() == 1);2402 auto vecTyInfo{getVecTypeFromEle(argBases[0])};2403 2404 splatOp = mlir::vector::BroadcastOp::create(2405 builder, loc, vecTyInfo.toMlirVectorType(context), argBases[0]);2406 retTy = vecTyInfo.toFirVectorType();2407 break;2408 }2409 case VecOp::Splat_s32: {2410 assert(args.size() == 1);2411 auto eleTy{builder.getIntegerType(32)};2412 auto intOp{builder.createConvert(loc, eleTy, argBases[0])};2413 2414 // the intrinsic always returns vector(integer(4))2415 splatOp = mlir::vector::BroadcastOp::create(2416 builder, loc, mlir::VectorType::get(4, eleTy), intOp);2417 retTy = fir::VectorType::get(4, eleTy);2418 break;2419 }2420 default:2421 llvm_unreachable("invalid vector operation for generator");2422 }2423 return builder.createConvert(loc, retTy, splatOp);2424}2425 2426fir::ExtendedValue2427PPCIntrinsicLibrary::genVecXlds(mlir::Type resultType,2428 llvm::ArrayRef<fir::ExtendedValue> args) {2429 assert(args.size() == 2);2430 auto arg0{getBase(args[0])};2431 auto arg1{getBase(args[1])};2432 2433 // Prepare the return type in FIR.2434 auto vecTyInfo{getVecTypeFromFirType(resultType)};2435 auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};2436 auto firTy{vecTyInfo.toFirVectorType()};2437 2438 // Add the %val of arg0 to %addr of arg12439 auto addr{addOffsetToAddress(builder, loc, arg1, arg0)};2440 2441 auto i64Ty{mlir::IntegerType::get(builder.getContext(), 64)};2442 auto i64VecTy{mlir::VectorType::get(2, i64Ty)};2443 auto i64RefTy{builder.getRefType(i64Ty)};2444 auto addrConv{fir::ConvertOp::create(builder, loc, i64RefTy, addr)};2445 2446 auto addrVal{fir::LoadOp::create(builder, loc, addrConv)};2447 auto splatRes{2448 mlir::vector::BroadcastOp::create(builder, loc, i64VecTy, addrVal)};2449 2450 mlir::Value result{nullptr};2451 if (mlirTy != splatRes.getType()) {2452 result = mlir::vector::BitCastOp::create(builder, loc, mlirTy, splatRes);2453 } else2454 result = splatRes;2455 2456 return builder.createConvert(loc, firTy, result);2457}2458 2459const char *getMmaIrIntrName(MMAOp mmaOp) {2460 switch (mmaOp) {2461 case MMAOp::AssembleAcc:2462 return "llvm.ppc.mma.assemble.acc";2463 case MMAOp::AssemblePair:2464 return "llvm.ppc.vsx.assemble.pair";2465 case MMAOp::DisassembleAcc:2466 return "llvm.ppc.mma.disassemble.acc";2467 case MMAOp::DisassemblePair:2468 return "llvm.ppc.vsx.disassemble.pair";2469 case MMAOp::Xxmfacc:2470 return "llvm.ppc.mma.xxmfacc";2471 case MMAOp::Xxmtacc:2472 return "llvm.ppc.mma.xxmtacc";2473 case MMAOp::Xxsetaccz:2474 return "llvm.ppc.mma.xxsetaccz";2475 case MMAOp::Pmxvbf16ger2:2476 return "llvm.ppc.mma.pmxvbf16ger2";2477 case MMAOp::Pmxvbf16ger2nn:2478 return "llvm.ppc.mma.pmxvbf16ger2nn";2479 case MMAOp::Pmxvbf16ger2np:2480 return "llvm.ppc.mma.pmxvbf16ger2np";2481 case MMAOp::Pmxvbf16ger2pn:2482 return "llvm.ppc.mma.pmxvbf16ger2pn";2483 case MMAOp::Pmxvbf16ger2pp:2484 return "llvm.ppc.mma.pmxvbf16ger2pp";2485 case MMAOp::Pmxvf16ger2:2486 return "llvm.ppc.mma.pmxvf16ger2";2487 case MMAOp::Pmxvf16ger2nn:2488 return "llvm.ppc.mma.pmxvf16ger2nn";2489 case MMAOp::Pmxvf16ger2np:2490 return "llvm.ppc.mma.pmxvf16ger2np";2491 case MMAOp::Pmxvf16ger2pn:2492 return "llvm.ppc.mma.pmxvf16ger2pn";2493 case MMAOp::Pmxvf16ger2pp:2494 return "llvm.ppc.mma.pmxvf16ger2pp";2495 case MMAOp::Pmxvf32ger:2496 return "llvm.ppc.mma.pmxvf32ger";2497 case MMAOp::Pmxvf32gernn:2498 return "llvm.ppc.mma.pmxvf32gernn";2499 case MMAOp::Pmxvf32gernp:2500 return "llvm.ppc.mma.pmxvf32gernp";2501 case MMAOp::Pmxvf32gerpn:2502 return "llvm.ppc.mma.pmxvf32gerpn";2503 case MMAOp::Pmxvf32gerpp:2504 return "llvm.ppc.mma.pmxvf32gerpp";2505 case MMAOp::Pmxvf64ger:2506 return "llvm.ppc.mma.pmxvf64ger";2507 case MMAOp::Pmxvf64gernn:2508 return "llvm.ppc.mma.pmxvf64gernn";2509 case MMAOp::Pmxvf64gernp:2510 return "llvm.ppc.mma.pmxvf64gernp";2511 case MMAOp::Pmxvf64gerpn:2512 return "llvm.ppc.mma.pmxvf64gerpn";2513 case MMAOp::Pmxvf64gerpp:2514 return "llvm.ppc.mma.pmxvf64gerpp";2515 case MMAOp::Pmxvi16ger2:2516 return "llvm.ppc.mma.pmxvi16ger2";2517 case MMAOp::Pmxvi16ger2pp:2518 return "llvm.ppc.mma.pmxvi16ger2pp";2519 case MMAOp::Pmxvi16ger2s:2520 return "llvm.ppc.mma.pmxvi16ger2s";2521 case MMAOp::Pmxvi16ger2spp:2522 return "llvm.ppc.mma.pmxvi16ger2spp";2523 case MMAOp::Pmxvi4ger8:2524 return "llvm.ppc.mma.pmxvi4ger8";2525 case MMAOp::Pmxvi4ger8pp:2526 return "llvm.ppc.mma.pmxvi4ger8pp";2527 case MMAOp::Pmxvi8ger4:2528 return "llvm.ppc.mma.pmxvi8ger4";2529 case MMAOp::Pmxvi8ger4pp:2530 return "llvm.ppc.mma.pmxvi8ger4pp";2531 case MMAOp::Pmxvi8ger4spp:2532 return "llvm.ppc.mma.pmxvi8ger4spp";2533 case MMAOp::Xvbf16ger2:2534 return "llvm.ppc.mma.xvbf16ger2";2535 case MMAOp::Xvbf16ger2nn:2536 return "llvm.ppc.mma.xvbf16ger2nn";2537 case MMAOp::Xvbf16ger2np:2538 return "llvm.ppc.mma.xvbf16ger2np";2539 case MMAOp::Xvbf16ger2pn:2540 return "llvm.ppc.mma.xvbf16ger2pn";2541 case MMAOp::Xvbf16ger2pp:2542 return "llvm.ppc.mma.xvbf16ger2pp";2543 case MMAOp::Xvf16ger2:2544 return "llvm.ppc.mma.xvf16ger2";2545 case MMAOp::Xvf16ger2nn:2546 return "llvm.ppc.mma.xvf16ger2nn";2547 case MMAOp::Xvf16ger2np:2548 return "llvm.ppc.mma.xvf16ger2np";2549 case MMAOp::Xvf16ger2pn:2550 return "llvm.ppc.mma.xvf16ger2pn";2551 case MMAOp::Xvf16ger2pp:2552 return "llvm.ppc.mma.xvf16ger2pp";2553 case MMAOp::Xvf32ger:2554 return "llvm.ppc.mma.xvf32ger";2555 case MMAOp::Xvf32gernn:2556 return "llvm.ppc.mma.xvf32gernn";2557 case MMAOp::Xvf32gernp:2558 return "llvm.ppc.mma.xvf32gernp";2559 case MMAOp::Xvf32gerpn:2560 return "llvm.ppc.mma.xvf32gerpn";2561 case MMAOp::Xvf32gerpp:2562 return "llvm.ppc.mma.xvf32gerpp";2563 case MMAOp::Xvf64ger:2564 return "llvm.ppc.mma.xvf64ger";2565 case MMAOp::Xvf64gernn:2566 return "llvm.ppc.mma.xvf64gernn";2567 case MMAOp::Xvf64gernp:2568 return "llvm.ppc.mma.xvf64gernp";2569 case MMAOp::Xvf64gerpn:2570 return "llvm.ppc.mma.xvf64gerpn";2571 case MMAOp::Xvf64gerpp:2572 return "llvm.ppc.mma.xvf64gerpp";2573 case MMAOp::Xvi16ger2:2574 return "llvm.ppc.mma.xvi16ger2";2575 case MMAOp::Xvi16ger2pp:2576 return "llvm.ppc.mma.xvi16ger2pp";2577 case MMAOp::Xvi16ger2s:2578 return "llvm.ppc.mma.xvi16ger2s";2579 case MMAOp::Xvi16ger2spp:2580 return "llvm.ppc.mma.xvi16ger2spp";2581 case MMAOp::Xvi4ger8:2582 return "llvm.ppc.mma.xvi4ger8";2583 case MMAOp::Xvi4ger8pp:2584 return "llvm.ppc.mma.xvi4ger8pp";2585 case MMAOp::Xvi8ger4:2586 return "llvm.ppc.mma.xvi8ger4";2587 case MMAOp::Xvi8ger4pp:2588 return "llvm.ppc.mma.xvi8ger4pp";2589 case MMAOp::Xvi8ger4spp:2590 return "llvm.ppc.mma.xvi8ger4spp";2591 }2592 llvm_unreachable("getMmaIrIntrName");2593}2594 2595mlir::FunctionType getMmaIrFuncType(mlir::MLIRContext *context, MMAOp mmaOp) {2596 switch (mmaOp) {2597 case MMAOp::AssembleAcc:2598 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 4);2599 case MMAOp::AssemblePair:2600 return genMmaVpFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2601 case MMAOp::DisassembleAcc:2602 return genMmaDisassembleFuncType(context, mmaOp);2603 case MMAOp::DisassemblePair:2604 return genMmaDisassembleFuncType(context, mmaOp);2605 case MMAOp::Xxmfacc:2606 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 0);2607 case MMAOp::Xxmtacc:2608 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 0);2609 case MMAOp::Xxsetaccz:2610 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 0);2611 case MMAOp::Pmxvbf16ger2:2612 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,2613 /*Integer*/ 3);2614 case MMAOp::Pmxvbf16ger2nn:2615 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2616 /*Integer*/ 3);2617 case MMAOp::Pmxvbf16ger2np:2618 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2619 /*Integer*/ 3);2620 case MMAOp::Pmxvbf16ger2pn:2621 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2622 /*Integer*/ 3);2623 case MMAOp::Pmxvbf16ger2pp:2624 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2625 /*Integer*/ 3);2626 case MMAOp::Pmxvf16ger2:2627 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,2628 /*Integer*/ 3);2629 case MMAOp::Pmxvf16ger2nn:2630 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2631 /*Integer*/ 3);2632 case MMAOp::Pmxvf16ger2np:2633 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2634 /*Integer*/ 3);2635 case MMAOp::Pmxvf16ger2pn:2636 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2637 /*Integer*/ 3);2638 case MMAOp::Pmxvf16ger2pp:2639 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2640 /*Integer*/ 3);2641 case MMAOp::Pmxvf32ger:2642 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,2643 /*Integer*/ 2);2644 case MMAOp::Pmxvf32gernn:2645 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2646 /*Integer*/ 2);2647 case MMAOp::Pmxvf32gernp:2648 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2649 /*Integer*/ 2);2650 case MMAOp::Pmxvf32gerpn:2651 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2652 /*Integer*/ 2);2653 case MMAOp::Pmxvf32gerpp:2654 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2655 /*Integer*/ 2);2656 case MMAOp::Pmxvf64ger:2657 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 1, /*Vector*/ 1,2658 /*Integer*/ 2);2659 case MMAOp::Pmxvf64gernn:2660 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,2661 /*Integer*/ 2);2662 case MMAOp::Pmxvf64gernp:2663 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,2664 /*Integer*/ 2);2665 case MMAOp::Pmxvf64gerpn:2666 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,2667 /*Integer*/ 2);2668 case MMAOp::Pmxvf64gerpp:2669 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,2670 /*Integer*/ 2);2671 case MMAOp::Pmxvi16ger2:2672 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,2673 /*Integer*/ 3);2674 case MMAOp::Pmxvi16ger2pp:2675 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2676 /*Integer*/ 3);2677 case MMAOp::Pmxvi16ger2s:2678 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,2679 /*Integer*/ 3);2680 case MMAOp::Pmxvi16ger2spp:2681 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2682 /*Integer*/ 3);2683 case MMAOp::Pmxvi4ger8:2684 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,2685 /*Integer*/ 3);2686 case MMAOp::Pmxvi4ger8pp:2687 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2688 /*Integer*/ 3);2689 case MMAOp::Pmxvi8ger4:2690 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,2691 /*Integer*/ 3);2692 case MMAOp::Pmxvi8ger4pp:2693 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2694 /*Integer*/ 3);2695 case MMAOp::Pmxvi8ger4spp:2696 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,2697 /*Integer*/ 3);2698 case MMAOp::Xvbf16ger2:2699 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2700 case MMAOp::Xvbf16ger2nn:2701 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2702 case MMAOp::Xvbf16ger2np:2703 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2704 case MMAOp::Xvbf16ger2pn:2705 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2706 case MMAOp::Xvbf16ger2pp:2707 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2708 case MMAOp::Xvf16ger2:2709 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2710 case MMAOp::Xvf16ger2nn:2711 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2712 case MMAOp::Xvf16ger2np:2713 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2714 case MMAOp::Xvf16ger2pn:2715 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2716 case MMAOp::Xvf16ger2pp:2717 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2718 case MMAOp::Xvf32ger:2719 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2720 case MMAOp::Xvf32gernn:2721 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2722 case MMAOp::Xvf32gernp:2723 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2724 case MMAOp::Xvf32gerpn:2725 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2726 case MMAOp::Xvf32gerpp:2727 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2728 case MMAOp::Xvf64ger:2729 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 1, /*Vector*/ 1);2730 case MMAOp::Xvf64gernn:2731 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);2732 case MMAOp::Xvf64gernp:2733 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);2734 case MMAOp::Xvf64gerpn:2735 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);2736 case MMAOp::Xvf64gerpp:2737 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);2738 case MMAOp::Xvi16ger2:2739 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2740 case MMAOp::Xvi16ger2pp:2741 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2742 case MMAOp::Xvi16ger2s:2743 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2744 case MMAOp::Xvi16ger2spp:2745 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2746 case MMAOp::Xvi4ger8:2747 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2748 case MMAOp::Xvi4ger8pp:2749 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2750 case MMAOp::Xvi8ger4:2751 return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);2752 case MMAOp::Xvi8ger4pp:2753 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2754 case MMAOp::Xvi8ger4spp:2755 return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);2756 }2757 llvm_unreachable("getMmaIrFuncType");2758}2759 2760template <MMAOp IntrId, MMAHandlerOp HandlerOp>2761void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef<fir::ExtendedValue> args) {2762 auto context{builder.getContext()};2763 mlir::FunctionType intrFuncType{getMmaIrFuncType(context, IntrId)};2764 mlir::func::FuncOp funcOp{2765 builder.createFunction(loc, getMmaIrIntrName(IntrId), intrFuncType)};2766 llvm::SmallVector<mlir::Value> intrArgs;2767 2768 // Depending on SubToFunc, change the subroutine call to a function call.2769 // First argument represents the result. Rest of the arguments2770 // are shifted one position to form the actual argument list.2771 size_t argStart{0};2772 size_t argStep{1};2773 size_t e{args.size()};2774 if (HandlerOp == MMAHandlerOp::SubToFunc) {2775 // The first argument becomes function result. Start from the second2776 // argument.2777 argStart = 1;2778 } else if (HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE) {2779 // Reverse argument order on little-endian target only.2780 // The reversal does not depend on the setting of non-native-order option.2781 const auto triple{fir::getTargetTriple(builder.getModule())};2782 if (triple.isLittleEndian()) {2783 // Load the arguments in reverse order.2784 argStart = args.size() - 1;2785 // The first argument becomes function result. Stop at the second2786 // argument.2787 e = 0;2788 argStep = -1;2789 } else {2790 // Load the arguments in natural order.2791 // The first argument becomes function result. Start from the second2792 // argument.2793 argStart = 1;2794 }2795 }2796 2797 for (size_t i = argStart, j = 0; i != e; i += argStep, ++j) {2798 auto v{fir::getBase(args[i])};2799 if (i == 0 && HandlerOp == MMAHandlerOp::FirstArgIsResult) {2800 // First argument is passed in as an address. We need to load2801 // the content to match the LLVM interface.2802 v = fir::LoadOp::create(builder, loc, v);2803 }2804 auto vType{v.getType()};2805 mlir::Type targetType{intrFuncType.getInput(j)};2806 if (vType != targetType) {2807 if (mlir::isa<mlir::VectorType>(targetType)) {2808 // Perform vector type conversion for arguments passed by value.2809 auto eleTy{mlir::dyn_cast<fir::VectorType>(vType).getElementType()};2810 auto len{mlir::dyn_cast<fir::VectorType>(vType).getLen()};2811 mlir::VectorType mlirType = mlir::VectorType::get(len, eleTy);2812 auto v0{builder.createConvert(loc, mlirType, v)};2813 auto v1{mlir::vector::BitCastOp::create(builder, loc, targetType, v0)};2814 intrArgs.push_back(v1);2815 } else if (mlir::isa<mlir::IntegerType>(targetType) &&2816 mlir::isa<mlir::IntegerType>(vType)) {2817 auto v0{builder.createConvert(loc, targetType, v)};2818 intrArgs.push_back(v0);2819 } else {2820 llvm::errs() << "\nUnexpected type conversion requested: "2821 << " from " << vType << " to " << targetType << "\n";2822 llvm_unreachable("Unsupported type conversion for argument to PowerPC "2823 "MMA intrinsic");2824 }2825 } else {2826 intrArgs.push_back(v);2827 }2828 }2829 auto callSt{fir::CallOp::create(builder, loc, funcOp, intrArgs)};2830 if (HandlerOp == MMAHandlerOp::SubToFunc ||2831 HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE ||2832 HandlerOp == MMAHandlerOp::FirstArgIsResult) {2833 // Convert pointer type if needed.2834 mlir::Value callResult{callSt.getResult(0)};2835 mlir::Value destPtr{fir::getBase(args[0])};2836 mlir::Type callResultPtrType{builder.getRefType(callResult.getType())};2837 if (destPtr.getType() != callResultPtrType) {2838 destPtr =2839 fir::ConvertOp::create(builder, loc, callResultPtrType, destPtr);2840 }2841 // Copy the result.2842 fir::StoreOp::create(builder, loc, callResult, destPtr);2843 }2844}2845 2846// VEC_ST, VEC_STE2847template <VecOp vop>2848void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) {2849 assert(args.size() == 3);2850 2851 auto context{builder.getContext()};2852 auto argBases{getBasesForArgs(args)};2853 auto arg1TyInfo{getVecTypeFromFir(argBases[0])};2854 2855 auto addr{addOffsetToAddress(builder, loc, argBases[2], argBases[1])};2856 2857 llvm::StringRef fname{};2858 mlir::VectorType stTy{nullptr};2859 auto i32ty{mlir::IntegerType::get(context, 32)};2860 switch (vop) {2861 case VecOp::St:2862 stTy = mlir::VectorType::get(4, i32ty);2863 fname = "llvm.ppc.altivec.stvx";2864 break;2865 case VecOp::Ste: {2866 const auto width{arg1TyInfo.eleTy.getIntOrFloatBitWidth()};2867 const auto len{arg1TyInfo.len};2868 2869 if (arg1TyInfo.isFloat32()) {2870 stTy = mlir::VectorType::get(len, i32ty);2871 fname = "llvm.ppc.altivec.stvewx";2872 } else if (mlir::isa<mlir::IntegerType>(arg1TyInfo.eleTy)) {2873 stTy = mlir::VectorType::get(len, mlir::IntegerType::get(context, width));2874 2875 switch (width) {2876 case 8:2877 fname = "llvm.ppc.altivec.stvebx";2878 break;2879 case 16:2880 fname = "llvm.ppc.altivec.stvehx";2881 break;2882 case 32:2883 fname = "llvm.ppc.altivec.stvewx";2884 break;2885 default:2886 assert(false && "invalid element size");2887 }2888 } else2889 assert(false && "unknown type");2890 break;2891 }2892 case VecOp::Stxvp:2893 // __vector_pair type2894 stTy = mlir::VectorType::get(256, mlir::IntegerType::get(context, 1));2895 fname = "llvm.ppc.vsx.stxvp";2896 break;2897 default:2898 llvm_unreachable("invalid vector operation for generator");2899 }2900 2901 auto funcType{mlir::FunctionType::get(context, {stTy, addr.getType()}, {})};2902 mlir::func::FuncOp funcOp = builder.createFunction(loc, fname, funcType);2903 2904 llvm::SmallVector<mlir::Value, 4> biArgs;2905 2906 if (vop == VecOp::Stxvp) {2907 biArgs.push_back(argBases[0]);2908 biArgs.push_back(addr);2909 fir::CallOp::create(builder, loc, funcOp, biArgs);2910 return;2911 }2912 2913 auto vecTyInfo{getVecTypeFromFirType(argBases[0].getType())};2914 auto cnv{builder.createConvert(loc, vecTyInfo.toMlirVectorType(context),2915 argBases[0])};2916 2917 mlir::Value newArg1{nullptr};2918 if (stTy != arg1TyInfo.toMlirVectorType(context))2919 newArg1 = mlir::vector::BitCastOp::create(builder, loc, stTy, cnv);2920 else2921 newArg1 = cnv;2922 2923 if (isBEVecElemOrderOnLE())2924 newArg1 = builder.createConvert(2925 loc, stTy, reverseVectorElements(builder, loc, newArg1, 4));2926 2927 biArgs.push_back(newArg1);2928 biArgs.push_back(addr);2929 2930 fir::CallOp::create(builder, loc, funcOp, biArgs);2931}2932 2933// VEC_XST, VEC_XST_BE, VEC_STXV, VEC_XSTD2, VEC_XSTW42934template <VecOp vop>2935void PPCIntrinsicLibrary::genVecXStore(2936 llvm::ArrayRef<fir::ExtendedValue> args) {2937 assert(args.size() == 3);2938 auto context{builder.getContext()};2939 auto argBases{getBasesForArgs(args)};2940 VecTypeInfo arg1TyInfo{getVecTypeFromFir(argBases[0])};2941 2942 auto addr{addOffsetToAddress(builder, loc, argBases[2], argBases[1])};2943 2944 mlir::Value trg{nullptr};2945 mlir::Value src{nullptr};2946 2947 switch (vop) {2948 case VecOp::Xst:2949 case VecOp::Xst_be: {2950 src = argBases[0];2951 trg = builder.createConvert(loc, builder.getRefType(argBases[0].getType()),2952 addr);2953 2954 if (vop == VecOp::Xst_be || isBEVecElemOrderOnLE()) {2955 auto cnv{builder.createConvert(loc, arg1TyInfo.toMlirVectorType(context),2956 argBases[0])};2957 auto shf{reverseVectorElements(builder, loc, cnv, arg1TyInfo.len)};2958 2959 src = builder.createConvert(loc, arg1TyInfo.toFirVectorType(), shf);2960 }2961 break;2962 }2963 case VecOp::Xstd2:2964 case VecOp::Xstw4: {2965 // an 16-byte vector arg1 is treated as two 8-byte elements or2966 // four 4-byte elements2967 mlir::IntegerType elemTy;2968 uint64_t numElem = (vop == VecOp::Xstd2) ? 2 : 4;2969 elemTy = builder.getIntegerType(128 / numElem);2970 2971 mlir::VectorType mlirVecTy{mlir::VectorType::get(numElem, elemTy)};2972 fir::VectorType firVecTy{fir::VectorType::get(numElem, elemTy)};2973 2974 auto cnv{builder.createConvert(loc, arg1TyInfo.toMlirVectorType(context),2975 argBases[0])};2976 2977 mlir::Type srcTy{nullptr};2978 if (numElem != arg1TyInfo.len) {2979 cnv = mlir::vector::BitCastOp::create(builder, loc, mlirVecTy, cnv);2980 srcTy = firVecTy;2981 } else {2982 srcTy = arg1TyInfo.toFirVectorType();2983 }2984 2985 trg = builder.createConvert(loc, builder.getRefType(srcTy), addr);2986 2987 if (isBEVecElemOrderOnLE()) {2988 cnv = reverseVectorElements(builder, loc, cnv, numElem);2989 }2990 2991 src = builder.createConvert(loc, srcTy, cnv);2992 break;2993 }2994 case VecOp::Stxv:2995 src = argBases[0];2996 trg = builder.createConvert(loc, builder.getRefType(argBases[0].getType()),2997 addr);2998 break;2999 default:3000 assert(false && "Invalid vector operation for generator");3001 }3002 fir::StoreOp::create(builder, loc, mlir::TypeRange{},3003 mlir::ValueRange{src, trg},3004 getAlignmentAttr(builder, 1));3005}3006 3007} // namespace fir3008