194 lines · cpp
1//===-- X86InstrFMA3Info.cpp - X86 FMA3 Instruction Information -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the implementation of the classes providing information10// about existing X86 FMA3 opcodes, classifying and grouping them.11//12//===----------------------------------------------------------------------===//13 14#include "X86InstrFMA3Info.h"15#include "X86InstrInfo.h"16#include <atomic>17#include <cassert>18#include <cstdint>19 20using namespace llvm;21 22#define FMA3GROUP(Name, Suf, Attrs) \23 { { X86::Name##132##Suf, X86::Name##213##Suf, X86::Name##231##Suf }, Attrs },24 25#define FMA3GROUP_MASKED(Name, Suf, Attrs) \26 FMA3GROUP(Name, Suf, Attrs) \27 FMA3GROUP(Name, Suf##k, Attrs | X86InstrFMA3Group::KMergeMasked) \28 FMA3GROUP(Name, Suf##kz, Attrs | X86InstrFMA3Group::KZeroMasked)29 30#define FMA3GROUP_MASKED_INT(Name, Suf, Attrs) \31 FMA3GROUP(Name, Suf##_Int, Attrs) \32 FMA3GROUP(Name, Suf##k_Int, Attrs | X86InstrFMA3Group::KMergeMasked) \33 FMA3GROUP(Name, Suf##kz_Int, Attrs | X86InstrFMA3Group::KZeroMasked)34 35#define FMA3GROUP_PACKED_WIDTHS_Z(Name, Suf, Attrs) \36 FMA3GROUP_MASKED(Name, Suf##Z128m, Attrs) \37 FMA3GROUP_MASKED(Name, Suf##Z128r, Attrs) \38 FMA3GROUP_MASKED(Name, Suf##Z256m, Attrs) \39 FMA3GROUP_MASKED(Name, Suf##Z256r, Attrs) \40 FMA3GROUP_MASKED(Name, Suf##Zm, Attrs) \41 FMA3GROUP_MASKED(Name, Suf##Zr, Attrs) \42 43#define FMA3GROUP_PACKED_WIDTHS_ALL(Name, Suf, Attrs) \44 FMA3GROUP(Name, Suf##Ym, Attrs) \45 FMA3GROUP(Name, Suf##Yr, Attrs) \46 FMA3GROUP_PACKED_WIDTHS_Z(Name, Suf, Attrs) \47 FMA3GROUP(Name, Suf##m, Attrs) \48 FMA3GROUP(Name, Suf##r, Attrs)49 50#define FMA3GROUP_PACKED_DHS(Name, Attrs) \51 FMA3GROUP_PACKED_WIDTHS_ALL(Name, PD, Attrs) \52 FMA3GROUP_PACKED_WIDTHS_Z(Name, PH, Attrs) \53 FMA3GROUP_PACKED_WIDTHS_ALL(Name, PS, Attrs)54 55#define FMA3GROUP_PACKED_BF16(Name, Attrs) \56 FMA3GROUP_PACKED_WIDTHS_Z(Name, BF16, Attrs)57 58#define FMA3GROUP_SCALAR_WIDTHS_Z(Name, Suf, Attrs) \59 FMA3GROUP(Name, Suf##Zm, Attrs) \60 FMA3GROUP_MASKED_INT(Name, Suf##Zm, Attrs | X86InstrFMA3Group::Intrinsic) \61 FMA3GROUP(Name, Suf##Zr, Attrs) \62 FMA3GROUP_MASKED_INT(Name, Suf##Zr, Attrs | X86InstrFMA3Group::Intrinsic) \63 64#define FMA3GROUP_SCALAR_WIDTHS_ALL(Name, Suf, Attrs) \65 FMA3GROUP_SCALAR_WIDTHS_Z(Name, Suf, Attrs) \66 FMA3GROUP(Name, Suf##m, Attrs) \67 FMA3GROUP(Name, Suf##m_Int, Attrs | X86InstrFMA3Group::Intrinsic) \68 FMA3GROUP(Name, Suf##r, Attrs) \69 FMA3GROUP(Name, Suf##r_Int, Attrs | X86InstrFMA3Group::Intrinsic)70 71#define FMA3GROUP_SCALAR(Name, Attrs) \72 FMA3GROUP_SCALAR_WIDTHS_ALL(Name, SD, Attrs) \73 FMA3GROUP_SCALAR_WIDTHS_Z(Name, SH, Attrs) \74 FMA3GROUP_SCALAR_WIDTHS_ALL(Name, SS, Attrs)75 76#define FMA3GROUP_FULL(Name, Attrs) \77 FMA3GROUP_PACKED_BF16(Name, Attrs) \78 FMA3GROUP_PACKED_DHS(Name, Attrs) \79 FMA3GROUP_SCALAR(Name, Attrs)80 81static const X86InstrFMA3Group Groups[] = {82 FMA3GROUP_FULL(VFMADD, 0)83 FMA3GROUP_PACKED_DHS(VFMADDSUB, 0)84 FMA3GROUP_FULL(VFMSUB, 0)85 FMA3GROUP_PACKED_DHS(VFMSUBADD, 0)86 FMA3GROUP_FULL(VFNMADD, 0)87 FMA3GROUP_FULL(VFNMSUB, 0)88};89 90#define FMA3GROUP_PACKED_AVX512_WIDTHS(Name, Type, Suf, Attrs) \91 FMA3GROUP_MASKED(Name, Type##Z128##Suf, Attrs) \92 FMA3GROUP_MASKED(Name, Type##Z256##Suf, Attrs) \93 FMA3GROUP_MASKED(Name, Type##Z##Suf, Attrs)94 95#define FMA3GROUP_PACKED_AVX512_ALL(Name, Suf, Attrs) \96 FMA3GROUP_PACKED_AVX512_WIDTHS(Name, BF16, Suf, Attrs) \97 FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PD, Suf, Attrs) \98 FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PH, Suf, Attrs) \99 FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PS, Suf, Attrs)100 101#define FMA3GROUP_PACKED_AVX512_DHS(Name, Suf, Attrs) \102 FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PD, Suf, Attrs) \103 FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PH, Suf, Attrs) \104 FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PS, Suf, Attrs)105 106#define FMA3GROUP_PACKED_AVX512_ROUND(Name, Suf, Attrs) \107 FMA3GROUP_MASKED(Name, PDZ##Suf, Attrs) \108 FMA3GROUP_MASKED(Name, PHZ##Suf, Attrs) \109 FMA3GROUP_MASKED(Name, PSZ##Suf, Attrs)110 111#define FMA3GROUP_SCALAR_AVX512_ROUND(Name, Suf, Attrs) \112 FMA3GROUP(Name, SDZ##Suf, Attrs) \113 FMA3GROUP_MASKED_INT(Name, SDZ##Suf, Attrs) \114 FMA3GROUP(Name, SHZ##Suf, Attrs) \115 FMA3GROUP_MASKED_INT(Name, SHZ##Suf, Attrs) \116 FMA3GROUP(Name, SSZ##Suf, Attrs) \117 FMA3GROUP_MASKED_INT(Name, SSZ##Suf, Attrs)118 119static const X86InstrFMA3Group BroadcastGroups[] = {120 FMA3GROUP_PACKED_AVX512_ALL(VFMADD, mb, 0)121 FMA3GROUP_PACKED_AVX512_DHS(VFMADDSUB, mb, 0)122 FMA3GROUP_PACKED_AVX512_ALL(VFMSUB, mb, 0)123 FMA3GROUP_PACKED_AVX512_DHS(VFMSUBADD, mb, 0)124 FMA3GROUP_PACKED_AVX512_ALL(VFNMADD, mb, 0)125 FMA3GROUP_PACKED_AVX512_ALL(VFNMSUB, mb, 0)126};127 128static const X86InstrFMA3Group RoundGroups[] = {129 FMA3GROUP_PACKED_AVX512_ROUND(VFMADD, rb, 0)130 FMA3GROUP_SCALAR_AVX512_ROUND(VFMADD, rb, X86InstrFMA3Group::Intrinsic)131 FMA3GROUP_PACKED_AVX512_ROUND(VFMADDSUB, rb, 0)132 FMA3GROUP_PACKED_AVX512_ROUND(VFMSUB, rb, 0)133 FMA3GROUP_SCALAR_AVX512_ROUND(VFMSUB, rb, X86InstrFMA3Group::Intrinsic)134 FMA3GROUP_PACKED_AVX512_ROUND(VFMSUBADD, rb, 0)135 FMA3GROUP_PACKED_AVX512_ROUND(VFNMADD, rb, 0)136 FMA3GROUP_SCALAR_AVX512_ROUND(VFNMADD, rb, X86InstrFMA3Group::Intrinsic)137 FMA3GROUP_PACKED_AVX512_ROUND(VFNMSUB, rb, 0)138 FMA3GROUP_SCALAR_AVX512_ROUND(VFNMSUB, rb, X86InstrFMA3Group::Intrinsic)139};140 141static void verifyTables() {142#ifndef NDEBUG143 static std::atomic<bool> TableChecked(false);144 if (!TableChecked.load(std::memory_order_relaxed)) {145 assert(llvm::is_sorted(Groups) && llvm::is_sorted(RoundGroups) &&146 llvm::is_sorted(BroadcastGroups) && "FMA3 tables not sorted!");147 TableChecked.store(true, std::memory_order_relaxed);148 }149#endif150}151 152/// Returns a reference to a group of FMA3 opcodes to where the given153/// \p Opcode is included. If the given \p Opcode is not recognized as FMA3154/// and not included into any FMA3 group, then nullptr is returned.155const X86InstrFMA3Group *llvm::getFMA3Group(unsigned Opcode, uint64_t TSFlags) {156 157 // FMA3 instructions have a well defined encoding pattern we can exploit.158 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);159 bool IsFMA3Opcode = ((BaseOpcode >= 0x96 && BaseOpcode <= 0x9F) ||160 (BaseOpcode >= 0xA6 && BaseOpcode <= 0xAF) ||161 (BaseOpcode >= 0xB6 && BaseOpcode <= 0xBF));162 bool IsFMA3Encoding = ((TSFlags & X86II::EncodingMask) == X86II::VEX &&163 (TSFlags & X86II::OpMapMask) == X86II::T8) ||164 ((TSFlags & X86II::EncodingMask) == X86II::EVEX &&165 ((TSFlags & X86II::OpMapMask) == X86II::T8 ||166 (TSFlags & X86II::OpMapMask) == X86II::T_MAP6));167 bool IsFMA3Prefix = (TSFlags & X86II::OpPrefixMask) == X86II::PD ||168 (TSFlags & X86II::OpPrefixMask) == 0; // X86II::PS169 if (!IsFMA3Opcode || !IsFMA3Encoding || !IsFMA3Prefix)170 return nullptr;171 172 verifyTables();173 174 ArrayRef<X86InstrFMA3Group> Table;175 if (TSFlags & X86II::EVEX_RC)176 Table = ArrayRef(RoundGroups);177 else if (TSFlags & X86II::EVEX_B)178 Table = ArrayRef(BroadcastGroups);179 else180 Table = ArrayRef(Groups);181 182 // FMA 132 instructions have an opcode of 0x96-0x9F183 // FMA 213 instructions have an opcode of 0xA6-0xAF184 // FMA 231 instructions have an opcode of 0xB6-0xBF185 unsigned FormIndex = ((BaseOpcode - 0x90) >> 4) & 0x3;186 187 auto I = partition_point(Table, [=](const X86InstrFMA3Group &Group) {188 return Group.Opcodes[FormIndex] < Opcode;189 });190 assert(I != Table.end() && I->Opcodes[FormIndex] == Opcode &&191 "Couldn't find FMA3 opcode!");192 return I;193}194