695 lines · plain
1//===----------------------Hexagon builtin routine ------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG10#define END(TAG) .size TAG,.-TAG11 12// Double Precision Multiply13 14 15#define A r1:016#define AH r117#define AL r018#define B r3:219#define BH r320#define BL r221#define C r5:422#define CH r523#define CL r424 25 26 27#define BTMP r15:1428#define BTMPH r1529#define BTMPL r1430 31#define ATMP r13:1232#define ATMPH r1333#define ATMPL r1234 35#define CTMP r11:1036#define CTMPH r1137#define CTMPL r1038 39#define PP_LL r9:840#define PP_LL_H r941#define PP_LL_L r842 43#define PP_ODD r7:644#define PP_ODD_H r745#define PP_ODD_L r646 47 48#define PP_HH r17:1649#define PP_HH_H r1750#define PP_HH_L r1651 52#define EXPA r1853#define EXPB r1954#define EXPBA r19:1855 56#define TMP r2857 58#define P_TMP p059#define PROD_NEG p360#define EXACT p261#define SWAP p162 63#define MANTBITS 5264#define HI_MANTBITS 2065#define EXPBITS 1166#define BIAS 102367#define STACKSPACE 3268 69#define ADJUST 470 71#define FUDGE 772#define FUDGE2 373 74#ifndef SR_ROUND_OFF75#define SR_ROUND_OFF 2276#endif77 78 // First, classify for normal values, and abort if abnormal79 //80 // Next, unpack mantissa into 0x1000_0000_0000_0000 + mant<<881 //82 // Since we know that the 2 MSBs of the H registers is zero, we should never carry83 // the partial products that involve the H registers84 //85 // Try to buy X slots, at the expense of latency if needed86 //87 // We will have PP_HH with the upper bits of the product, PP_LL with the lower88 // PP_HH can have a maximum of 0x03FF_FFFF_FFFF_FFFF or thereabouts89 // PP_HH can have a minimum of 0x0100_0000_0000_000090 //91 // 0x0100_0000_0000_0000 has EXP of EXPA+EXPB-BIAS92 //93 // We need to align CTMP.94 // If CTMP >> PP, convert PP to 64 bit with sticky, align CTMP, and follow normal add95 // If CTMP << PP align CTMP and add 128 bits. Then compute sticky96 // If CTMP ~= PP, align CTMP and add 128 bits. May have massive cancellation.97 //98 // Convert partial product and CTMP to 2's complement prior to addition99 //100 // After we add, we need to normalize into upper 64 bits, then compute sticky.101 102 .text103 .global __hexagon_fmadf4104 .type __hexagon_fmadf4,@function105 .global __hexagon_fmadf5106 .type __hexagon_fmadf5,@function107 Q6_ALIAS(fmadf5)108 .p2align 5109__hexagon_fmadf4:110__hexagon_fmadf5:111.Lfma_begin:112 {113 P_TMP = dfclass(A,#2)114 P_TMP = dfclass(B,#2)115 ATMP = #0116 BTMP = #0117 }118 {119 ATMP = insert(A,#MANTBITS,#EXPBITS-3)120 BTMP = insert(B,#MANTBITS,#EXPBITS-3)121 PP_ODD_H = ##0x10000000122 allocframe(#STACKSPACE)123 }124 {125 PP_LL = mpyu(ATMPL,BTMPL)126 if (!P_TMP) jump .Lfma_abnormal_ab127 ATMPH = or(ATMPH,PP_ODD_H)128 BTMPH = or(BTMPH,PP_ODD_H)129 }130 {131 P_TMP = dfclass(C,#2)132 if (!P_TMP.new) jump:nt .Lfma_abnormal_c133 CTMP = combine(PP_ODD_H,#0)134 PP_ODD = combine(#0,PP_LL_H)135 }136.Lfma_abnormal_c_restart:137 {138 PP_ODD += mpyu(BTMPL,ATMPH)139 CTMP = insert(C,#MANTBITS,#EXPBITS-3)140 memd(r29+#0) = PP_HH141 memd(r29+#8) = EXPBA142 }143 {144 PP_ODD += mpyu(ATMPL,BTMPH)145 EXPBA = neg(CTMP)146 P_TMP = cmp.gt(CH,#-1)147 TMP = xor(AH,BH)148 }149 {150 EXPA = extractu(AH,#EXPBITS,#HI_MANTBITS)151 EXPB = extractu(BH,#EXPBITS,#HI_MANTBITS)152 PP_HH = combine(#0,PP_ODD_H)153 if (!P_TMP) CTMP = EXPBA154 }155 {156 PP_HH += mpyu(ATMPH,BTMPH)157 PP_LL = combine(PP_ODD_L,PP_LL_L)158#undef PP_ODD159#undef PP_ODD_H160#undef PP_ODD_L161#undef ATMP162#undef ATMPL163#undef ATMPH164#undef BTMP165#undef BTMPL166#undef BTMPH167#define RIGHTLEFTSHIFT r13:12168#define RIGHTSHIFT r13169#define LEFTSHIFT r12170 171 EXPA = add(EXPA,EXPB)172#undef EXPB173#undef EXPBA174#define EXPC r19175#define EXPCA r19:18176 EXPC = extractu(CH,#EXPBITS,#HI_MANTBITS)177 }178 // PP_HH:PP_LL now has product179 // CTMP is negated180 // EXPA,B,C are extracted181 // We need to negate PP182 // Since we will be adding with carry later, if we need to negate,183 // just invert all bits now, which we can do conditionally and in parallel184#define PP_HH_TMP r15:14185#define PP_LL_TMP r7:6186 {187 EXPA = add(EXPA,#-BIAS+(ADJUST))188 PROD_NEG = !cmp.gt(TMP,#-1)189 PP_LL_TMP = #0190 PP_HH_TMP = #0191 }192 {193 PP_LL_TMP = sub(PP_LL_TMP,PP_LL,PROD_NEG):carry194 P_TMP = !cmp.gt(TMP,#-1)195 SWAP = cmp.gt(EXPC,EXPA) // If C >> PP196 if (SWAP.new) EXPCA = combine(EXPA,EXPC)197 }198 {199 PP_HH_TMP = sub(PP_HH_TMP,PP_HH,PROD_NEG):carry200 if (P_TMP) PP_LL = PP_LL_TMP201#undef PP_LL_TMP202#define CTMP2 r7:6203#define CTMP2H r7204#define CTMP2L r6205 CTMP2 = #0206 EXPC = sub(EXPA,EXPC)207 }208 {209 if (P_TMP) PP_HH = PP_HH_TMP210 P_TMP = cmp.gt(EXPC,#63)211 if (SWAP) PP_LL = CTMP2212 if (SWAP) CTMP2 = PP_LL213 }214#undef PP_HH_TMP215//#define ONE r15:14216//#define S_ONE r14217#define ZERO r15:14218#define S_ZERO r15219#undef PROD_NEG220#define P_CARRY p3221 {222 if (SWAP) PP_HH = CTMP // Swap C and PP223 if (SWAP) CTMP = PP_HH224 if (P_TMP) EXPC = add(EXPC,#-64)225 TMP = #63226 }227 {228 // If diff > 63, pre-shift-right by 64...229 if (P_TMP) CTMP2 = CTMP230 TMP = asr(CTMPH,#31)231 RIGHTSHIFT = min(EXPC,TMP)232 LEFTSHIFT = #0233 }234#undef C235#undef CH236#undef CL237#define STICKIES r5:4238#define STICKIESH r5239#define STICKIESL r4240 {241 if (P_TMP) CTMP = combine(TMP,TMP) // sign extension of pre-shift-right-64242 STICKIES = extract(CTMP2,RIGHTLEFTSHIFT)243 CTMP2 = lsr(CTMP2,RIGHTSHIFT)244 LEFTSHIFT = sub(#64,RIGHTSHIFT)245 }246 {247 ZERO = #0248 TMP = #-2249 CTMP2 |= lsl(CTMP,LEFTSHIFT)250 CTMP = asr(CTMP,RIGHTSHIFT)251 }252 {253 P_CARRY = cmp.gtu(STICKIES,ZERO) // If we have sticky bits from C shift254 if (P_CARRY.new) CTMP2L = and(CTMP2L,TMP) // make sure adding 1 == OR255#undef ZERO256#define ONE r15:14257#define S_ONE r14258 ONE = #1259 STICKIES = #0260 }261 {262 PP_LL = add(CTMP2,PP_LL,P_CARRY):carry // use the carry to add the sticky263 }264 {265 PP_HH = add(CTMP,PP_HH,P_CARRY):carry266 TMP = #62267 }268 // PP_HH:PP_LL now holds the sum269 // We may need to normalize left, up to ??? bits.270 //271 // I think that if we have massive cancellation, the range we normalize by272 // is still limited273 {274 LEFTSHIFT = add(clb(PP_HH),#-2)275 if (!cmp.eq(LEFTSHIFT.new,TMP)) jump:t 1f // all sign bits?276 }277 // We had all sign bits, shift left by 62.278 {279 CTMP = extractu(PP_LL,#62,#2)280 PP_LL = asl(PP_LL,#62)281 EXPA = add(EXPA,#-62) // And adjust exponent of result282 }283 {284 PP_HH = insert(CTMP,#62,#0) // Then shift 63285 }286 {287 LEFTSHIFT = add(clb(PP_HH),#-2)288 }289 .falign2901:291 {292 CTMP = asl(PP_HH,LEFTSHIFT)293 STICKIES |= asl(PP_LL,LEFTSHIFT)294 RIGHTSHIFT = sub(#64,LEFTSHIFT)295 EXPA = sub(EXPA,LEFTSHIFT)296 }297 {298 CTMP |= lsr(PP_LL,RIGHTSHIFT)299 EXACT = cmp.gtu(ONE,STICKIES)300 TMP = #BIAS+BIAS-2301 }302 {303 if (!EXACT) CTMPL = or(CTMPL,S_ONE)304 // If EXPA is overflow/underflow, jump to ovf_unf305 P_TMP = !cmp.gt(EXPA,TMP)306 P_TMP = cmp.gt(EXPA,#1)307 if (!P_TMP.new) jump:nt .Lfma_ovf_unf308 }309 {310 // XXX: FIXME: should PP_HH for check of zero be CTMP?311 P_TMP = cmp.gtu(ONE,CTMP) // is result true zero?312 A = convert_d2df(CTMP)313 EXPA = add(EXPA,#-BIAS-60)314 PP_HH = memd(r29+#0)315 }316 {317 AH += asl(EXPA,#HI_MANTBITS)318 EXPCA = memd(r29+#8)319 if (!P_TMP) dealloc_return // not zero, return320 }321.Ladd_yields_zero:322 // We had full cancellation. Return +/- zero (-0 when round-down)323 {324 TMP = USR325 A = #0326 }327 {328 TMP = extractu(TMP,#2,#SR_ROUND_OFF)329 PP_HH = memd(r29+#0)330 EXPCA = memd(r29+#8)331 }332 {333 p0 = cmp.eq(TMP,#2)334 if (p0.new) AH = ##0x80000000335 dealloc_return336 }337 338#undef RIGHTLEFTSHIFT339#undef RIGHTSHIFT340#undef LEFTSHIFT341#undef CTMP2342#undef CTMP2H343#undef CTMP2L344 345.Lfma_ovf_unf:346 {347 p0 = cmp.gtu(ONE,CTMP)348 if (p0.new) jump:nt .Ladd_yields_zero349 }350 {351 A = convert_d2df(CTMP)352 EXPA = add(EXPA,#-BIAS-60)353 TMP = EXPA354 }355#define NEW_EXPB r7356#define NEW_EXPA r6357 {358 AH += asl(EXPA,#HI_MANTBITS)359 NEW_EXPB = extractu(AH,#EXPBITS,#HI_MANTBITS)360 }361 {362 NEW_EXPA = add(EXPA,NEW_EXPB)363 PP_HH = memd(r29+#0)364 EXPCA = memd(r29+#8)365#undef PP_HH366#undef PP_HH_H367#undef PP_HH_L368#undef EXPCA369#undef EXPC370#undef EXPA371#undef PP_LL372#undef PP_LL_H373#undef PP_LL_L374#define EXPA r6375#define EXPB r7376#define EXPBA r7:6377#define ATMP r9:8378#define ATMPH r9379#define ATMPL r8380#undef NEW_EXPB381#undef NEW_EXPA382 ATMP = abs(CTMP)383 }384 {385 p0 = cmp.gt(EXPA,##BIAS+BIAS)386 if (p0.new) jump:nt .Lfma_ovf387 }388 {389 p0 = cmp.gt(EXPA,#0)390 if (p0.new) jump:nt .Lpossible_unf391 }392 {393 // TMP has original EXPA.394 // ATMP is corresponding value395 // Normalize ATMP and shift right to correct location396 EXPB = add(clb(ATMP),#-2) // Amount to left shift to normalize397 EXPA = sub(#1+5,TMP) // Amount to right shift to denormalize398 p3 = cmp.gt(CTMPH,#-1)399 }400 // Underflow401 // We know that the infinte range exponent should be EXPA402 // CTMP is 2's complement, ATMP is abs(CTMP)403 {404 EXPA = add(EXPA,EXPB) // how much to shift back right405 ATMP = asl(ATMP,EXPB) // shift left406 AH = USR407 TMP = #63408 }409 {410 EXPB = min(EXPA,TMP)411 EXPA = #0412 AL = #0x0030413 }414 {415 B = extractu(ATMP,EXPBA)416 ATMP = asr(ATMP,EXPB)417 }418 {419 p0 = cmp.gtu(ONE,B)420 if (!p0.new) ATMPL = or(ATMPL,S_ONE)421 ATMPH = setbit(ATMPH,#HI_MANTBITS+FUDGE2)422 }423 {424 CTMP = neg(ATMP)425 p1 = bitsclr(ATMPL,#(1<<FUDGE2)-1)426 if (!p1.new) AH = or(AH,AL)427 B = #0428 }429 {430 if (p3) CTMP = ATMP431 USR = AH432 TMP = #-BIAS-(MANTBITS+FUDGE2)433 }434 {435 A = convert_d2df(CTMP)436 }437 {438 AH += asl(TMP,#HI_MANTBITS)439 dealloc_return440 }441.Lpossible_unf:442 {443 TMP = ##0x7fefffff444 ATMP = abs(CTMP)445 }446 {447 p0 = cmp.eq(AL,#0)448 p0 = bitsclr(AH,TMP)449 if (!p0.new) dealloc_return:t450 TMP = #0x7fff451 }452 {453 p0 = bitsset(ATMPH,TMP)454 BH = USR455 BL = #0x0030456 }457 {458 if (p0) BH = or(BH,BL)459 }460 {461 USR = BH462 }463 {464 p0 = dfcmp.eq(A,A)465 dealloc_return466 }467.Lfma_ovf:468 {469 TMP = USR470 CTMP = combine(##0x7fefffff,#-1)471 A = CTMP472 }473 {474 ATMP = combine(##0x7ff00000,#0)475 BH = extractu(TMP,#2,#SR_ROUND_OFF)476 TMP = or(TMP,#0x28)477 }478 {479 USR = TMP480 BH ^= lsr(AH,#31)481 BL = BH482 }483 {484 p0 = !cmp.eq(BL,#1)485 p0 = !cmp.eq(BH,#2)486 }487 {488 p0 = dfcmp.eq(ATMP,ATMP)489 if (p0.new) CTMP = ATMP490 }491 {492 A = insert(CTMP,#63,#0)493 dealloc_return494 }495#undef CTMP496#undef CTMPH497#undef CTMPL498#define BTMP r11:10499#define BTMPH r11500#define BTMPL r10501 502#undef STICKIES503#undef STICKIESH504#undef STICKIESL505#define C r5:4506#define CH r5507#define CL r4508 509.Lfma_abnormal_ab:510 {511 ATMP = extractu(A,#63,#0)512 BTMP = extractu(B,#63,#0)513 deallocframe514 }515 {516 p3 = cmp.gtu(ATMP,BTMP)517 if (!p3.new) A = B // sort values518 if (!p3.new) B = A519 }520 {521 p0 = dfclass(A,#0x0f) // A NaN?522 if (!p0.new) jump:nt .Lnan523 if (!p3) ATMP = BTMP524 if (!p3) BTMP = ATMP525 }526 {527 p1 = dfclass(A,#0x08) // A is infinity528 p1 = dfclass(B,#0x0e) // B is nonzero529 }530 {531 p0 = dfclass(A,#0x08) // a is inf532 p0 = dfclass(B,#0x01) // b is zero533 }534 {535 if (p1) jump .Lab_inf536 p2 = dfclass(B,#0x01)537 }538 {539 if (p0) jump .Linvalid540 if (p2) jump .Lab_true_zero541 TMP = ##0x7c000000542 }543 // We are left with a normal or subnormal times a subnormal, A > B544 // If A and B are both very small, we will go to a single sticky bit; replace545 // A and B lower 63 bits with 0x0010_0000_0000_0000, which yields equivalent results546 // if A and B might multiply to something bigger, decrease A exp and increase B exp547 // and start over548 {549 p0 = bitsclr(AH,TMP)550 if (p0.new) jump:nt .Lfma_ab_tiny551 }552 {553 TMP = add(clb(BTMP),#-EXPBITS)554 }555 {556 BTMP = asl(BTMP,TMP)557 }558 {559 B = insert(BTMP,#63,#0)560 AH -= asl(TMP,#HI_MANTBITS)561 }562 jump .Lfma_begin563 564.Lfma_ab_tiny:565 ATMP = combine(##0x00100000,#0)566 {567 A = insert(ATMP,#63,#0)568 B = insert(ATMP,#63,#0)569 }570 jump .Lfma_begin571 572.Lab_inf:573 {574 B = lsr(B,#63)575 p0 = dfclass(C,#0x10)576 }577 {578 A ^= asl(B,#63)579 if (p0) jump .Lnan580 }581 {582 p1 = dfclass(C,#0x08)583 if (p1.new) jump:nt .Lfma_inf_plus_inf584 }585 // A*B is +/- inf, C is finite. Return A586 {587 jumpr r31588 }589 .falign590.Lfma_inf_plus_inf:591 { // adding infinities of different signs is invalid592 p0 = dfcmp.eq(A,C)593 if (!p0.new) jump:nt .Linvalid594 }595 {596 jumpr r31597 }598 599.Lnan:600 {601 p0 = dfclass(B,#0x10)602 p1 = dfclass(C,#0x10)603 if (!p0.new) B = A604 if (!p1.new) C = A605 }606 { // find sNaNs607 BH = convert_df2sf(B)608 BL = convert_df2sf(C)609 }610 {611 BH = convert_df2sf(A)612 A = #-1613 jumpr r31614 }615 616.Linvalid:617 {618 TMP = ##0x7f800001 // sp snan619 }620 {621 A = convert_sf2df(TMP)622 jumpr r31623 }624 625.Lab_true_zero:626 // B is zero, A is finite number627 {628 p0 = dfclass(C,#0x10)629 if (p0.new) jump:nt .Lnan630 if (p0.new) A = C631 }632 {633 p0 = dfcmp.eq(B,C) // is C also zero?634 AH = lsr(AH,#31) // get sign635 }636 {637 BH ^= asl(AH,#31) // form correctly signed zero in B638 if (!p0) A = C // If C is not zero, return C639 if (!p0) jumpr r31640 }641 // B has correctly signed zero, C is also zero642.Lzero_plus_zero:643 {644 p0 = cmp.eq(B,C) // yes, scalar equals. +0++0 or -0+-0645 if (p0.new) jumpr:t r31646 A = B647 }648 {649 TMP = USR650 }651 {652 TMP = extractu(TMP,#2,#SR_ROUND_OFF)653 A = #0654 }655 {656 p0 = cmp.eq(TMP,#2)657 if (p0.new) AH = ##0x80000000658 jumpr r31659 }660#undef BTMP661#undef BTMPH662#undef BTMPL663#define CTMP r11:10664 .falign665.Lfma_abnormal_c:666 // We know that AB is normal * normal667 // C is not normal: zero, subnormal, inf, or NaN.668 {669 p0 = dfclass(C,#0x10) // is C NaN?670 if (p0.new) jump:nt .Lnan671 if (p0.new) A = C // move NaN to A672 deallocframe673 }674 {675 p0 = dfclass(C,#0x08) // is C inf?676 if (p0.new) A = C // return C677 if (p0.new) jumpr:nt r31678 }679 // zero or subnormal680 // If we have a zero, and we know AB is normal*normal, we can just call normal multiply681 {682 p0 = dfclass(C,#0x01) // is C zero?683 if (p0.new) jump:nt __hexagon_muldf3684 TMP = #1685 }686 // Left with: subnormal687 // Adjust C and jump back to restart688 {689 allocframe(#STACKSPACE) // oops, deallocated above, re-allocate frame690 CTMP = #0691 CH = insert(TMP,#EXPBITS,#HI_MANTBITS)692 jump .Lfma_abnormal_c_restart693 }694END(fma)695