2524 lines · plain
1// z_Linux_asm.S: - microtasking routines specifically2// written for Intel platforms running Linux* OS3 4//5////===----------------------------------------------------------------------===//6////7//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.8//// See https://llvm.org/LICENSE.txt for license information.9//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception10////11////===----------------------------------------------------------------------===//12//13 14// -----------------------------------------------------------------------15// macros16// -----------------------------------------------------------------------17 18#include "kmp_config.h"19 20#if KMP_ARCH_X86 || KMP_ARCH_X86_6421.att_syntax22 23# if defined(__ELF__) && defined(__CET__) && defined(__has_include)24# if __has_include(<cet.h>)25# include <cet.h>26# endif27# endif28 29# if !defined(_CET_ENDBR)30# define _CET_ENDBR31# endif32 33# if KMP_MIC34// the 'delay r16/r32/r64' should be used instead of the 'pause'.35// The delay operation has the effect of removing the current thread from36// the round-robin HT mechanism, and therefore speeds up the issue rate of37// the other threads on the same core.38//39// A value of 0 works fine for <= 2 threads per core, but causes the EPCC40// barrier time to increase greatly for 3 or more threads per core.41//42// A value of 100 works pretty well for up to 4 threads per core, but isn't43// quite as fast as 0 for 2 threads per core.44//45// We need to check what happens for oversubscription / > 4 threads per core.46// It is possible that we need to pass the delay value in as a parameter47// that the caller determines based on the total # threads / # cores.48//49//.macro pause_op50// mov $100, %rax51// delay %rax52//.endm53# else54# define pause_op .byte 0xf3,0x9055# endif // KMP_MIC56 57# if KMP_OS_DARWIN58# define KMP_PREFIX_UNDERSCORE(x) _##x // extra underscore for OS X* symbols59# define KMP_LABEL(x) L_##x // form the name of label60.macro KMP_CFI_DEF_OFFSET61.endmacro62.macro KMP_CFI_OFFSET63.endmacro64.macro KMP_CFI_REGISTER65.endmacro66.macro KMP_CFI_DEF67.endmacro68.macro ALIGN69 .align $070.endmacro71.macro DEBUG_INFO72/* Not sure what .size does in icc, not sure if we need to do something73 similar for OS X*.74*/75.endmacro76.macro PROC77 ALIGN 478 .globl KMP_PREFIX_UNDERSCORE($0)79KMP_PREFIX_UNDERSCORE($0):80 _CET_ENDBR81.endmacro82# else // KMP_OS_DARWIN83# define KMP_PREFIX_UNDERSCORE(x) x //no extra underscore for Linux* OS symbols84// Format labels so that they don't override function names in gdb's backtraces85// MIC assembler doesn't accept .L syntax, the L works fine there (as well as86// on OS X*)87# if KMP_MIC88# define KMP_LABEL(x) L_##x // local label89# else90# define KMP_LABEL(x) .L_##x // local label hidden from backtraces91# endif // KMP_MIC92.macro ALIGN size93 .align 1<<(\size)94.endm95.macro DEBUG_INFO proc96 .cfi_endproc97// Not sure why we need .type and .size for the functions98 .align 1699 .type \proc,@function100 .size \proc,.-\proc101.endm102.macro PROC proc103 ALIGN 4104 .globl KMP_PREFIX_UNDERSCORE(\proc)105KMP_PREFIX_UNDERSCORE(\proc):106 .cfi_startproc107 _CET_ENDBR108.endm109.macro KMP_CFI_DEF_OFFSET sz110 .cfi_def_cfa_offset \sz111.endm112.macro KMP_CFI_OFFSET reg, sz113 .cfi_offset \reg,\sz114.endm115.macro KMP_CFI_REGISTER reg116 .cfi_def_cfa_register \reg117.endm118.macro KMP_CFI_DEF reg, sz119 .cfi_def_cfa \reg,\sz120.endm121# endif // KMP_OS_DARWIN122#endif // KMP_ARCH_X86 || KMP_ARCH_x86_64123 124#if KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32 || KMP_ARCH_ARM125 126# if KMP_OS_DARWIN127# define KMP_PREFIX_UNDERSCORE(x) _##x // extra underscore for OS X* symbols128# define KMP_LABEL(x) L_##x // form the name of label129 130.macro ALIGN131 .align $0132.endmacro133 134.macro DEBUG_INFO135/* Not sure what .size does in icc, not sure if we need to do something136 similar for OS X*.137*/138.endmacro139 140.macro PROC141 ALIGN 4142 .globl KMP_PREFIX_UNDERSCORE($0)143KMP_PREFIX_UNDERSCORE($0):144.endmacro145# elif KMP_OS_WINDOWS146# define KMP_PREFIX_UNDERSCORE(x) x // no extra underscore for Windows/ARM64 symbols147// Format labels so that they don't override function names in gdb's backtraces148# define KMP_LABEL(x) .L_##x // local label hidden from backtraces149 150.macro ALIGN size151 .align 1<<(\size)152.endm153 154.macro DEBUG_INFO proc155 ALIGN 2156.endm157 158.macro PROC proc159 ALIGN 2160 .globl KMP_PREFIX_UNDERSCORE(\proc)161KMP_PREFIX_UNDERSCORE(\proc):162.endm163# else // KMP_OS_DARWIN || KMP_OS_WINDOWS164# define KMP_PREFIX_UNDERSCORE(x) x // no extra underscore for Linux* OS symbols165// Format labels so that they don't override function names in gdb's backtraces166# define KMP_LABEL(x) .L_##x // local label hidden from backtraces167 168.macro ALIGN size169 .align 1<<(\size)170.endm171 172.macro DEBUG_INFO proc173 .cfi_endproc174// Not sure why we need .type and .size for the functions175 ALIGN 2176#if KMP_ARCH_ARM177 .type \proc,%function178#else179 .type \proc,@function180#endif181 .size \proc,.-\proc182.endm183 184.macro PROC proc185 ALIGN 2186 .globl KMP_PREFIX_UNDERSCORE(\proc)187KMP_PREFIX_UNDERSCORE(\proc):188 .cfi_startproc189.endm190# endif // KMP_OS_DARWIN191 192# if KMP_OS_LINUX || KMP_OS_OPENBSD193// BTI and PAC gnu property note194# define NT_GNU_PROPERTY_TYPE_0 5195# define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000196# define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1197# define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2198 199# define GNU_PROPERTY(type, value) \200 .pushsection .note.gnu.property, "a"; \201 .p2align 3; \202 .word 4; \203 .word 16; \204 .word NT_GNU_PROPERTY_TYPE_0; \205 .asciz "GNU"; \206 .word type; \207 .word 4; \208 .word value; \209 .word 0; \210 .popsection211# endif212 213# if defined(__ARM_FEATURE_BTI_DEFAULT)214# define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI215# else216# define BTI_FLAG 0217# endif218# if __ARM_FEATURE_PAC_DEFAULT & 3219# define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC220# else221# define PAC_FLAG 0222# endif223 224# if (BTI_FLAG | PAC_FLAG) != 0225# if PAC_FLAG != 0226# define PACBTI_C hint #25227# define PACBTI_RET hint #29228# else229# define PACBTI_C hint #34230# define PACBTI_RET231# endif232# define GNU_PROPERTY_BTI_PAC \233 GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG)234# else235# define PACBTI_C236# define PACBTI_RET237# define GNU_PROPERTY_BTI_PAC238# endif239#endif // KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32 || KMP_ARCH_ARM240 241.macro COMMON name, size, align_power242#if KMP_OS_DARWIN243 .comm \name, \size244#elif KMP_OS_WINDOWS245 .comm \name, \size, \align_power246#else // !KMP_OS_DARWIN && !KMP_OS_WINDOWS247 .comm \name, \size, (1<<(\align_power))248#endif249.endm250 251// -----------------------------------------------------------------------252// data253// -----------------------------------------------------------------------254 255#ifdef KMP_GOMP_COMPAT256 257// Support for unnamed common blocks.258//259// Because the symbol ".gomp_critical_user_" contains a ".", we have to260// put this stuff in assembly.261 262# if KMP_ARCH_X86263# if KMP_OS_DARWIN264 .data265 .comm .gomp_critical_user_,32266 .data267 .globl ___kmp_unnamed_critical_addr268___kmp_unnamed_critical_addr:269 .long .gomp_critical_user_270# else /* Linux* OS */271 .data272 .comm .gomp_critical_user_,32,8273 .data274 ALIGN 4275 .global __kmp_unnamed_critical_addr276__kmp_unnamed_critical_addr:277 .4byte .gomp_critical_user_278 .type __kmp_unnamed_critical_addr,@object279 .size __kmp_unnamed_critical_addr,4280# endif /* KMP_OS_DARWIN */281# endif /* KMP_ARCH_X86 */282 283# if KMP_ARCH_X86_64284# if KMP_OS_DARWIN285 .data286 .comm .gomp_critical_user_,32287 .data288 .globl ___kmp_unnamed_critical_addr289___kmp_unnamed_critical_addr:290 .quad .gomp_critical_user_291# else /* Linux* OS */292 .data293 .comm .gomp_critical_user_,32,8294 .data295 ALIGN 8296 .global __kmp_unnamed_critical_addr297__kmp_unnamed_critical_addr:298 .8byte .gomp_critical_user_299 .type __kmp_unnamed_critical_addr,@object300 .size __kmp_unnamed_critical_addr,8301# endif /* KMP_OS_DARWIN */302# endif /* KMP_ARCH_X86_64 */303 304#endif /* KMP_GOMP_COMPAT */305 306 307#if KMP_ARCH_X86 && !KMP_ARCH_PPC64308 309// -----------------------------------------------------------------------310// microtasking routines specifically written for IA-32 architecture311// running Linux* OS312// -----------------------------------------------------------------------313 314 .ident "Intel Corporation"315 .data316 ALIGN 4317// void318// __kmp_x86_pause( void );319 320 .text321 PROC __kmp_x86_pause322 323 pause_op324 ret325 326 DEBUG_INFO __kmp_x86_pause327 328# if !KMP_ASM_INTRINS329 330//------------------------------------------------------------------------331// kmp_int32332// __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d );333 334 PROC __kmp_test_then_add32335 336 movl 4(%esp), %ecx337 movl 8(%esp), %eax338 lock339 xaddl %eax,(%ecx)340 ret341 342 DEBUG_INFO __kmp_test_then_add32343 344//------------------------------------------------------------------------345// FUNCTION __kmp_xchg_fixed8346//347// kmp_int32348// __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d );349//350// parameters:351// p: 4(%esp)352// d: 8(%esp)353//354// return: %al355 PROC __kmp_xchg_fixed8356 357 movl 4(%esp), %ecx // "p"358 movb 8(%esp), %al // "d"359 360 lock361 xchgb %al,(%ecx)362 ret363 364 DEBUG_INFO __kmp_xchg_fixed8365 366 367//------------------------------------------------------------------------368// FUNCTION __kmp_xchg_fixed16369//370// kmp_int16371// __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d );372//373// parameters:374// p: 4(%esp)375// d: 8(%esp)376// return: %ax377 PROC __kmp_xchg_fixed16378 379 movl 4(%esp), %ecx // "p"380 movw 8(%esp), %ax // "d"381 382 lock383 xchgw %ax,(%ecx)384 ret385 386 DEBUG_INFO __kmp_xchg_fixed16387 388 389//------------------------------------------------------------------------390// FUNCTION __kmp_xchg_fixed32391//392// kmp_int32393// __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d );394//395// parameters:396// p: 4(%esp)397// d: 8(%esp)398//399// return: %eax400 PROC __kmp_xchg_fixed32401 402 movl 4(%esp), %ecx // "p"403 movl 8(%esp), %eax // "d"404 405 lock406 xchgl %eax,(%ecx)407 ret408 409 DEBUG_INFO __kmp_xchg_fixed32410 411 412// kmp_int8413// __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );414 PROC __kmp_compare_and_store8415 416 movl 4(%esp), %ecx417 movb 8(%esp), %al418 movb 12(%esp), %dl419 lock420 cmpxchgb %dl,(%ecx)421 sete %al // if %al == (%ecx) set %al = 1 else set %al = 0422 and $1, %eax // sign extend previous instruction423 ret424 425 DEBUG_INFO __kmp_compare_and_store8426 427// kmp_int16428// __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv);429 PROC __kmp_compare_and_store16430 431 movl 4(%esp), %ecx432 movw 8(%esp), %ax433 movw 12(%esp), %dx434 lock435 cmpxchgw %dx,(%ecx)436 sete %al // if %ax == (%ecx) set %al = 1 else set %al = 0437 and $1, %eax // sign extend previous instruction438 ret439 440 DEBUG_INFO __kmp_compare_and_store16441 442// kmp_int32443// __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv);444 PROC __kmp_compare_and_store32445 446 movl 4(%esp), %ecx447 movl 8(%esp), %eax448 movl 12(%esp), %edx449 lock450 cmpxchgl %edx,(%ecx)451 sete %al // if %eax == (%ecx) set %al = 1 else set %al = 0452 and $1, %eax // sign extend previous instruction453 ret454 455 DEBUG_INFO __kmp_compare_and_store32456 457// kmp_int32458// __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 s );459 PROC __kmp_compare_and_store64460 461 pushl %ebp462 movl %esp, %ebp463 pushl %ebx464 pushl %edi465 movl 8(%ebp), %edi466 movl 12(%ebp), %eax // "cv" low order word467 movl 16(%ebp), %edx // "cv" high order word468 movl 20(%ebp), %ebx // "sv" low order word469 movl 24(%ebp), %ecx // "sv" high order word470 lock471 cmpxchg8b (%edi)472 sete %al // if %edx:eax == (%edi) set %al = 1 else set %al = 0473 and $1, %eax // sign extend previous instruction474 popl %edi475 popl %ebx476 movl %ebp, %esp477 popl %ebp478 ret479 480 DEBUG_INFO __kmp_compare_and_store64481 482// kmp_int8483// __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv);484 PROC __kmp_compare_and_store_ret8485 486 movl 4(%esp), %ecx487 movb 8(%esp), %al488 movb 12(%esp), %dl489 lock490 cmpxchgb %dl,(%ecx)491 ret492 493 DEBUG_INFO __kmp_compare_and_store_ret8494 495// kmp_int16496// __kmp_compare_and_store_ret16(volatile kmp_int16 *p, kmp_int16 cv,497// kmp_int16 sv);498 PROC __kmp_compare_and_store_ret16499 500 movl 4(%esp), %ecx501 movw 8(%esp), %ax502 movw 12(%esp), %dx503 lock504 cmpxchgw %dx,(%ecx)505 ret506 507 DEBUG_INFO __kmp_compare_and_store_ret16508 509// kmp_int32510// __kmp_compare_and_store_ret32(volatile kmp_int32 *p, kmp_int32 cv,511// kmp_int32 sv);512 PROC __kmp_compare_and_store_ret32513 514 movl 4(%esp), %ecx515 movl 8(%esp), %eax516 movl 12(%esp), %edx517 lock518 cmpxchgl %edx,(%ecx)519 ret520 521 DEBUG_INFO __kmp_compare_and_store_ret32522 523// kmp_int64524// __kmp_compare_and_store_ret64(volatile kmp_int64 *p, kmp_int64 cv,525// kmp_int64 sv);526 PROC __kmp_compare_and_store_ret64527 528 pushl %ebp529 movl %esp, %ebp530 pushl %ebx531 pushl %edi532 movl 8(%ebp), %edi533 movl 12(%ebp), %eax // "cv" low order word534 movl 16(%ebp), %edx // "cv" high order word535 movl 20(%ebp), %ebx // "sv" low order word536 movl 24(%ebp), %ecx // "sv" high order word537 lock538 cmpxchg8b (%edi)539 popl %edi540 popl %ebx541 movl %ebp, %esp542 popl %ebp543 ret544 545 DEBUG_INFO __kmp_compare_and_store_ret64546 547 548//------------------------------------------------------------------------549// FUNCTION __kmp_xchg_real32550//551// kmp_real32552// __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data );553//554// parameters:555// addr: 4(%esp)556// data: 8(%esp)557//558// return: %eax559 PROC __kmp_xchg_real32560 561 pushl %ebp562 movl %esp, %ebp563 subl $4, %esp564 pushl %esi565 566 movl 4(%ebp), %esi567 flds (%esi)568 // load <addr>569 fsts -4(%ebp)570 // store old value571 572 movl 8(%ebp), %eax573 574 lock575 xchgl %eax, (%esi)576 577 flds -4(%ebp)578 // return old value579 580 popl %esi581 movl %ebp, %esp582 popl %ebp583 ret584 585 DEBUG_INFO __kmp_xchg_real32586 587# endif /* !KMP_ASM_INTRINS */588 589//------------------------------------------------------------------------590// int591// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),592// int gtid, int tid,593// int argc, void *p_argv[]594// #if OMPT_SUPPORT595// ,596// void **exit_frame_ptr597// #endif598// ) {599// #if OMPT_SUPPORT600// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);601// #endif602//603// (*pkfn)( & gtid, & tid, argv[0], ... );604// return 1;605// }606 607// -- Begin __kmp_invoke_microtask608// mark_begin;609 PROC __kmp_invoke_microtask610 611 pushl %ebp612 KMP_CFI_DEF_OFFSET 8613 KMP_CFI_OFFSET ebp,-8614 movl %esp,%ebp // establish the base pointer for this routine.615 KMP_CFI_REGISTER ebp616 subl $8,%esp // allocate space for two local variables.617 // These varibales are:618 // argv: -4(%ebp)619 // temp: -8(%ebp)620 //621 pushl %ebx // save %ebx to use during this routine622 //623#if OMPT_SUPPORT624 movl 28(%ebp),%ebx // get exit_frame address625 movl %ebp,(%ebx) // save exit_frame626#endif627 628 movl 20(%ebp),%ebx // Stack alignment - # args629 addl $2,%ebx // #args +2 Always pass at least 2 args (gtid and tid)630 shll $2,%ebx // Number of bytes used on stack: (#args+2)*4631 movl %esp,%eax //632 subl %ebx,%eax // %esp-((#args+2)*4) -> %eax -- without mods, stack ptr would be this633 movl %eax,%ebx // Save to %ebx634 andl $0xFFFFFF80,%eax // mask off 7 bits635 subl %eax,%ebx // Amount to subtract from %esp636 subl %ebx,%esp // Prepare the stack ptr --637 // now it will be aligned on 128-byte boundary at the call638 639 movl 24(%ebp),%eax // copy from p_argv[]640 movl %eax,-4(%ebp) // into the local variable *argv.641 642 movl 20(%ebp),%ebx // argc is 20(%ebp)643 shll $2,%ebx644 645KMP_LABEL(invoke_2):646 cmpl $0,%ebx647 jg KMP_LABEL(invoke_4)648 jmp KMP_LABEL(invoke_3)649 ALIGN 2650KMP_LABEL(invoke_4):651 movl -4(%ebp),%eax652 subl $4,%ebx // decrement argc.653 addl %ebx,%eax // index into argv.654 movl (%eax),%edx655 pushl %edx656 657 jmp KMP_LABEL(invoke_2)658 ALIGN 2659KMP_LABEL(invoke_3):660 leal 16(%ebp),%eax // push & tid661 pushl %eax662 663 leal 12(%ebp),%eax // push & gtid664 pushl %eax665 666 movl 8(%ebp),%ebx667 call *%ebx // call (*pkfn)();668 669 movl $1,%eax // return 1;670 671 movl -12(%ebp),%ebx // restore %ebx672 leave673 KMP_CFI_DEF esp,4674 ret675 676 DEBUG_INFO __kmp_invoke_microtask677// -- End __kmp_invoke_microtask678 679 680// kmp_uint64681// __kmp_hardware_timestamp(void)682 PROC __kmp_hardware_timestamp683 rdtsc684 ret685 686 DEBUG_INFO __kmp_hardware_timestamp687// -- End __kmp_hardware_timestamp688 689#endif /* KMP_ARCH_X86 */690 691 692#if KMP_ARCH_X86_64693 694// -----------------------------------------------------------------------695// microtasking routines specifically written for IA-32 architecture and696// Intel(R) 64 running Linux* OS697// -----------------------------------------------------------------------698 699// -- Machine type P700// mark_description "Intel Corporation";701 .ident "Intel Corporation"702// -- .file "z_Linux_asm.S"703 .data704 ALIGN 4705 706// To prevent getting our code into .data section .text added to every routine707// definition for x86_64.708//------------------------------------------------------------------------709# if !KMP_ASM_INTRINS710 711//------------------------------------------------------------------------712// FUNCTION __kmp_test_then_add32713//714// kmp_int32715// __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d );716//717// parameters:718// p: %rdi719// d: %esi720//721// return: %eax722 .text723 PROC __kmp_test_then_add32724 725 movl %esi, %eax // "d"726 lock727 xaddl %eax,(%rdi)728 ret729 730 DEBUG_INFO __kmp_test_then_add32731 732 733//------------------------------------------------------------------------734// FUNCTION __kmp_test_then_add64735//736// kmp_int64737// __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d );738//739// parameters:740// p: %rdi741// d: %rsi742// return: %rax743 .text744 PROC __kmp_test_then_add64745 746 movq %rsi, %rax // "d"747 lock748 xaddq %rax,(%rdi)749 ret750 751 DEBUG_INFO __kmp_test_then_add64752 753 754//------------------------------------------------------------------------755// FUNCTION __kmp_xchg_fixed8756//757// kmp_int32758// __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d );759//760// parameters:761// p: %rdi762// d: %sil763//764// return: %al765 .text766 PROC __kmp_xchg_fixed8767 768 movb %sil, %al // "d"769 770 lock771 xchgb %al,(%rdi)772 ret773 774 DEBUG_INFO __kmp_xchg_fixed8775 776 777//------------------------------------------------------------------------778// FUNCTION __kmp_xchg_fixed16779//780// kmp_int16781// __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d );782//783// parameters:784// p: %rdi785// d: %si786// return: %ax787 .text788 PROC __kmp_xchg_fixed16789 790 movw %si, %ax // "d"791 792 lock793 xchgw %ax,(%rdi)794 ret795 796 DEBUG_INFO __kmp_xchg_fixed16797 798 799//------------------------------------------------------------------------800// FUNCTION __kmp_xchg_fixed32801//802// kmp_int32803// __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d );804//805// parameters:806// p: %rdi807// d: %esi808//809// return: %eax810 .text811 PROC __kmp_xchg_fixed32812 813 movl %esi, %eax // "d"814 815 lock816 xchgl %eax,(%rdi)817 ret818 819 DEBUG_INFO __kmp_xchg_fixed32820 821 822//------------------------------------------------------------------------823// FUNCTION __kmp_xchg_fixed64824//825// kmp_int64826// __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 d );827//828// parameters:829// p: %rdi830// d: %rsi831// return: %rax832 .text833 PROC __kmp_xchg_fixed64834 835 movq %rsi, %rax // "d"836 837 lock838 xchgq %rax,(%rdi)839 ret840 841 DEBUG_INFO __kmp_xchg_fixed64842 843 844//------------------------------------------------------------------------845// FUNCTION __kmp_compare_and_store8846//847// kmp_int8848// __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );849//850// parameters:851// p: %rdi852// cv: %esi853// sv: %edx854//855// return: %eax856 .text857 PROC __kmp_compare_and_store8858 859 movb %sil, %al // "cv"860 lock861 cmpxchgb %dl,(%rdi)862 sete %al // if %al == (%rdi) set %al = 1 else set %al = 0863 andq $1, %rax // sign extend previous instruction for return value864 ret865 866 DEBUG_INFO __kmp_compare_and_store8867 868 869//------------------------------------------------------------------------870// FUNCTION __kmp_compare_and_store16871//872// kmp_int16873// __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );874//875// parameters:876// p: %rdi877// cv: %si878// sv: %dx879//880// return: %eax881 .text882 PROC __kmp_compare_and_store16883 884 movw %si, %ax // "cv"885 lock886 cmpxchgw %dx,(%rdi)887 sete %al // if %ax == (%rdi) set %al = 1 else set %al = 0888 andq $1, %rax // sign extend previous instruction for return value889 ret890 891 DEBUG_INFO __kmp_compare_and_store16892 893 894//------------------------------------------------------------------------895// FUNCTION __kmp_compare_and_store32896//897// kmp_int32898// __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );899//900// parameters:901// p: %rdi902// cv: %esi903// sv: %edx904//905// return: %eax906 .text907 PROC __kmp_compare_and_store32908 909 movl %esi, %eax // "cv"910 lock911 cmpxchgl %edx,(%rdi)912 sete %al // if %eax == (%rdi) set %al = 1 else set %al = 0913 andq $1, %rax // sign extend previous instruction for return value914 ret915 916 DEBUG_INFO __kmp_compare_and_store32917 918 919//------------------------------------------------------------------------920// FUNCTION __kmp_compare_and_store64921//922// kmp_int32923// __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );924//925// parameters:926// p: %rdi927// cv: %rsi928// sv: %rdx929// return: %eax930 .text931 PROC __kmp_compare_and_store64932 933 movq %rsi, %rax // "cv"934 lock935 cmpxchgq %rdx,(%rdi)936 sete %al // if %rax == (%rdi) set %al = 1 else set %al = 0937 andq $1, %rax // sign extend previous instruction for return value938 ret939 940 DEBUG_INFO __kmp_compare_and_store64941 942//------------------------------------------------------------------------943// FUNCTION __kmp_compare_and_store_ret8944//945// kmp_int8946// __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );947//948// parameters:949// p: %rdi950// cv: %esi951// sv: %edx952//953// return: %eax954 .text955 PROC __kmp_compare_and_store_ret8956 957 movb %sil, %al // "cv"958 lock959 cmpxchgb %dl,(%rdi)960 ret961 962 DEBUG_INFO __kmp_compare_and_store_ret8963 964 965//------------------------------------------------------------------------966// FUNCTION __kmp_compare_and_store_ret16967//968// kmp_int16969// __kmp_compare_and_store16_ret( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );970//971// parameters:972// p: %rdi973// cv: %si974// sv: %dx975//976// return: %eax977 .text978 PROC __kmp_compare_and_store_ret16979 980 movw %si, %ax // "cv"981 lock982 cmpxchgw %dx,(%rdi)983 ret984 985 DEBUG_INFO __kmp_compare_and_store_ret16986 987 988//------------------------------------------------------------------------989// FUNCTION __kmp_compare_and_store_ret32990//991// kmp_int32992// __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );993//994// parameters:995// p: %rdi996// cv: %esi997// sv: %edx998//999// return: %eax1000 .text1001 PROC __kmp_compare_and_store_ret321002 1003 movl %esi, %eax // "cv"1004 lock1005 cmpxchgl %edx,(%rdi)1006 ret1007 1008 DEBUG_INFO __kmp_compare_and_store_ret321009 1010 1011//------------------------------------------------------------------------1012// FUNCTION __kmp_compare_and_store_ret641013//1014// kmp_int641015// __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );1016//1017// parameters:1018// p: %rdi1019// cv: %rsi1020// sv: %rdx1021// return: %eax1022 .text1023 PROC __kmp_compare_and_store_ret641024 1025 movq %rsi, %rax // "cv"1026 lock1027 cmpxchgq %rdx,(%rdi)1028 ret1029 1030 DEBUG_INFO __kmp_compare_and_store_ret641031 1032# endif /* !KMP_ASM_INTRINS */1033 1034 1035# if !KMP_MIC1036 1037# if !KMP_ASM_INTRINS1038 1039//------------------------------------------------------------------------1040// FUNCTION __kmp_xchg_real321041//1042// kmp_real321043// __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data );1044//1045// parameters:1046// addr: %rdi1047// data: %xmm0 (lower 4 bytes)1048//1049// return: %xmm0 (lower 4 bytes)1050 .text1051 PROC __kmp_xchg_real321052 1053 movd %xmm0, %eax // load "data" to eax1054 1055 lock1056 xchgl %eax, (%rdi)1057 1058 movd %eax, %xmm0 // load old value into return register1059 1060 ret1061 1062 DEBUG_INFO __kmp_xchg_real321063 1064 1065//------------------------------------------------------------------------1066// FUNCTION __kmp_xchg_real641067//1068// kmp_real641069// __kmp_xchg_real64( volatile kmp_real64 *addr, kmp_real64 data );1070//1071// parameters:1072// addr: %rdi1073// data: %xmm0 (lower 8 bytes)1074// return: %xmm0 (lower 8 bytes)1075 .text1076 PROC __kmp_xchg_real641077 1078 movd %xmm0, %rax // load "data" to rax1079 1080 lock1081 xchgq %rax, (%rdi)1082 1083 movd %rax, %xmm0 // load old value into return register1084 ret1085 1086 DEBUG_INFO __kmp_xchg_real641087 1088 1089# endif /* !KMP_MIC */1090 1091# endif /* !KMP_ASM_INTRINS */1092 1093//------------------------------------------------------------------------1094// int1095// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),1096// int gtid, int tid,1097// int argc, void *p_argv[]1098// #if OMPT_SUPPORT1099// ,1100// void **exit_frame_ptr1101// #endif1102// ) {1103// #if OMPT_SUPPORT1104// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);1105// #endif1106//1107// (*pkfn)( & gtid, & tid, argv[0], ... );1108// return 1;1109// }1110//1111// note: at call to pkfn must have %rsp 128-byte aligned for compiler1112//1113// parameters:1114// %rdi: pkfn1115// %esi: gtid1116// %edx: tid1117// %ecx: argc1118// %r8: p_argv1119// %r9: &exit_frame1120//1121// locals:1122// __gtid: gtid parm pushed on stack so can pass >id to pkfn1123// __tid: tid parm pushed on stack so can pass &tid to pkfn1124//1125// reg temps:1126// %rax: used all over the place1127// %rdx: used in stack pointer alignment calculation1128// %r11: used to traverse p_argv array1129// %rsi: used as temporary for stack parameters1130// used as temporary for number of pkfn parms to push1131// %rbx: used to hold pkfn address, and zero constant, callee-save1132//1133// return: %eax (always 1/TRUE)1134__gtid = -161135__tid = -241136 1137// -- Begin __kmp_invoke_microtask1138// mark_begin;1139 .text1140 PROC __kmp_invoke_microtask1141 1142 pushq %rbp // save base pointer1143 KMP_CFI_DEF_OFFSET 161144 KMP_CFI_OFFSET rbp,-161145 movq %rsp,%rbp // establish the base pointer for this routine.1146 KMP_CFI_REGISTER rbp1147 1148#if OMPT_SUPPORT1149 movq %rbp, (%r9) // save exit_frame1150#endif1151 1152 pushq %rbx // %rbx is callee-saved register1153 pushq %rsi // Put gtid on stack so can pass &tgid to pkfn1154 pushq %rdx // Put tid on stack so can pass &tid to pkfn1155 1156 movq %rcx, %rax // Stack alignment calculation begins; argc -> %rax1157 movq $0, %rbx // constant for cmovs later1158 subq $4, %rax // subtract four args passed in registers to pkfn1159#if KMP_MIC1160 js KMP_LABEL(kmp_0) // jump to movq1161 jmp KMP_LABEL(kmp_0_exit) // jump ahead1162KMP_LABEL(kmp_0):1163 movq %rbx, %rax // zero negative value in %rax <- max(0, argc-4)1164KMP_LABEL(kmp_0_exit):1165#else1166 cmovsq %rbx, %rax // zero negative value in %rax <- max(0, argc-4)1167#endif // KMP_MIC1168 1169 movq %rax, %rsi // save max(0, argc-4) -> %rsi for later1170 shlq $3, %rax // Number of bytes used on stack: max(0, argc-4)*81171 1172 movq %rsp, %rdx //1173 subq %rax, %rdx // %rsp-(max(0,argc-4)*8) -> %rdx --1174 // without align, stack ptr would be this1175 movq %rdx, %rax // Save to %rax1176 1177 andq $0xFFFFFFFFFFFFFF80, %rax // mask off lower 7 bits (128 bytes align)1178 subq %rax, %rdx // Amount to subtract from %rsp1179 subq %rdx, %rsp // Prepare the stack ptr --1180 // now %rsp will align to 128-byte boundary at call site1181 1182 // setup pkfn parameter reg and stack1183 movq %rcx, %rax // argc -> %rax1184 cmpq $0, %rsi1185 je KMP_LABEL(kmp_invoke_pass_parms) // jump ahead if no parms to push1186 shlq $3, %rcx // argc*8 -> %rcx1187 movq %r8, %rdx // p_argv -> %rdx1188 addq %rcx, %rdx // &p_argv[argc] -> %rdx1189 1190 movq %rsi, %rcx // max (0, argc-4) -> %rcx1191 1192KMP_LABEL(kmp_invoke_push_parms):1193 // push nth - 7th parms to pkfn on stack1194 subq $8, %rdx // decrement p_argv pointer to previous parm1195 movq (%rdx), %rsi // p_argv[%rcx-1] -> %rsi1196 pushq %rsi // push p_argv[%rcx-1] onto stack (reverse order)1197 subl $1, %ecx1198 1199// C69570: "X86_64_RELOC_BRANCH not supported" error at linking on mac_32e1200// if the name of the label that is an operand of this jecxz starts with a dot (".");1201// Apple's linker does not support 1-byte length relocation;1202// Resolution: replace all .labelX entries with L_labelX.1203 1204 jecxz KMP_LABEL(kmp_invoke_pass_parms) // stop when four p_argv[] parms left1205 jmp KMP_LABEL(kmp_invoke_push_parms)1206 ALIGN 31207KMP_LABEL(kmp_invoke_pass_parms): // put 1st - 6th parms to pkfn in registers.1208 // order here is important to avoid trashing1209 // registers used for both input and output parms!1210 movq %rdi, %rbx // pkfn -> %rbx1211 leaq __gtid(%rbp), %rdi // >id -> %rdi (store 1st parm to pkfn)1212 leaq __tid(%rbp), %rsi // &tid -> %rsi (store 2nd parm to pkfn)1213 // Check if argc is 01214 cmpq $0, %rax1215 je KMP_LABEL(kmp_no_args) // Jump ahead1216 1217 movq %r8, %r11 // p_argv -> %r111218 1219#if KMP_MIC1220 cmpq $4, %rax // argc >= 4?1221 jns KMP_LABEL(kmp_4) // jump to movq1222 jmp KMP_LABEL(kmp_4_exit) // jump ahead1223KMP_LABEL(kmp_4):1224 movq 24(%r11), %r9 // p_argv[3] -> %r9 (store 6th parm to pkfn)1225KMP_LABEL(kmp_4_exit):1226 1227 cmpq $3, %rax // argc >= 3?1228 jns KMP_LABEL(kmp_3) // jump to movq1229 jmp KMP_LABEL(kmp_3_exit) // jump ahead1230KMP_LABEL(kmp_3):1231 movq 16(%r11), %r8 // p_argv[2] -> %r8 (store 5th parm to pkfn)1232KMP_LABEL(kmp_3_exit):1233 1234 cmpq $2, %rax // argc >= 2?1235 jns KMP_LABEL(kmp_2) // jump to movq1236 jmp KMP_LABEL(kmp_2_exit) // jump ahead1237KMP_LABEL(kmp_2):1238 movq 8(%r11), %rcx // p_argv[1] -> %rcx (store 4th parm to pkfn)1239KMP_LABEL(kmp_2_exit):1240 1241 cmpq $1, %rax // argc >= 1?1242 jns KMP_LABEL(kmp_1) // jump to movq1243 jmp KMP_LABEL(kmp_1_exit) // jump ahead1244KMP_LABEL(kmp_1):1245 movq (%r11), %rdx // p_argv[0] -> %rdx (store 3rd parm to pkfn)1246KMP_LABEL(kmp_1_exit):1247#else1248 cmpq $4, %rax // argc >= 4?1249 cmovnsq 24(%r11), %r9 // p_argv[3] -> %r9 (store 6th parm to pkfn)1250 1251 cmpq $3, %rax // argc >= 3?1252 cmovnsq 16(%r11), %r8 // p_argv[2] -> %r8 (store 5th parm to pkfn)1253 1254 cmpq $2, %rax // argc >= 2?1255 cmovnsq 8(%r11), %rcx // p_argv[1] -> %rcx (store 4th parm to pkfn)1256 1257 cmpq $1, %rax // argc >= 1?1258 cmovnsq (%r11), %rdx // p_argv[0] -> %rdx (store 3rd parm to pkfn)1259#endif // KMP_MIC1260 1261KMP_LABEL(kmp_no_args):1262 call *%rbx // call (*pkfn)();1263 movq $1, %rax // move 1 into return register;1264 1265 movq -8(%rbp), %rbx // restore %rbx using %rbp since %rsp was modified1266 movq %rbp, %rsp // restore stack pointer1267 popq %rbp // restore frame pointer1268 KMP_CFI_DEF rsp,81269 ret1270 1271 DEBUG_INFO __kmp_invoke_microtask1272// -- End __kmp_invoke_microtask1273 1274// kmp_uint641275// __kmp_hardware_timestamp(void)1276 .text1277 PROC __kmp_hardware_timestamp1278 rdtsc1279 shlq $32, %rdx1280 orq %rdx, %rax1281 ret1282 1283 DEBUG_INFO __kmp_hardware_timestamp1284// -- End __kmp_hardware_timestamp1285 1286//------------------------------------------------------------------------1287// FUNCTION __kmp_bsr321288//1289// int1290// __kmp_bsr32( int );1291 .text1292 PROC __kmp_bsr321293 1294 bsr %edi,%eax1295 ret1296 1297 DEBUG_INFO __kmp_bsr321298 1299// -----------------------------------------------------------------------1300#endif /* KMP_ARCH_X86_64 */1301 1302// '1303#if KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_321304 1305//------------------------------------------------------------------------1306// int1307// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),1308// int gtid, int tid,1309// int argc, void *p_argv[]1310// #if OMPT_SUPPORT1311// ,1312// void **exit_frame_ptr1313// #endif1314// ) {1315// #if OMPT_SUPPORT1316// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);1317// #endif1318//1319// (*pkfn)( & gtid, & tid, argv[0], ... );1320//1321// // FIXME: This is done at call-site and can be removed here.1322// #if OMPT_SUPPORT1323// *exit_frame_ptr = 0;1324// #endif1325//1326// return 1;1327// }1328//1329// parameters:1330// x0: pkfn1331// w1: gtid1332// w2: tid1333// w3: argc1334// x4: p_argv1335// x5: &exit_frame1336//1337// locals:1338// __gtid: gtid parm pushed on stack so can pass >id to pkfn1339// __tid: tid parm pushed on stack so can pass &tid to pkfn1340//1341// reg temps:1342// x8: used to hold pkfn address1343// w9: used as temporary for number of pkfn parms1344// x10: used to traverse p_argv array1345// x11: used as temporary for stack placement calculation1346// x12: used as temporary for stack parameters1347// x19: used to preserve exit_frame_ptr, callee-save1348//1349// return: w0 (always 1/TRUE)1350//1351 1352__gtid = 41353__tid = 81354 1355// -- Begin __kmp_invoke_microtask1356// mark_begin;1357 .text1358 PROC __kmp_invoke_microtask1359 PACBTI_C1360 1361# if OMPT_SUPPORT1362 stp x19, x20, [sp, #-16]!1363# endif1364 stp x29, x30, [sp, #-16]!1365 mov x29, sp1366 1367 orr w9, wzr, #11368 add w9, w9, w3, lsr #11369 sub sp, sp, w9, uxtw #41370 mov x11, sp1371 1372 mov x8, x01373 str w1, [x29, #-__gtid]1374 str w2, [x29, #-__tid]1375 mov w9, w31376 mov x10, x41377# if OMPT_SUPPORT1378 mov x19, x51379 str x29, [x19]1380# endif1381 1382 sub x0, x29, #__gtid1383 sub x1, x29, #__tid1384 1385 cbz w9, KMP_LABEL(kmp_1)1386 ldr x2, [x10]1387 1388 sub w9, w9, #11389 cbz w9, KMP_LABEL(kmp_1)1390 ldr x3, [x10, #8]!1391 1392 sub w9, w9, #11393 cbz w9, KMP_LABEL(kmp_1)1394 ldr x4, [x10, #8]!1395 1396 sub w9, w9, #11397 cbz w9, KMP_LABEL(kmp_1)1398 ldr x5, [x10, #8]!1399 1400 sub w9, w9, #11401 cbz w9, KMP_LABEL(kmp_1)1402 ldr x6, [x10, #8]!1403 1404 sub w9, w9, #11405 cbz w9, KMP_LABEL(kmp_1)1406 ldr x7, [x10, #8]!1407 1408KMP_LABEL(kmp_0):1409 sub w9, w9, #11410 cbz w9, KMP_LABEL(kmp_1)1411 ldr x12, [x10, #8]!1412 str x12, [x11], #81413 b KMP_LABEL(kmp_0)1414KMP_LABEL(kmp_1):1415 blr x81416 orr w0, wzr, #11417 mov sp, x291418 ldp x29, x30, [sp], #161419# if OMPT_SUPPORT1420 str xzr, [x19]1421 ldp x19, x20, [sp], #161422# endif1423 PACBTI_RET1424 ret1425 1426 DEBUG_INFO __kmp_invoke_microtask1427// -- End __kmp_invoke_microtask1428 1429#endif /* KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32 */1430 1431#if KMP_ARCH_ARM1432 1433//------------------------------------------------------------------------1434// int1435// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),1436// int gtid, int tid,1437// int argc, void *p_argv[]1438// #if OMPT_SUPPORT1439// ,1440// void **exit_frame_ptr1441// #endif1442// ) {1443// #if OMPT_SUPPORT1444// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);1445// #endif1446//1447// (*pkfn)( & gtid, & tid, argv[0], ... );1448//1449// // FIXME: This is done at call-site and can be removed here.1450// #if OMPT_SUPPORT1451// *exit_frame_ptr = 0;1452// #endif1453//1454// return 1;1455// }1456//1457// parameters:1458// r0: pkfn1459// r1: gtid1460// r2: tid1461// r3: argc1462// r4(stack): p_argv1463// r5(stack): &exit_frame1464//1465// locals:1466// __gtid: gtid parm pushed on stack so can pass >id to pkfn1467// __tid: tid parm pushed on stack so can pass &tid to pkfn1468//1469// reg temps:1470// r4: used to hold pkfn address1471// r5: used as temporary for number of pkfn parms1472// r6: used to traverse p_argv array1473// r7: frame pointer (in some configurations)1474// r8: used as temporary for stack placement calculation1475// and as pointer to base of callee saved area1476// r9: used as temporary for stack parameters1477// r10: used to preserve exit_frame_ptr, callee-save1478// r11: frame pointer (in some configurations)1479//1480// return: r0 (always 1/TRUE)1481//1482 1483__gtid = 41484__tid = 81485 1486// -- Begin __kmp_invoke_microtask1487// mark_begin;1488 .text1489 PROC __kmp_invoke_microtask1490 1491 // Pushing one extra register (r3) to keep the stack aligned1492 // for when we call pkfn below1493 push {r3-r11,lr}1494 // Load p_argv and &exit_frame1495 ldr r4, [sp, #10*4]1496# if OMPT_SUPPORT1497 ldr r5, [sp, #11*4]1498# endif1499 1500# if KMP_OS_DARWIN || (defined(__thumb__) && !KMP_OS_WINDOWS)1501# define FP r71502# define FPOFF 4*41503#else1504# define FP r111505# define FPOFF 8*41506#endif1507 add FP, sp, #FPOFF1508# if OMPT_SUPPORT1509 mov r10, r51510 str FP, [r10]1511# endif1512 mov r8, sp1513 1514 // Calculate how much stack to allocate, in increments of 8 bytes.1515 // We strictly need 4*(argc-2) bytes (2 arguments are passed in1516 // registers) but allocate 4*argc for simplicity (to avoid needing1517 // to handle the argc<2 cases). We align the number of bytes1518 // allocated to 8 bytes, to keep the stack aligned. (Since we1519 // already allocate more than enough, it's ok to round down1520 // instead of up for the alignment.) We allocate another extra1521 // 8 bytes for gtid and tid.1522 mov r5, #11523 add r5, r5, r3, lsr #11524 sub sp, sp, r5, lsl #31525 1526 str r1, [r8, #-__gtid]1527 str r2, [r8, #-__tid]1528 mov r5, r31529 mov r6, r41530 mov r4, r01531 1532 // Prepare the first 2 parameters to pkfn - pointers to gtid and tid1533 // in our stack frame.1534 sub r0, r8, #__gtid1535 sub r1, r8, #__tid1536 1537 mov r8, sp1538 1539 // Load p_argv[0] and p_argv[1] into r2 and r3, if argc >= 1/21540 cmp r5, #01541 beq KMP_LABEL(kmp_1)1542 ldr r2, [r6]1543 1544 subs r5, r5, #11545 beq KMP_LABEL(kmp_1)1546 ldr r3, [r6, #4]!1547 1548 // Loop, loading the rest of p_argv and writing the elements on the1549 // stack.1550KMP_LABEL(kmp_0):1551 subs r5, r5, #11552 beq KMP_LABEL(kmp_1)1553 ldr r12, [r6, #4]!1554 str r12, [r8], #41555 b KMP_LABEL(kmp_0)1556KMP_LABEL(kmp_1):1557 blx r41558 mov r0, #11559 1560 sub r4, FP, #FPOFF1561 mov sp, r41562# undef FP1563# undef FPOFF1564 1565# if OMPT_SUPPORT1566 mov r1, #01567 str r1, [r10]1568# endif1569 pop {r3-r11,pc}1570 1571 DEBUG_INFO __kmp_invoke_microtask1572// -- End __kmp_invoke_microtask1573 1574#endif /* KMP_ARCH_ARM */1575 1576#if KMP_ARCH_PPC641577 1578//------------------------------------------------------------------------1579// int1580// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),1581// int gtid, int tid,1582// int argc, void *p_argv[]1583// #if OMPT_SUPPORT1584// ,1585// void **exit_frame_ptr1586// #endif1587// ) {1588// #if OMPT_SUPPORT1589// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);1590// #endif1591//1592// (*pkfn)( & gtid, & tid, argv[0], ... );1593//1594// // FIXME: This is done at call-site and can be removed here.1595// #if OMPT_SUPPORT1596// *exit_frame_ptr = 0;1597// #endif1598//1599// return 1;1600// }1601//1602// parameters:1603// r3: pkfn1604// r4: gtid1605// r5: tid1606// r6: argc1607// r7: p_argv1608// r8: &exit_frame1609//1610// return: r3 (always 1/TRUE)1611//1612 .text1613# if KMP_ARCH_PPC64_ELFv21614 .abiversion 21615# endif1616 .globl __kmp_invoke_microtask1617 1618# if KMP_ARCH_PPC64_ELFv21619 .p2align 41620# else1621 .p2align 21622# endif1623 1624 .type __kmp_invoke_microtask,@function1625 1626# if KMP_ARCH_PPC64_ELFv21627__kmp_invoke_microtask:1628.Lfunc_begin0:1629.Lfunc_gep0:1630 addis 2, 12, .TOC.-.Lfunc_gep0@ha1631 addi 2, 2, .TOC.-.Lfunc_gep0@l1632.Lfunc_lep0:1633 .localentry __kmp_invoke_microtask, .Lfunc_lep0-.Lfunc_gep01634# else1635 .section .opd,"aw",@progbits1636__kmp_invoke_microtask:1637 .p2align 31638 .quad .Lfunc_begin01639 .quad .TOC.@tocbase1640 .quad 01641 .text1642.Lfunc_begin0:1643# endif1644 1645// -- Begin __kmp_invoke_microtask1646// mark_begin;1647 1648// We need to allocate a stack frame large enough to hold all of the parameters1649// on the stack for the microtask plus what this function needs. That's 481650// bytes under the ELFv1 ABI (32 bytes under ELFv2), plus 8*(2 + argc) for the1651// parameters to the microtask, plus 8 bytes to store the values of r4 and r5,1652// and 8 bytes to store r31. With OMP-T support, we need an additional 8 bytes1653// to save r30 to hold a copy of r8.1654 1655 .cfi_startproc1656 mflr 01657 std 31, -8(1)1658 std 0, 16(1)1659 1660// This is unusual because normally we'd set r31 equal to r1 after the stack1661// frame is established. In this case, however, we need to dynamically compute1662// the stack frame size, and so we keep a direct copy of r1 to access our1663// register save areas and restore the r1 value before returning.1664 mr 31, 11665 .cfi_def_cfa_register r311666 .cfi_offset r31, -81667 .cfi_offset lr, 161668 1669// Compute the size necessary for the local stack frame.1670# if KMP_ARCH_PPC64_ELFv21671 li 12, 721672# else1673 li 12, 881674# endif1675 sldi 0, 6, 31676 add 12, 0, 121677 neg 12, 121678 1679// We need to make sure that the stack frame stays aligned (to 16 bytes).1680 li 0, -161681 and 12, 0, 121682 1683// Establish the local stack frame.1684 stdux 1, 1, 121685 1686# if OMPT_SUPPORT1687 .cfi_offset r30, -161688 std 30, -16(31)1689 std 1, 0(8)1690 mr 30, 81691# endif1692 1693// Store gtid and tid to the stack because they're passed by reference to the microtask.1694 stw 4, -20(31)1695 stw 5, -24(31)1696 1697 mr 12, 61698 mr 4, 71699 1700 cmpwi 0, 12, 11701 blt 0, .Lcall1702 1703 ld 5, 0(4)1704 1705 cmpwi 0, 12, 21706 blt 0, .Lcall1707 1708 ld 6, 8(4)1709 1710 cmpwi 0, 12, 31711 blt 0, .Lcall1712 1713 ld 7, 16(4)1714 1715 cmpwi 0, 12, 41716 blt 0, .Lcall1717 1718 ld 8, 24(4)1719 1720 cmpwi 0, 12, 51721 blt 0, .Lcall1722 1723 ld 9, 32(4)1724 1725 cmpwi 0, 12, 61726 blt 0, .Lcall1727 1728 ld 10, 40(4)1729 1730 cmpwi 0, 12, 71731 blt 0, .Lcall1732 1733// There are more than 6 microtask parameters, so we need to store the1734// remainder to the stack.1735 addi 12, 12, -61736 mtctr 121737 1738// These are set to 8 bytes before the first desired store address (we're using1739// pre-increment loads and stores in the loop below). The parameter save area1740// for the microtask begins 48 + 8*8 == 112 bytes above r1 for ELFv1 and1741// 32 + 8*8 == 96 bytes above r1 for ELFv2.1742 addi 4, 4, 401743# if KMP_ARCH_PPC64_ELFv21744 addi 12, 1, 881745# else1746 addi 12, 1, 1041747# endif1748 1749.Lnext:1750 ldu 0, 8(4)1751 stdu 0, 8(12)1752 bdnz .Lnext1753 1754.Lcall:1755# if KMP_ARCH_PPC64_ELFv21756 std 2, 24(1)1757 mr 12, 31758#else1759 std 2, 40(1)1760// For ELFv1, we need to load the actual function address from the function descriptor.1761 ld 12, 0(3)1762 ld 2, 8(3)1763 ld 11, 16(3)1764#endif1765 1766 addi 3, 31, -201767 addi 4, 31, -241768 1769 mtctr 121770 bctrl1771# if KMP_ARCH_PPC64_ELFv21772 ld 2, 24(1)1773# else1774 ld 2, 40(1)1775# endif1776 1777# if OMPT_SUPPORT1778 li 3, 01779 std 3, 0(30)1780# endif1781 1782 li 3, 11783 1784# if OMPT_SUPPORT1785 ld 30, -16(31)1786# endif1787 1788 mr 1, 311789 ld 0, 16(1)1790 ld 31, -8(1)1791 mtlr 01792 blr1793 1794 .long 01795 .quad 01796.Lfunc_end0:1797 .size __kmp_invoke_microtask, .Lfunc_end0-.Lfunc_begin01798 .cfi_endproc1799 1800// -- End __kmp_invoke_microtask1801 1802#endif /* KMP_ARCH_PPC64 */1803 1804#if KMP_ARCH_RISCV641805 1806//------------------------------------------------------------------------1807//1808// typedef void (*microtask_t)(int *gtid, int *tid, ...);1809//1810// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,1811// void *p_argv[]1812// #if OMPT_SUPPORT1813// ,1814// void **exit_frame_ptr1815// #endif1816// ) {1817// #if OMPT_SUPPORT1818// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);1819// #endif1820//1821// (*pkfn)(>id, &tid, argv[0], ...);1822//1823// return 1;1824// }1825//1826// Parameters:1827// a0: pkfn1828// a1: gtid1829// a2: tid1830// a3: argc1831// a4: p_argv1832// a5: exit_frame_ptr1833//1834// Locals:1835// __gtid: gtid param pushed on stack so can pass >id to pkfn1836// __tid: tid param pushed on stack so can pass &tid to pkfn1837//1838// Temp. registers:1839//1840// t0: used to calculate the dynamic stack size / used to hold pkfn address1841// t1: used as temporary for stack placement calculation1842// t2: used as temporary for stack arguments1843// t3: used as temporary for number of remaining pkfn parms1844// t4: used to traverse p_argv array1845//1846// return: a0 (always 1/TRUE)1847//1848 1849__gtid = -201850__tid = -241851 1852// -- Begin __kmp_invoke_microtask1853// mark_begin;1854 .text1855 .globl __kmp_invoke_microtask1856 .p2align 11857 .type __kmp_invoke_microtask,@function1858__kmp_invoke_microtask:1859 .cfi_startproc1860 1861 // First, save ra and fp1862 addi sp, sp, -161863 sd ra, 8(sp)1864 sd fp, 0(sp)1865 addi fp, sp, 161866 .cfi_def_cfa fp, 01867 .cfi_offset ra, -81868 .cfi_offset fp, -161869 1870 // Compute the dynamic stack size:1871 //1872 // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by1873 // reference1874 // - We need 8 bytes for each argument that cannot be passed to the 'pkfn'1875 // function by register. Given that we have 8 of such registers (a[0-7])1876 // and two + 'argc' arguments (consider >id and &tid), we need to1877 // reserve max(0, argc - 6)*8 extra bytes1878 //1879 // The total number of bytes is then max(0, argc - 6)*8 + 81880 1881 // Compute max(0, argc - 6) using the following bithack:1882 // max(0, x) = x - (x & (x >> 31)), where x := argc - 61883 // Source: http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax1884 addi t0, a3, -61885 srai t1, t0, 311886 and t1, t0, t11887 sub t0, t0, t11888 1889 addi t0, t0, 11890 1891 slli t0, t0, 31892 sub sp, sp, t01893 1894 // Align the stack to 16 bytes1895 andi sp, sp, -161896 1897 mv t0, a01898 mv t3, a31899 mv t4, a41900 1901#if OMPT_SUPPORT1902 // Save frame pointer into exit_frame1903 sd fp, 0(a5)1904#endif1905 1906 // Prepare arguments for the pkfn function (first 8 using a0-a7 registers)1907 1908 sw a1, __gtid(fp)1909 sw a2, __tid(fp)1910 1911 addi a0, fp, __gtid1912 addi a1, fp, __tid1913 1914 beqz t3, .L_kmp_31915 ld a2, 0(t4)1916 1917 addi t3, t3, -11918 beqz t3, .L_kmp_31919 ld a3, 8(t4)1920 1921 addi t3, t3, -11922 beqz t3, .L_kmp_31923 ld a4, 16(t4)1924 1925 addi t3, t3, -11926 beqz t3, .L_kmp_31927 ld a5, 24(t4)1928 1929 addi t3, t3, -11930 beqz t3, .L_kmp_31931 ld a6, 32(t4)1932 1933 addi t3, t3, -11934 beqz t3, .L_kmp_31935 ld a7, 40(t4)1936 1937 // Prepare any additional argument passed through the stack1938 addi t4, t4, 481939 mv t1, sp1940 j .L_kmp_21941.L_kmp_1:1942 ld t2, 0(t4)1943 sd t2, 0(t1)1944 addi t4, t4, 81945 addi t1, t1, 81946.L_kmp_2:1947 addi t3, t3, -11948 bnez t3, .L_kmp_11949 1950.L_kmp_3:1951 // Call pkfn function1952 jalr t01953 1954 // Restore stack and return1955 1956 addi a0, zero, 11957 1958 addi sp, fp, -161959 ld fp, 0(sp)1960 ld ra, 8(sp)1961 addi sp, sp, 161962 ret1963.Lfunc_end0:1964 .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask1965 .cfi_endproc1966 1967// -- End __kmp_invoke_microtask1968 1969#endif /* KMP_ARCH_RISCV64 */1970 1971#if KMP_ARCH_LOONGARCH641972 1973//------------------------------------------------------------------------1974//1975// typedef void (*microtask_t)(int *gtid, int *tid, ...);1976//1977// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,1978// void *p_argv[]1979// #if OMPT_SUPPORT1980// ,1981// void **exit_frame_ptr1982// #endif1983// ) {1984// #if OMPT_SUPPORT1985// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);1986// #endif1987//1988// (*pkfn)(>id, &tid, argv[0], ...);1989//1990// return 1;1991// }1992//1993// Parameters:1994// a0: pkfn1995// a1: gtid1996// a2: tid1997// a3: argc1998// a4: p_argv1999// a5: exit_frame_ptr2000//2001// Locals:2002// __gtid: gtid param pushed on stack so can pass >id to pkfn2003// __tid: tid param pushed on stack so can pass &tid to pkfn2004//2005// Temp registers:2006//2007// t0: used to calculate the dynamic stack size / used to hold pkfn address2008// t1: used as temporary for stack placement calculation2009// t2: used as temporary for stack arguments2010// t3: used as temporary for number of remaining pkfn parms2011// t4: used to traverse p_argv array2012//2013// return: a0 (always 1/TRUE)2014//2015 2016// -- Begin __kmp_invoke_microtask2017// mark_begin;2018 .text2019 .globl __kmp_invoke_microtask2020 .p2align 22021 .type __kmp_invoke_microtask,@function2022__kmp_invoke_microtask:2023 .cfi_startproc2024 2025 // First, save ra and fp2026 addi.d $sp, $sp, -162027 st.d $ra, $sp, 82028 st.d $fp, $sp, 02029 addi.d $fp, $sp, 162030 .cfi_def_cfa 22, 02031 .cfi_offset 1, -82032 .cfi_offset 22, -162033 2034 // Compute the dynamic stack size:2035 //2036 // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by2037 // reference2038 // - We need 8 bytes for each argument that cannot be passed to the 'pkfn'2039 // function by register. Given that we have 8 of such registers (a[0-7])2040 // and two + 'argc' arguments (consider >id and &tid), we need to2041 // reserve max(0, argc - 6)*8 extra bytes2042 //2043 // The total number of bytes is then max(0, argc - 6)*8 + 82044 2045 addi.d $t0, $a3, -62046 slt $t1, $t0, $zero2047 masknez $t0, $t0, $t12048 addi.d $t0, $t0, 12049 slli.d $t0, $t0, 32050 sub.d $sp, $sp, $t02051 2052 // Align the stack to 16 bytes2053 bstrins.d $sp, $zero, 3, 02054 2055 move $t0, $a02056 move $t3, $a32057 move $t4, $a42058 2059#if OMPT_SUPPORT2060 // Save frame pointer into exit_frame2061 st.d $fp, $a5, 02062#endif2063 2064 // Prepare arguments for the pkfn function (first 8 using a0-a7 registers)2065 2066 st.w $a1, $fp, -202067 st.w $a2, $fp, -242068 2069 addi.d $a0, $fp, -202070 addi.d $a1, $fp, -242071 2072 beqz $t3, .L_kmp_32073 ld.d $a2, $t4, 02074 2075 addi.d $t3, $t3, -12076 beqz $t3, .L_kmp_32077 ld.d $a3, $t4, 82078 2079 addi.d $t3, $t3, -12080 beqz $t3, .L_kmp_32081 ld.d $a4, $t4, 162082 2083 addi.d $t3, $t3, -12084 beqz $t3, .L_kmp_32085 ld.d $a5, $t4, 242086 2087 addi.d $t3, $t3, -12088 beqz $t3, .L_kmp_32089 ld.d $a6, $t4, 322090 2091 addi.d $t3, $t3, -12092 beqz $t3, .L_kmp_32093 ld.d $a7, $t4, 402094 2095 // Prepare any additional argument passed through the stack2096 addi.d $t4, $t4, 482097 move $t1, $sp2098 b .L_kmp_22099.L_kmp_1:2100 ld.d $t2, $t4, 02101 st.d $t2, $t1, 02102 addi.d $t4, $t4, 82103 addi.d $t1, $t1, 82104.L_kmp_2:2105 addi.d $t3, $t3, -12106 bnez $t3, .L_kmp_12107 2108.L_kmp_3:2109 // Call pkfn function2110 jirl $ra, $t0, 02111 2112 // Restore stack and return2113 2114 addi.d $a0, $zero, 12115 2116 addi.d $sp, $fp, -162117 ld.d $fp, $sp, 02118 ld.d $ra, $sp, 82119 addi.d $sp, $sp, 162120 jr $ra2121.Lfunc_end0:2122 .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask2123 .cfi_endproc2124 2125// -- End __kmp_invoke_microtask2126 2127#endif /* KMP_ARCH_LOONGARCH64 */2128 2129#if KMP_ARCH_VE2130 2131//------------------------------------------------------------------------2132//2133// typedef void (*microtask_t)(int *gtid, int *tid, ...);2134//2135// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,2136// void *p_argv[]2137// #if OMPT_SUPPORT2138// ,2139// void **exit_frame_ptr2140// #endif2141// ) {2142// #if OMPT_SUPPORT2143// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);2144// #endif2145//2146// (*pkfn)(>id, &tid, argv[0], ...);2147//2148// return 1;2149// }2150//2151// Parameters:2152// s0: pkfn2153// s1: gtid2154// s2: tid2155// s3: argc2156// s4: p_argv2157// s5: exit_frame_ptr2158//2159// Locals:2160// __gtid: gtid param pushed on stack so can pass >id to pkfn2161// __tid: tid param pushed on stack so can pass &tid to pkfn2162//2163// Temp. registers:2164//2165// s34: used to calculate the dynamic stack size2166// s35: used as temporary for stack placement calculation2167// s36: used as temporary for stack arguments2168// s37: used as temporary for number of remaining pkfn parms2169// s38: used to traverse p_argv array2170//2171// return: s0 (always 1/TRUE)2172//2173 2174__gtid = -42175__tid = -82176 2177// -- Begin __kmp_invoke_microtask2178// mark_begin;2179 .text2180 .globl __kmp_invoke_microtask2181 // A function requires 8 bytes align.2182 .p2align 32183 .type __kmp_invoke_microtask,@function2184__kmp_invoke_microtask:2185 .cfi_startproc2186 2187 // First, save fp and lr. VE stores them at caller stack frame.2188 st %fp, 0(, %sp)2189 st %lr, 8(, %sp)2190 or %fp, 0, %sp2191 .cfi_def_cfa %fp, 02192 .cfi_offset %lr, 82193 .cfi_offset %fp, 02194 2195 // Compute the dynamic stack size:2196 //2197 // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them2198 // by reference2199 // - We need 8 bytes for whole arguments. We have two + 'argc'2200 // arguments (condider >id and &tid). We need to reserve2201 // (argc + 2) * 8 bytes.2202 // - We need 176 bytes for RSA and others2203 //2204 // The total number of bytes is then (argc + 2) * 8 + 8 + 176.2205 //2206 // |------------------------------|2207 // | return address of callee | 8(%fp)2208 // |------------------------------|2209 // | frame pointer of callee | 0(%fp)2210 // |------------------------------| <------------------ %fp2211 // | __tid / __gtid | -8(%fp) / -4(%fp)2212 // |------------------------------|2213 // | argc+2 for arguments | 176(%sp)2214 // |------------------------------|2215 // | RSA |2216 // |------------------------------|2217 // | return address |2218 // |------------------------------|2219 // | frame pointer |2220 // |------------------------------| <------------------ %sp2221 2222 adds.w.sx %s34, 2, %s32223 sll %s34, %s34, 32224 lea %s34, 184(, %s34)2225 subs.l %sp, %sp, %s342226 2227 // Align the stack to 16 bytes.2228 and %sp, -16, %sp2229 2230 // Save pkfn.2231 or %s12, 0, %s02232 2233 // Call host to allocate stack if it is necessary.2234 brge.l %sp, %sl, .L_kmp_pass2235 ld %s61, 24(, %tp)2236 lea %s63, 0x13b2237 shm.l %s63, 0(%s61)2238 shm.l %sl, 8(%s61)2239 shm.l %sp, 16(%s61)2240 monc2241 2242.L_kmp_pass:2243 lea %s35, 176(, %sp)2244 adds.w.sx %s37, 0, %s32245 or %s38, 0, %s42246 2247#if OMPT_SUPPORT2248 // Save frame pointer into exit_frame.2249 st %fp, 0(%s5)2250#endif2251 2252 // Prepare arguments for the pkfn function (first 8 using s0-s72253 // registers, but need to store stack also because of varargs).2254 2255 stl %s1, __gtid(%fp)2256 stl %s2, __tid(%fp)2257 2258 adds.l %s0, __gtid, %fp2259 st %s0, 0(, %s35)2260 adds.l %s1, __tid, %fp2261 st %s1, 8(, %s35)2262 2263 breq.l 0, %s37, .L_kmp_call2264 ld %s2, 0(, %s38)2265 st %s2, 16(, %s35)2266 2267 breq.l 1, %s37, .L_kmp_call2268 ld %s3, 8(, %s38)2269 st %s3, 24(, %s35)2270 2271 breq.l 2, %s37, .L_kmp_call2272 ld %s4, 16(, %s38)2273 st %s4, 32(, %s35)2274 2275 breq.l 3, %s37, .L_kmp_call2276 ld %s5, 24(, %s38)2277 st %s5, 40(, %s35)2278 2279 breq.l 4, %s37, .L_kmp_call2280 ld %s6, 32(, %s38)2281 st %s6, 48(, %s35)2282 2283 breq.l 5, %s37, .L_kmp_call2284 ld %s7, 40(, %s38)2285 st %s7, 56(, %s35)2286 2287 breq.l 6, %s37, .L_kmp_call2288 2289 // Prepare any additional argument passed through the stack.2290 adds.l %s37, -6, %s372291 lea %s38, 48(, %s38)2292 lea %s35, 64(, %s35)2293.L_kmp_loop:2294 ld %s36, 0(, %s38)2295 st %s36, 0(, %s35)2296 adds.l %s37, -1, %s372297 adds.l %s38, 8, %s382298 adds.l %s35, 8, %s352299 brne.l 0, %s37, .L_kmp_loop2300 2301.L_kmp_call:2302 // Call pkfn function.2303 bsic %lr, (, %s12)2304 2305 // Return value.2306 lea %s0, 12307 2308 // Restore stack and return.2309 or %sp, 0, %fp2310 ld %lr, 8(, %sp)2311 ld %fp, 0(, %sp)2312 b.l.t (, %lr)2313.Lfunc_end0:2314 .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask2315 .cfi_endproc2316 2317// -- End __kmp_invoke_microtask2318 2319#endif /* KMP_ARCH_VE */2320 2321#if KMP_ARCH_S390X2322 2323//------------------------------------------------------------------------2324//2325// typedef void (*microtask_t)(int *gtid, int *tid, ...);2326//2327// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,2328// void *p_argv[]2329// #if OMPT_SUPPORT2330// ,2331// void **exit_frame_ptr2332// #endif2333// ) {2334// #if OMPT_SUPPORT2335// *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);2336// #endif2337//2338// (*pkfn)(>id, &tid, argv[0], ...);2339//2340// return 1;2341// }2342//2343// Parameters:2344// r2: pkfn2345// r3: gtid2346// r4: tid2347// r5: argc2348// r6: p_argv2349// SP+160: exit_frame_ptr2350//2351// Locals:2352// __gtid: gtid param pushed on stack so can pass >id to pkfn2353// __tid: tid param pushed on stack so can pass &tid to pkfn2354//2355// Temp. registers:2356//2357// r0: used to fetch argv slots2358// r7: used as temporary for number of remaining pkfn parms2359// r8: argv2360// r9: pkfn2361// r10: stack size2362// r11: previous fp2363// r12: stack parameter area2364// r13: argv slot2365//2366// return: r2 (always 1/TRUE)2367//2368 2369// -- Begin __kmp_invoke_microtask2370// mark_begin;2371 .text2372 .globl __kmp_invoke_microtask2373 .p2align 12374 .type __kmp_invoke_microtask,@function2375__kmp_invoke_microtask:2376 .cfi_startproc2377 2378 stmg %r6,%r14,48(%r15)2379 .cfi_offset %r6, -1122380 .cfi_offset %r7, -1042381 .cfi_offset %r8, -962382 .cfi_offset %r9, -882383 .cfi_offset %r10, -802384 .cfi_offset %r11, -722385 .cfi_offset %r12, -642386 .cfi_offset %r13, -562387 .cfi_offset %r14, -482388 .cfi_offset %r15, -402389 lgr %r11,%r152390 .cfi_def_cfa %r11, 1602391 2392 // Compute the dynamic stack size:2393 //2394 // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by2395 // reference2396 // - We need 8 bytes for each argument that cannot be passed to the 'pkfn'2397 // function by register. Given that we have 5 of such registers (r[2-6])2398 // and two + 'argc' arguments (consider >id and &tid), we need to2399 // reserve max(0, argc - 3)*8 extra bytes2400 //2401 // The total number of bytes is then max(0, argc - 3)*8 + 82402 2403 lgr %r10,%r52404 aghi %r10,-22405 jnm 0f2406 lghi %r10,024070:2408 sllg %r10,%r10,32409 lgr %r12,%r102410 aghi %r10,1762411 sgr %r15,%r102412 agr %r12,%r152413 stg %r11,0(%r15)2414 2415 lgr %r9,%r2 // pkfn2416 2417#if OMPT_SUPPORT2418 // Save frame pointer into exit_frame2419 lg %r8,160(%r11)2420 stg %r11,0(%r8)2421#endif2422 2423 // Prepare arguments for the pkfn function (first 5 using r2-r6 registers)2424 2425 stg %r3,160(%r12)2426 la %r2,164(%r12) // gid2427 stg %r4,168(%r12) 2428 la %r3,172(%r12) // tid2429 lgr %r8,%r6 // argv2430 2431 // If argc > 02432 ltgr %r7,%r52433 jz 1f2434 2435 lg %r4,0(%r8) // argv[0]2436 aghi %r7,-12437 jz 1f2438 2439 // If argc > 12440 lg %r5,8(%r8) // argv[1]2441 aghi %r7,-12442 jz 1f2443 2444 // If argc > 22445 lg %r6,16(%r8) // argv[2]2446 aghi %r7,-12447 jz 1f2448 2449 lghi %r13,0 // Index [n]24502:2451 lg %r0,24(%r13,%r8) // argv[2+n]2452 stg %r0,160(%r13,%r15) // parm[2+n]2453 aghi %r13,8 // Next2454 aghi %r7,-12455 jnz 2b2456 24571:2458 basr %r14,%r9 // Call pkfn2459 2460 // Restore stack and return2461 2462 lgr %r15,%r112463 lmg %r6,%r14,48(%r15)2464 lghi %r2,12465 br %r142466.Lfunc_end0:2467 .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask2468 .cfi_endproc2469 2470// -- End __kmp_invoke_microtask2471 2472#endif /* KMP_ARCH_S390X */2473 2474#if KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC322475#ifndef KMP_PREFIX_UNDERSCORE2476# define KMP_PREFIX_UNDERSCORE(x) x2477#endif2478 .data2479 COMMON .gomp_critical_user_, 32, 32480 .data2481 .align 42482 .global KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr)2483KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr):2484 .4byte .gomp_critical_user_2485#ifdef __ELF__2486 .type KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),%object2487 .size KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),42488#endif2489#endif /* KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32 */2490 2491#if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 || \2492 KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE || \2493 KMP_ARCH_S390X || KMP_ARCH_SPARC642494#ifndef KMP_PREFIX_UNDERSCORE2495# define KMP_PREFIX_UNDERSCORE(x) x2496#endif2497 .data2498 COMMON .gomp_critical_user_, 32, 32499 .data2500 .align 82501 .global KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr)2502KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr):2503 .8byte .gomp_critical_user_2504#ifdef __ELF__2505 .type KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),%object2506 .size KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),82507#endif2508#endif /* KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 ||2509 KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE || 2510 KMP_ARCH_S390X || KMP_ARCH_SPARC64 */2511 2512#if KMP_OS_LINUX2513# if KMP_ARCH_ARM || KMP_ARCH_AARCH642514.section .note.GNU-stack,"",%progbits2515# elif !KMP_ARCH_WASM2516.section .note.GNU-stack,"",@progbits2517# endif2518#endif2519 2520#if (KMP_OS_LINUX || KMP_OS_OPENBSD) && \2521 (KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32)2522GNU_PROPERTY_BTI_PAC2523#endif2524