198 lines · c
1// RUN: %libomp-compile && env LIBOMP_NUM_HIDDEN_HELPER_THREADS=0 OMP_PROC_BIND=close OMP_PLACES=cores KMP_AFFINITY=verbose %libomp-run 8 1 42// REQUIRES: linux3//4// This test pthread_creates 8 root threads before any OpenMP5// runtime entry is ever called. We have all the root threads6// register with the runtime by calling omp_set_num_threads(),7// but this does not initialize their affinity. The fourth root thread8// then calls a parallel region and we make sure its affinity9// is correct. We also make sure all the other root threads are10// free-floating since they have not called into a parallel region.11 12#define _GNU_SOURCE13#include <stdio.h>14#include <stdlib.h>15#include <omp.h>16#include <pthread.h>17#include <unistd.h>18#include <assert.h>19#include <sys/types.h>20#include <sys/syscall.h>21#include "libomp_test_affinity.h"22 23volatile int entry_flag = 0;24volatile int flag = 0;25volatile int num_roots_arrived = 0;26int num_roots;27int spawner = 0;28pthread_mutex_t lock;29int register_workers = 0; // boolean30affinity_mask_t *full_mask;31 32int __kmpc_global_thread_num(void*);33 34int get_os_thread_id() {35 return (int)syscall(SYS_gettid);36}37 38int place_and_affinity_match() {39 int i, max_cpu;40 char buf[512];41 affinity_mask_t *mask = affinity_mask_alloc();42 int place = omp_get_place_num();43 int num_procs = omp_get_place_num_procs(place);44 int *ids = (int*)malloc(sizeof(int) * num_procs);45 omp_get_place_proc_ids(place, ids);46 get_thread_affinity(mask);47 affinity_mask_snprintf(buf, sizeof(buf), mask);48 printf("Primary Thread Place: %d\n", place);49 printf("Primary Thread mask: %s\n", buf);50 51 for (i = 0; i < num_procs; ++i) {52 int cpu = ids[i];53 if (!affinity_mask_isset(mask, cpu))54 return 0;55 }56 57 max_cpu = AFFINITY_MAX_CPUS;58 for (i = 0; i < max_cpu; ++i) {59 int cpu = i;60 if (affinity_mask_isset(mask, cpu)) {61 int j, found = 0;62 for (j = 0; j < num_procs; ++j) {63 if (ids[j] == cpu) {64 found = 1;65 break;66 }67 }68 if (!found)69 return 0;70 }71 }72 73 affinity_mask_free(mask);74 free(ids);75 return 1;76}77 78void* thread_func(void *arg) {79 int place, nplaces;80 int root_id = *((int*)arg);81 int pid = getpid();82 int tid = get_os_thread_id();83 84 // Order how the root threads are assigned a gtid in the runtime85 // i.e., root_id = gtid86 while (1) {87 int v = entry_flag;88 if (v == root_id)89 break;90 }91 92 // If main root thread93 if (root_id == spawner) {94 printf("Initial application thread (pid=%d, tid=%d, spawner=%d) reached thread_func (will call OpenMP)\n", pid, tid, spawner);95 omp_set_num_threads(4);96 #pragma omp atomic97 entry_flag++;98 // Wait for the workers to signal their arrival before #pragma omp parallel99 while (num_roots_arrived < num_roots - 1) {}100 // This will trigger the output for KMP_AFFINITY in this case101 #pragma omp parallel102 {103 int gtid = __kmpc_global_thread_num(NULL);104 #pragma omp single105 {106 printf("Exactly %d threads in the #pragma omp parallel\n",107 omp_get_num_threads());108 }109 #pragma omp critical110 {111 printf("OpenMP thread %d: gtid=%d\n", omp_get_thread_num(), gtid);112 }113 }114 flag = 1;115 if (!place_and_affinity_match()) {116 fprintf(stderr, "error: place and affinity mask do not match for primary thread\n");117 exit (EXIT_FAILURE);118 }119 120 } else { // If worker root thread121 // Worker root threads, register with OpenMP through omp_set_num_threads()122 // if designated to, signal their arrival and then wait for the main root123 // thread to signal them to exit.124 printf("New root pthread (pid=%d, tid=%d) reached thread_func\n", pid, tid);125 if (register_workers)126 omp_set_num_threads(4);127 #pragma omp atomic128 entry_flag++;129 130 pthread_mutex_lock(&lock);131 num_roots_arrived++;132 pthread_mutex_unlock(&lock);133 while (flag == 0) {}134 135 // Main check whether root threads' mask is equal to the136 // initial affinity mask137 affinity_mask_t *mask = affinity_mask_alloc();138 get_thread_affinity(mask);139 if (!affinity_mask_equal(mask, full_mask)) {140 char buf[1024];141 printf("root thread %d mask: ", root_id);142 affinity_mask_snprintf(buf, sizeof(buf), mask);143 printf("initial affinity mask: %s\n", buf);144 fprintf(stderr, "error: root thread %d affinity mask not equal"145 " to initial full mask\n", root_id);146 affinity_mask_free(mask);147 exit(EXIT_FAILURE);148 }149 affinity_mask_free(mask);150 }151 return NULL;152}153 154int main(int argc, char** argv) {155 int i;156 if (argc != 3 && argc != 4) {157 fprintf(stderr, "usage: %s <num_roots> <register_workers_bool> [<spawn_root_number>]\n", argv[0]);158 exit(EXIT_FAILURE);159 }160 161 // Initialize pthread mutex162 pthread_mutex_init(&lock, NULL);163 164 // Get initial full mask165 full_mask = affinity_mask_alloc();166 get_thread_affinity(full_mask);167 168 // Get the number of root pthreads to create and allocate resources for them169 num_roots = atoi(argv[1]);170 pthread_t *roots = (pthread_t*)malloc(sizeof(pthread_t) * num_roots);171 int *root_ids = (int*)malloc(sizeof(int) * num_roots);172 173 // Get the flag indicating whether to have root pthreads call omp_set_num_threads() or not174 register_workers = atoi(argv[2]);175 176 if (argc == 4)177 spawner = atoi(argv[3]);178 179 // Spawn worker root threads180 for (i = 1; i < num_roots; ++i) {181 *(root_ids + i) = i;182 pthread_create(roots + i, NULL, thread_func, root_ids + i);183 }184 // Have main root thread (root 0) go into thread_func185 *root_ids = 0;186 thread_func(root_ids);187 188 // Cleanup all resources189 for (i = 1; i < num_roots; ++i) {190 void *status;191 pthread_join(roots[i], &status);192 }193 free(roots);194 free(root_ids);195 pthread_mutex_destroy(&lock);196 return EXIT_SUCCESS;197}198