729 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -mcpu=sm_90 -mattr=+ptx87 -S < %s | FileCheck %s -check-prefixes=CHECK,SM903; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -mcpu=sm_100 -mattr=+ptx88 -S < %s | FileCheck %s -check-prefixes=CHECK,SM1004 5; 256 bit loads/stores are only currently supported for:6; - global7; - blackwell (sm_100)8; - ptx 8.89; - 32/64-bit types10 11; Currently, the LSV produces 256 bit loads/stores if the first three conditions12; are satisfied, as the backend will either upsize or split vectors13; of smaller elements in the Type Legalization stage.14 15; In this file, we test i8, i16, i32, i64, f32, f64.16; The other floating point types are omitted for simplicity.17; We also test the negative case for non-global i32.18 19define void @int8x32(ptr addrspace(1) %ptr) {20; SM90-LABEL: define void @int8x32(21; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {22; SM90-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr addrspace(1) [[PTR]], i64 023; SM90-NEXT: [[PTR10:%.*]] = getelementptr i8, ptr addrspace(1) [[PTR]], i64 1624; SM90-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr addrspace(1) [[PTR0]], align 3225; SM90-NEXT: [[L01:%.*]] = extractelement <16 x i8> [[TMP1]], i32 026; SM90-NEXT: [[L110:%.*]] = extractelement <16 x i8> [[TMP1]], i32 127; SM90-NEXT: [[L211:%.*]] = extractelement <16 x i8> [[TMP1]], i32 228; SM90-NEXT: [[L312:%.*]] = extractelement <16 x i8> [[TMP1]], i32 329; SM90-NEXT: [[L413:%.*]] = extractelement <16 x i8> [[TMP1]], i32 430; SM90-NEXT: [[L514:%.*]] = extractelement <16 x i8> [[TMP1]], i32 531; SM90-NEXT: [[L615:%.*]] = extractelement <16 x i8> [[TMP1]], i32 632; SM90-NEXT: [[L716:%.*]] = extractelement <16 x i8> [[TMP1]], i32 733; SM90-NEXT: [[L817:%.*]] = extractelement <16 x i8> [[TMP1]], i32 834; SM90-NEXT: [[L918:%.*]] = extractelement <16 x i8> [[TMP1]], i32 935; SM90-NEXT: [[LA19:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1036; SM90-NEXT: [[LB20:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1137; SM90-NEXT: [[LC21:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1238; SM90-NEXT: [[LD22:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1339; SM90-NEXT: [[LE23:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1440; SM90-NEXT: [[LF24:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1541; SM90-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr addrspace(1) [[PTR10]], align 1642; SM90-NEXT: [[L1025:%.*]] = extractelement <16 x i8> [[TMP2]], i32 043; SM90-NEXT: [[L1126:%.*]] = extractelement <16 x i8> [[TMP2]], i32 144; SM90-NEXT: [[L1227:%.*]] = extractelement <16 x i8> [[TMP2]], i32 245; SM90-NEXT: [[L1328:%.*]] = extractelement <16 x i8> [[TMP2]], i32 346; SM90-NEXT: [[L1429:%.*]] = extractelement <16 x i8> [[TMP2]], i32 447; SM90-NEXT: [[L1530:%.*]] = extractelement <16 x i8> [[TMP2]], i32 548; SM90-NEXT: [[L1631:%.*]] = extractelement <16 x i8> [[TMP2]], i32 649; SM90-NEXT: [[L1732:%.*]] = extractelement <16 x i8> [[TMP2]], i32 750; SM90-NEXT: [[L1833:%.*]] = extractelement <16 x i8> [[TMP2]], i32 851; SM90-NEXT: [[L1934:%.*]] = extractelement <16 x i8> [[TMP2]], i32 952; SM90-NEXT: [[L1A35:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1053; SM90-NEXT: [[L1B36:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1154; SM90-NEXT: [[L1C37:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1255; SM90-NEXT: [[L1D38:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1356; SM90-NEXT: [[L1E39:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1457; SM90-NEXT: [[L1F40:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1558; SM90-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> poison, i8 [[LB20]], i32 059; SM90-NEXT: [[TMP4:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[LA19]], i32 160; SM90-NEXT: [[TMP5:%.*]] = insertelement <16 x i8> [[TMP4]], i8 [[L918]], i32 261; SM90-NEXT: [[TMP6:%.*]] = insertelement <16 x i8> [[TMP5]], i8 [[L817]], i32 362; SM90-NEXT: [[TMP7:%.*]] = insertelement <16 x i8> [[TMP6]], i8 [[L716]], i32 463; SM90-NEXT: [[TMP8:%.*]] = insertelement <16 x i8> [[TMP7]], i8 [[L615]], i32 564; SM90-NEXT: [[TMP9:%.*]] = insertelement <16 x i8> [[TMP8]], i8 [[L514]], i32 665; SM90-NEXT: [[TMP10:%.*]] = insertelement <16 x i8> [[TMP9]], i8 [[L413]], i32 766; SM90-NEXT: [[TMP11:%.*]] = insertelement <16 x i8> [[TMP10]], i8 [[L312]], i32 867; SM90-NEXT: [[TMP12:%.*]] = insertelement <16 x i8> [[TMP11]], i8 [[L211]], i32 968; SM90-NEXT: [[TMP13:%.*]] = insertelement <16 x i8> [[TMP12]], i8 [[L110]], i32 1069; SM90-NEXT: [[TMP14:%.*]] = insertelement <16 x i8> [[TMP13]], i8 [[L01]], i32 1170; SM90-NEXT: [[TMP15:%.*]] = insertelement <16 x i8> [[TMP14]], i8 [[LF24]], i32 1271; SM90-NEXT: [[TMP16:%.*]] = insertelement <16 x i8> [[TMP15]], i8 [[LE23]], i32 1372; SM90-NEXT: [[TMP17:%.*]] = insertelement <16 x i8> [[TMP16]], i8 [[LD22]], i32 1473; SM90-NEXT: [[TMP18:%.*]] = insertelement <16 x i8> [[TMP17]], i8 [[LC21]], i32 1574; SM90-NEXT: store <16 x i8> [[TMP18]], ptr addrspace(1) [[PTR0]], align 3275; SM90-NEXT: [[TMP19:%.*]] = insertelement <16 x i8> poison, i8 [[L1B36]], i32 076; SM90-NEXT: [[TMP20:%.*]] = insertelement <16 x i8> [[TMP19]], i8 [[L1A35]], i32 177; SM90-NEXT: [[TMP21:%.*]] = insertelement <16 x i8> [[TMP20]], i8 [[L1934]], i32 278; SM90-NEXT: [[TMP22:%.*]] = insertelement <16 x i8> [[TMP21]], i8 [[L1833]], i32 379; SM90-NEXT: [[TMP23:%.*]] = insertelement <16 x i8> [[TMP22]], i8 [[L1732]], i32 480; SM90-NEXT: [[TMP24:%.*]] = insertelement <16 x i8> [[TMP23]], i8 [[L1631]], i32 581; SM90-NEXT: [[TMP25:%.*]] = insertelement <16 x i8> [[TMP24]], i8 [[L1530]], i32 682; SM90-NEXT: [[TMP26:%.*]] = insertelement <16 x i8> [[TMP25]], i8 [[L1429]], i32 783; SM90-NEXT: [[TMP27:%.*]] = insertelement <16 x i8> [[TMP26]], i8 [[L1328]], i32 884; SM90-NEXT: [[TMP28:%.*]] = insertelement <16 x i8> [[TMP27]], i8 [[L1227]], i32 985; SM90-NEXT: [[TMP29:%.*]] = insertelement <16 x i8> [[TMP28]], i8 [[L1126]], i32 1086; SM90-NEXT: [[TMP30:%.*]] = insertelement <16 x i8> [[TMP29]], i8 [[L1025]], i32 1187; SM90-NEXT: [[TMP31:%.*]] = insertelement <16 x i8> [[TMP30]], i8 [[L1F40]], i32 1288; SM90-NEXT: [[TMP32:%.*]] = insertelement <16 x i8> [[TMP31]], i8 [[L1E39]], i32 1389; SM90-NEXT: [[TMP33:%.*]] = insertelement <16 x i8> [[TMP32]], i8 [[L1D38]], i32 1490; SM90-NEXT: [[TMP34:%.*]] = insertelement <16 x i8> [[TMP33]], i8 [[L1C37]], i32 1591; SM90-NEXT: store <16 x i8> [[TMP34]], ptr addrspace(1) [[PTR10]], align 1692; SM90-NEXT: ret void93;94; SM100-LABEL: define void @int8x32(95; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {96; SM100-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr addrspace(1) [[PTR]], i64 097; SM100-NEXT: [[TMP1:%.*]] = load <32 x i8>, ptr addrspace(1) [[PTR0]], align 3298; SM100-NEXT: [[L01:%.*]] = extractelement <32 x i8> [[TMP1]], i32 099; SM100-NEXT: [[L110:%.*]] = extractelement <32 x i8> [[TMP1]], i32 1100; SM100-NEXT: [[L211:%.*]] = extractelement <32 x i8> [[TMP1]], i32 2101; SM100-NEXT: [[L312:%.*]] = extractelement <32 x i8> [[TMP1]], i32 3102; SM100-NEXT: [[L413:%.*]] = extractelement <32 x i8> [[TMP1]], i32 4103; SM100-NEXT: [[L514:%.*]] = extractelement <32 x i8> [[TMP1]], i32 5104; SM100-NEXT: [[L615:%.*]] = extractelement <32 x i8> [[TMP1]], i32 6105; SM100-NEXT: [[L716:%.*]] = extractelement <32 x i8> [[TMP1]], i32 7106; SM100-NEXT: [[L817:%.*]] = extractelement <32 x i8> [[TMP1]], i32 8107; SM100-NEXT: [[L918:%.*]] = extractelement <32 x i8> [[TMP1]], i32 9108; SM100-NEXT: [[LA19:%.*]] = extractelement <32 x i8> [[TMP1]], i32 10109; SM100-NEXT: [[LB20:%.*]] = extractelement <32 x i8> [[TMP1]], i32 11110; SM100-NEXT: [[LC21:%.*]] = extractelement <32 x i8> [[TMP1]], i32 12111; SM100-NEXT: [[LD22:%.*]] = extractelement <32 x i8> [[TMP1]], i32 13112; SM100-NEXT: [[LE23:%.*]] = extractelement <32 x i8> [[TMP1]], i32 14113; SM100-NEXT: [[LF24:%.*]] = extractelement <32 x i8> [[TMP1]], i32 15114; SM100-NEXT: [[L1025:%.*]] = extractelement <32 x i8> [[TMP1]], i32 16115; SM100-NEXT: [[L1126:%.*]] = extractelement <32 x i8> [[TMP1]], i32 17116; SM100-NEXT: [[L1227:%.*]] = extractelement <32 x i8> [[TMP1]], i32 18117; SM100-NEXT: [[L1328:%.*]] = extractelement <32 x i8> [[TMP1]], i32 19118; SM100-NEXT: [[L1429:%.*]] = extractelement <32 x i8> [[TMP1]], i32 20119; SM100-NEXT: [[L1530:%.*]] = extractelement <32 x i8> [[TMP1]], i32 21120; SM100-NEXT: [[L1631:%.*]] = extractelement <32 x i8> [[TMP1]], i32 22121; SM100-NEXT: [[L1732:%.*]] = extractelement <32 x i8> [[TMP1]], i32 23122; SM100-NEXT: [[L1833:%.*]] = extractelement <32 x i8> [[TMP1]], i32 24123; SM100-NEXT: [[L1934:%.*]] = extractelement <32 x i8> [[TMP1]], i32 25124; SM100-NEXT: [[L1A35:%.*]] = extractelement <32 x i8> [[TMP1]], i32 26125; SM100-NEXT: [[L1B36:%.*]] = extractelement <32 x i8> [[TMP1]], i32 27126; SM100-NEXT: [[L1C37:%.*]] = extractelement <32 x i8> [[TMP1]], i32 28127; SM100-NEXT: [[L1D38:%.*]] = extractelement <32 x i8> [[TMP1]], i32 29128; SM100-NEXT: [[L1E39:%.*]] = extractelement <32 x i8> [[TMP1]], i32 30129; SM100-NEXT: [[L1F40:%.*]] = extractelement <32 x i8> [[TMP1]], i32 31130; SM100-NEXT: [[TMP2:%.*]] = insertelement <32 x i8> poison, i8 [[LB20]], i32 0131; SM100-NEXT: [[TMP3:%.*]] = insertelement <32 x i8> [[TMP2]], i8 [[LA19]], i32 1132; SM100-NEXT: [[TMP4:%.*]] = insertelement <32 x i8> [[TMP3]], i8 [[L918]], i32 2133; SM100-NEXT: [[TMP5:%.*]] = insertelement <32 x i8> [[TMP4]], i8 [[L817]], i32 3134; SM100-NEXT: [[TMP6:%.*]] = insertelement <32 x i8> [[TMP5]], i8 [[L716]], i32 4135; SM100-NEXT: [[TMP7:%.*]] = insertelement <32 x i8> [[TMP6]], i8 [[L615]], i32 5136; SM100-NEXT: [[TMP8:%.*]] = insertelement <32 x i8> [[TMP7]], i8 [[L514]], i32 6137; SM100-NEXT: [[TMP9:%.*]] = insertelement <32 x i8> [[TMP8]], i8 [[L413]], i32 7138; SM100-NEXT: [[TMP10:%.*]] = insertelement <32 x i8> [[TMP9]], i8 [[L312]], i32 8139; SM100-NEXT: [[TMP11:%.*]] = insertelement <32 x i8> [[TMP10]], i8 [[L211]], i32 9140; SM100-NEXT: [[TMP12:%.*]] = insertelement <32 x i8> [[TMP11]], i8 [[L110]], i32 10141; SM100-NEXT: [[TMP13:%.*]] = insertelement <32 x i8> [[TMP12]], i8 [[L01]], i32 11142; SM100-NEXT: [[TMP14:%.*]] = insertelement <32 x i8> [[TMP13]], i8 [[LF24]], i32 12143; SM100-NEXT: [[TMP15:%.*]] = insertelement <32 x i8> [[TMP14]], i8 [[LE23]], i32 13144; SM100-NEXT: [[TMP16:%.*]] = insertelement <32 x i8> [[TMP15]], i8 [[LD22]], i32 14145; SM100-NEXT: [[TMP17:%.*]] = insertelement <32 x i8> [[TMP16]], i8 [[LC21]], i32 15146; SM100-NEXT: [[TMP18:%.*]] = insertelement <32 x i8> [[TMP17]], i8 [[L1B36]], i32 16147; SM100-NEXT: [[TMP19:%.*]] = insertelement <32 x i8> [[TMP18]], i8 [[L1A35]], i32 17148; SM100-NEXT: [[TMP20:%.*]] = insertelement <32 x i8> [[TMP19]], i8 [[L1934]], i32 18149; SM100-NEXT: [[TMP21:%.*]] = insertelement <32 x i8> [[TMP20]], i8 [[L1833]], i32 19150; SM100-NEXT: [[TMP22:%.*]] = insertelement <32 x i8> [[TMP21]], i8 [[L1732]], i32 20151; SM100-NEXT: [[TMP23:%.*]] = insertelement <32 x i8> [[TMP22]], i8 [[L1631]], i32 21152; SM100-NEXT: [[TMP24:%.*]] = insertelement <32 x i8> [[TMP23]], i8 [[L1530]], i32 22153; SM100-NEXT: [[TMP25:%.*]] = insertelement <32 x i8> [[TMP24]], i8 [[L1429]], i32 23154; SM100-NEXT: [[TMP26:%.*]] = insertelement <32 x i8> [[TMP25]], i8 [[L1328]], i32 24155; SM100-NEXT: [[TMP27:%.*]] = insertelement <32 x i8> [[TMP26]], i8 [[L1227]], i32 25156; SM100-NEXT: [[TMP28:%.*]] = insertelement <32 x i8> [[TMP27]], i8 [[L1126]], i32 26157; SM100-NEXT: [[TMP29:%.*]] = insertelement <32 x i8> [[TMP28]], i8 [[L1025]], i32 27158; SM100-NEXT: [[TMP30:%.*]] = insertelement <32 x i8> [[TMP29]], i8 [[L1F40]], i32 28159; SM100-NEXT: [[TMP31:%.*]] = insertelement <32 x i8> [[TMP30]], i8 [[L1E39]], i32 29160; SM100-NEXT: [[TMP32:%.*]] = insertelement <32 x i8> [[TMP31]], i8 [[L1D38]], i32 30161; SM100-NEXT: [[TMP33:%.*]] = insertelement <32 x i8> [[TMP32]], i8 [[L1C37]], i32 31162; SM100-NEXT: store <32 x i8> [[TMP33]], ptr addrspace(1) [[PTR0]], align 32163; SM100-NEXT: ret void164;165 166 %ptr0 = getelementptr i8, ptr addrspace(1) %ptr, i64 0167 %ptr1 = getelementptr i8, ptr addrspace(1) %ptr, i64 1168 %ptr2 = getelementptr i8, ptr addrspace(1) %ptr, i64 2169 %ptr3 = getelementptr i8, ptr addrspace(1) %ptr, i64 3170 %ptr4 = getelementptr i8, ptr addrspace(1) %ptr, i64 4171 %ptr5 = getelementptr i8, ptr addrspace(1) %ptr, i64 5172 %ptr6 = getelementptr i8, ptr addrspace(1) %ptr, i64 6173 %ptr7 = getelementptr i8, ptr addrspace(1) %ptr, i64 7174 %ptr8 = getelementptr i8, ptr addrspace(1) %ptr, i64 8175 %ptr9 = getelementptr i8, ptr addrspace(1) %ptr, i64 9176 %ptra = getelementptr i8, ptr addrspace(1) %ptr, i64 10177 %ptrb = getelementptr i8, ptr addrspace(1) %ptr, i64 11178 %ptrc = getelementptr i8, ptr addrspace(1) %ptr, i64 12179 %ptrd = getelementptr i8, ptr addrspace(1) %ptr, i64 13180 %ptre = getelementptr i8, ptr addrspace(1) %ptr, i64 14181 %ptrf = getelementptr i8, ptr addrspace(1) %ptr, i64 15182 %ptr10 = getelementptr i8, ptr addrspace(1) %ptr, i64 16183 %ptr11 = getelementptr i8, ptr addrspace(1) %ptr, i64 17184 %ptr12 = getelementptr i8, ptr addrspace(1) %ptr, i64 18185 %ptr13 = getelementptr i8, ptr addrspace(1) %ptr, i64 19186 %ptr14 = getelementptr i8, ptr addrspace(1) %ptr, i64 20187 %ptr15 = getelementptr i8, ptr addrspace(1) %ptr, i64 21188 %ptr16 = getelementptr i8, ptr addrspace(1) %ptr, i64 22189 %ptr17 = getelementptr i8, ptr addrspace(1) %ptr, i64 23190 %ptr18 = getelementptr i8, ptr addrspace(1) %ptr, i64 24191 %ptr19 = getelementptr i8, ptr addrspace(1) %ptr, i64 25192 %ptr1a = getelementptr i8, ptr addrspace(1) %ptr, i64 26193 %ptr1b = getelementptr i8, ptr addrspace(1) %ptr, i64 27194 %ptr1c = getelementptr i8, ptr addrspace(1) %ptr, i64 28195 %ptr1d = getelementptr i8, ptr addrspace(1) %ptr, i64 29196 %ptr1e = getelementptr i8, ptr addrspace(1) %ptr, i64 30197 %ptr1f = getelementptr i8, ptr addrspace(1) %ptr, i64 31198 199 %l0 = load i8, ptr addrspace(1) %ptr0, align 32200 %l1 = load i8, ptr addrspace(1) %ptr1201 %l2 = load i8, ptr addrspace(1) %ptr2202 %l3 = load i8, ptr addrspace(1) %ptr3203 %l4 = load i8, ptr addrspace(1) %ptr4204 %l5 = load i8, ptr addrspace(1) %ptr5205 %l6 = load i8, ptr addrspace(1) %ptr6206 %l7 = load i8, ptr addrspace(1) %ptr7207 %l8 = load i8, ptr addrspace(1) %ptr8208 %l9 = load i8, ptr addrspace(1) %ptr9209 %la = load i8, ptr addrspace(1) %ptra210 %lb = load i8, ptr addrspace(1) %ptrb211 %lc = load i8, ptr addrspace(1) %ptrc212 %ld = load i8, ptr addrspace(1) %ptrd213 %le = load i8, ptr addrspace(1) %ptre214 %lf = load i8, ptr addrspace(1) %ptrf215 %l10 = load i8, ptr addrspace(1) %ptr10, align 16216 %l11 = load i8, ptr addrspace(1) %ptr11217 %l12 = load i8, ptr addrspace(1) %ptr12218 %l13 = load i8, ptr addrspace(1) %ptr13219 %l14 = load i8, ptr addrspace(1) %ptr14220 %l15 = load i8, ptr addrspace(1) %ptr15221 %l16 = load i8, ptr addrspace(1) %ptr16222 %l17 = load i8, ptr addrspace(1) %ptr17223 %l18 = load i8, ptr addrspace(1) %ptr18224 %l19 = load i8, ptr addrspace(1) %ptr19225 %l1a = load i8, ptr addrspace(1) %ptr1a226 %l1b = load i8, ptr addrspace(1) %ptr1b227 %l1c = load i8, ptr addrspace(1) %ptr1c228 %l1d = load i8, ptr addrspace(1) %ptr1d229 %l1e = load i8, ptr addrspace(1) %ptr1e230 %l1f = load i8, ptr addrspace(1) %ptr1f231 232 store i8 %lf, ptr addrspace(1) %ptrc233 store i8 %le, ptr addrspace(1) %ptrd234 store i8 %ld, ptr addrspace(1) %ptre235 store i8 %lc, ptr addrspace(1) %ptrf236 store i8 %lb, ptr addrspace(1) %ptr0, align 32237 store i8 %la, ptr addrspace(1) %ptr1238 store i8 %l9, ptr addrspace(1) %ptr2239 store i8 %l8, ptr addrspace(1) %ptr3240 store i8 %l7, ptr addrspace(1) %ptr4241 store i8 %l6, ptr addrspace(1) %ptr5242 store i8 %l5, ptr addrspace(1) %ptr6243 store i8 %l4, ptr addrspace(1) %ptr7244 store i8 %l3, ptr addrspace(1) %ptr8245 store i8 %l2, ptr addrspace(1) %ptr9246 store i8 %l1, ptr addrspace(1) %ptra247 store i8 %l0, ptr addrspace(1) %ptrb248 store i8 %l1f, ptr addrspace(1) %ptr1c249 store i8 %l1e, ptr addrspace(1) %ptr1d250 store i8 %l1d, ptr addrspace(1) %ptr1e251 store i8 %l1c, ptr addrspace(1) %ptr1f252 store i8 %l1b, ptr addrspace(1) %ptr10, align 16253 store i8 %l1a, ptr addrspace(1) %ptr11254 store i8 %l19, ptr addrspace(1) %ptr12255 store i8 %l18, ptr addrspace(1) %ptr13256 store i8 %l17, ptr addrspace(1) %ptr14257 store i8 %l16, ptr addrspace(1) %ptr15258 store i8 %l15, ptr addrspace(1) %ptr16259 store i8 %l14, ptr addrspace(1) %ptr17260 store i8 %l13, ptr addrspace(1) %ptr18261 store i8 %l12, ptr addrspace(1) %ptr19262 store i8 %l11, ptr addrspace(1) %ptr1a263 store i8 %l10, ptr addrspace(1) %ptr1b264 265 ret void266}267 268define void @int16x16(ptr addrspace(1) %ptr) {269; SM90-LABEL: define void @int16x16(270; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {271; SM90-NEXT: [[PTR0:%.*]] = getelementptr i16, ptr addrspace(1) [[PTR]], i64 0272; SM90-NEXT: [[PTR8:%.*]] = getelementptr i16, ptr addrspace(1) [[PTR]], i64 8273; SM90-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr addrspace(1) [[PTR0]], align 32274; SM90-NEXT: [[L01:%.*]] = extractelement <8 x i16> [[TMP1]], i32 0275; SM90-NEXT: [[L12:%.*]] = extractelement <8 x i16> [[TMP1]], i32 1276; SM90-NEXT: [[L23:%.*]] = extractelement <8 x i16> [[TMP1]], i32 2277; SM90-NEXT: [[L34:%.*]] = extractelement <8 x i16> [[TMP1]], i32 3278; SM90-NEXT: [[L45:%.*]] = extractelement <8 x i16> [[TMP1]], i32 4279; SM90-NEXT: [[L56:%.*]] = extractelement <8 x i16> [[TMP1]], i32 5280; SM90-NEXT: [[L67:%.*]] = extractelement <8 x i16> [[TMP1]], i32 6281; SM90-NEXT: [[L78:%.*]] = extractelement <8 x i16> [[TMP1]], i32 7282; SM90-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr addrspace(1) [[PTR8]], align 16283; SM90-NEXT: [[L89:%.*]] = extractelement <8 x i16> [[TMP2]], i32 0284; SM90-NEXT: [[L910:%.*]] = extractelement <8 x i16> [[TMP2]], i32 1285; SM90-NEXT: [[LA11:%.*]] = extractelement <8 x i16> [[TMP2]], i32 2286; SM90-NEXT: [[LB12:%.*]] = extractelement <8 x i16> [[TMP2]], i32 3287; SM90-NEXT: [[LC13:%.*]] = extractelement <8 x i16> [[TMP2]], i32 4288; SM90-NEXT: [[LD14:%.*]] = extractelement <8 x i16> [[TMP2]], i32 5289; SM90-NEXT: [[LE15:%.*]] = extractelement <8 x i16> [[TMP2]], i32 6290; SM90-NEXT: [[LF16:%.*]] = extractelement <8 x i16> [[TMP2]], i32 7291; SM90-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> poison, i16 [[LB12]], i32 0292; SM90-NEXT: [[TMP4:%.*]] = insertelement <8 x i16> [[TMP3]], i16 [[LA11]], i32 1293; SM90-NEXT: [[TMP5:%.*]] = insertelement <8 x i16> [[TMP4]], i16 [[L910]], i32 2294; SM90-NEXT: [[TMP6:%.*]] = insertelement <8 x i16> [[TMP5]], i16 [[L89]], i32 3295; SM90-NEXT: [[TMP7:%.*]] = insertelement <8 x i16> [[TMP6]], i16 [[L78]], i32 4296; SM90-NEXT: [[TMP8:%.*]] = insertelement <8 x i16> [[TMP7]], i16 [[L67]], i32 5297; SM90-NEXT: [[TMP9:%.*]] = insertelement <8 x i16> [[TMP8]], i16 [[L56]], i32 6298; SM90-NEXT: [[TMP10:%.*]] = insertelement <8 x i16> [[TMP9]], i16 [[L45]], i32 7299; SM90-NEXT: store <8 x i16> [[TMP10]], ptr addrspace(1) [[PTR0]], align 32300; SM90-NEXT: [[TMP11:%.*]] = insertelement <8 x i16> poison, i16 [[L34]], i32 0301; SM90-NEXT: [[TMP12:%.*]] = insertelement <8 x i16> [[TMP11]], i16 [[L23]], i32 1302; SM90-NEXT: [[TMP13:%.*]] = insertelement <8 x i16> [[TMP12]], i16 [[L12]], i32 2303; SM90-NEXT: [[TMP14:%.*]] = insertelement <8 x i16> [[TMP13]], i16 [[L01]], i32 3304; SM90-NEXT: [[TMP15:%.*]] = insertelement <8 x i16> [[TMP14]], i16 [[LF16]], i32 4305; SM90-NEXT: [[TMP16:%.*]] = insertelement <8 x i16> [[TMP15]], i16 [[LE15]], i32 5306; SM90-NEXT: [[TMP17:%.*]] = insertelement <8 x i16> [[TMP16]], i16 [[LD14]], i32 6307; SM90-NEXT: [[TMP18:%.*]] = insertelement <8 x i16> [[TMP17]], i16 [[LC13]], i32 7308; SM90-NEXT: store <8 x i16> [[TMP18]], ptr addrspace(1) [[PTR8]], align 16309; SM90-NEXT: ret void310;311; SM100-LABEL: define void @int16x16(312; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {313; SM100-NEXT: [[PTR0:%.*]] = getelementptr i16, ptr addrspace(1) [[PTR]], i64 0314; SM100-NEXT: [[TMP1:%.*]] = load <16 x i16>, ptr addrspace(1) [[PTR0]], align 32315; SM100-NEXT: [[L01:%.*]] = extractelement <16 x i16> [[TMP1]], i32 0316; SM100-NEXT: [[L12:%.*]] = extractelement <16 x i16> [[TMP1]], i32 1317; SM100-NEXT: [[L23:%.*]] = extractelement <16 x i16> [[TMP1]], i32 2318; SM100-NEXT: [[L34:%.*]] = extractelement <16 x i16> [[TMP1]], i32 3319; SM100-NEXT: [[L45:%.*]] = extractelement <16 x i16> [[TMP1]], i32 4320; SM100-NEXT: [[L56:%.*]] = extractelement <16 x i16> [[TMP1]], i32 5321; SM100-NEXT: [[L67:%.*]] = extractelement <16 x i16> [[TMP1]], i32 6322; SM100-NEXT: [[L78:%.*]] = extractelement <16 x i16> [[TMP1]], i32 7323; SM100-NEXT: [[L89:%.*]] = extractelement <16 x i16> [[TMP1]], i32 8324; SM100-NEXT: [[L910:%.*]] = extractelement <16 x i16> [[TMP1]], i32 9325; SM100-NEXT: [[LA11:%.*]] = extractelement <16 x i16> [[TMP1]], i32 10326; SM100-NEXT: [[LB12:%.*]] = extractelement <16 x i16> [[TMP1]], i32 11327; SM100-NEXT: [[LC13:%.*]] = extractelement <16 x i16> [[TMP1]], i32 12328; SM100-NEXT: [[LD14:%.*]] = extractelement <16 x i16> [[TMP1]], i32 13329; SM100-NEXT: [[LE15:%.*]] = extractelement <16 x i16> [[TMP1]], i32 14330; SM100-NEXT: [[LF16:%.*]] = extractelement <16 x i16> [[TMP1]], i32 15331; SM100-NEXT: [[TMP2:%.*]] = insertelement <16 x i16> poison, i16 [[LB12]], i32 0332; SM100-NEXT: [[TMP3:%.*]] = insertelement <16 x i16> [[TMP2]], i16 [[LA11]], i32 1333; SM100-NEXT: [[TMP4:%.*]] = insertelement <16 x i16> [[TMP3]], i16 [[L910]], i32 2334; SM100-NEXT: [[TMP5:%.*]] = insertelement <16 x i16> [[TMP4]], i16 [[L89]], i32 3335; SM100-NEXT: [[TMP6:%.*]] = insertelement <16 x i16> [[TMP5]], i16 [[L78]], i32 4336; SM100-NEXT: [[TMP7:%.*]] = insertelement <16 x i16> [[TMP6]], i16 [[L67]], i32 5337; SM100-NEXT: [[TMP8:%.*]] = insertelement <16 x i16> [[TMP7]], i16 [[L56]], i32 6338; SM100-NEXT: [[TMP9:%.*]] = insertelement <16 x i16> [[TMP8]], i16 [[L45]], i32 7339; SM100-NEXT: [[TMP10:%.*]] = insertelement <16 x i16> [[TMP9]], i16 [[L34]], i32 8340; SM100-NEXT: [[TMP11:%.*]] = insertelement <16 x i16> [[TMP10]], i16 [[L23]], i32 9341; SM100-NEXT: [[TMP12:%.*]] = insertelement <16 x i16> [[TMP11]], i16 [[L12]], i32 10342; SM100-NEXT: [[TMP13:%.*]] = insertelement <16 x i16> [[TMP12]], i16 [[L01]], i32 11343; SM100-NEXT: [[TMP14:%.*]] = insertelement <16 x i16> [[TMP13]], i16 [[LF16]], i32 12344; SM100-NEXT: [[TMP15:%.*]] = insertelement <16 x i16> [[TMP14]], i16 [[LE15]], i32 13345; SM100-NEXT: [[TMP16:%.*]] = insertelement <16 x i16> [[TMP15]], i16 [[LD14]], i32 14346; SM100-NEXT: [[TMP17:%.*]] = insertelement <16 x i16> [[TMP16]], i16 [[LC13]], i32 15347; SM100-NEXT: store <16 x i16> [[TMP17]], ptr addrspace(1) [[PTR0]], align 32348; SM100-NEXT: ret void349;350 351 %ptr0 = getelementptr i16, ptr addrspace(1) %ptr, i64 0352 %ptr1 = getelementptr i16, ptr addrspace(1) %ptr, i64 1353 %ptr2 = getelementptr i16, ptr addrspace(1) %ptr, i64 2354 %ptr3 = getelementptr i16, ptr addrspace(1) %ptr, i64 3355 %ptr4 = getelementptr i16, ptr addrspace(1) %ptr, i64 4356 %ptr5 = getelementptr i16, ptr addrspace(1) %ptr, i64 5357 %ptr6 = getelementptr i16, ptr addrspace(1) %ptr, i64 6358 %ptr7 = getelementptr i16, ptr addrspace(1) %ptr, i64 7359 %ptr8 = getelementptr i16, ptr addrspace(1) %ptr, i64 8360 %ptr9 = getelementptr i16, ptr addrspace(1) %ptr, i64 9361 %ptra = getelementptr i16, ptr addrspace(1) %ptr, i64 10362 %ptrb = getelementptr i16, ptr addrspace(1) %ptr, i64 11363 %ptrc = getelementptr i16, ptr addrspace(1) %ptr, i64 12364 %ptrd = getelementptr i16, ptr addrspace(1) %ptr, i64 13365 %ptre = getelementptr i16, ptr addrspace(1) %ptr, i64 14366 %ptrf = getelementptr i16, ptr addrspace(1) %ptr, i64 15367 368 %l0 = load i16, ptr addrspace(1) %ptr0, align 32369 %l1 = load i16, ptr addrspace(1) %ptr1370 %l2 = load i16, ptr addrspace(1) %ptr2371 %l3 = load i16, ptr addrspace(1) %ptr3372 %l4 = load i16, ptr addrspace(1) %ptr4373 %l5 = load i16, ptr addrspace(1) %ptr5374 %l6 = load i16, ptr addrspace(1) %ptr6375 %l7 = load i16, ptr addrspace(1) %ptr7376 %l8 = load i16, ptr addrspace(1) %ptr8, align 16377 %l9 = load i16, ptr addrspace(1) %ptr9378 %la = load i16, ptr addrspace(1) %ptra379 %lb = load i16, ptr addrspace(1) %ptrb380 %lc = load i16, ptr addrspace(1) %ptrc381 %ld = load i16, ptr addrspace(1) %ptrd382 %le = load i16, ptr addrspace(1) %ptre383 %lf = load i16, ptr addrspace(1) %ptrf384 385 store i16 %lf, ptr addrspace(1) %ptrc386 store i16 %le, ptr addrspace(1) %ptrd387 store i16 %ld, ptr addrspace(1) %ptre388 store i16 %lc, ptr addrspace(1) %ptrf389 store i16 %lb, ptr addrspace(1) %ptr0, align 32390 store i16 %la, ptr addrspace(1) %ptr1391 store i16 %l9, ptr addrspace(1) %ptr2392 store i16 %l8, ptr addrspace(1) %ptr3393 store i16 %l7, ptr addrspace(1) %ptr4394 store i16 %l6, ptr addrspace(1) %ptr5395 store i16 %l5, ptr addrspace(1) %ptr6396 store i16 %l4, ptr addrspace(1) %ptr7397 store i16 %l3, ptr addrspace(1) %ptr8, align 16398 store i16 %l2, ptr addrspace(1) %ptr9399 store i16 %l1, ptr addrspace(1) %ptra400 store i16 %l0, ptr addrspace(1) %ptrb401 402 ret void403}404 405define void @int32x8(ptr addrspace(1) %ptr) {406; SM90-LABEL: define void @int32x8(407; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {408; SM90-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 0409; SM90-NEXT: [[PTR4:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4410; SM90-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(1) [[PTR0]], align 32411; SM90-NEXT: [[L01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0412; SM90-NEXT: [[L12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1413; SM90-NEXT: [[L23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2414; SM90-NEXT: [[L34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3415; SM90-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[PTR4]], align 16416; SM90-NEXT: [[L45:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0417; SM90-NEXT: [[L56:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1418; SM90-NEXT: [[L67:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2419; SM90-NEXT: [[L78:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3420; SM90-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[L78]], i32 0421; SM90-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[L67]], i32 1422; SM90-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[L56]], i32 2423; SM90-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[L45]], i32 3424; SM90-NEXT: store <4 x i32> [[TMP6]], ptr addrspace(1) [[PTR0]], align 32425; SM90-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[L34]], i32 0426; SM90-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[L23]], i32 1427; SM90-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[L12]], i32 2428; SM90-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[L01]], i32 3429; SM90-NEXT: store <4 x i32> [[TMP10]], ptr addrspace(1) [[PTR4]], align 16430; SM90-NEXT: ret void431;432; SM100-LABEL: define void @int32x8(433; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {434; SM100-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 0435; SM100-NEXT: [[TMP1:%.*]] = load <8 x i32>, ptr addrspace(1) [[PTR0]], align 32436; SM100-NEXT: [[L01:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0437; SM100-NEXT: [[L12:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1438; SM100-NEXT: [[L23:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2439; SM100-NEXT: [[L34:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3440; SM100-NEXT: [[L45:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4441; SM100-NEXT: [[L56:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5442; SM100-NEXT: [[L67:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6443; SM100-NEXT: [[L78:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7444; SM100-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[L78]], i32 0445; SM100-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[L67]], i32 1446; SM100-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[L56]], i32 2447; SM100-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[L45]], i32 3448; SM100-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[L34]], i32 4449; SM100-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[L23]], i32 5450; SM100-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[L12]], i32 6451; SM100-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[L01]], i32 7452; SM100-NEXT: store <8 x i32> [[TMP9]], ptr addrspace(1) [[PTR0]], align 32453; SM100-NEXT: ret void454;455 456 %ptr0 = getelementptr i32, ptr addrspace(1) %ptr, i64 0457 %ptr1 = getelementptr i32, ptr addrspace(1) %ptr, i64 1458 %ptr2 = getelementptr i32, ptr addrspace(1) %ptr, i64 2459 %ptr3 = getelementptr i32, ptr addrspace(1) %ptr, i64 3460 %ptr4 = getelementptr i32, ptr addrspace(1) %ptr, i64 4461 %ptr5 = getelementptr i32, ptr addrspace(1) %ptr, i64 5462 %ptr6 = getelementptr i32, ptr addrspace(1) %ptr, i64 6463 %ptr7 = getelementptr i32, ptr addrspace(1) %ptr, i64 7464 465 %l0 = load i32, ptr addrspace(1) %ptr0, align 32466 %l1 = load i32, ptr addrspace(1) %ptr1467 %l2 = load i32, ptr addrspace(1) %ptr2468 %l3 = load i32, ptr addrspace(1) %ptr3469 %l4 = load i32, ptr addrspace(1) %ptr4, align 16470 %l5 = load i32, ptr addrspace(1) %ptr5471 %l6 = load i32, ptr addrspace(1) %ptr6472 %l7 = load i32, ptr addrspace(1) %ptr7473 474 store i32 %l7, ptr addrspace(1) %ptr0, align 32475 store i32 %l6, ptr addrspace(1) %ptr1476 store i32 %l5, ptr addrspace(1) %ptr2477 store i32 %l4, ptr addrspace(1) %ptr3478 store i32 %l3, ptr addrspace(1) %ptr4, align 16479 store i32 %l2, ptr addrspace(1) %ptr5480 store i32 %l1, ptr addrspace(1) %ptr6481 store i32 %l0, ptr addrspace(1) %ptr7482 483 ret void484}485 486define void @int64x4(ptr addrspace(1) %ptr) {487; SM90-LABEL: define void @int64x4(488; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {489; SM90-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr addrspace(1) [[PTR]], i64 0490; SM90-NEXT: [[PTR2:%.*]] = getelementptr i64, ptr addrspace(1) [[PTR]], i64 2491; SM90-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[PTR0]], align 32492; SM90-NEXT: [[L01:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0493; SM90-NEXT: [[L12:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1494; SM90-NEXT: [[TMP2:%.*]] = load <2 x i64>, ptr addrspace(1) [[PTR2]], align 16495; SM90-NEXT: [[L23:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0496; SM90-NEXT: [[L34:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1497; SM90-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> poison, i64 [[L34]], i32 0498; SM90-NEXT: [[TMP4:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[L23]], i32 1499; SM90-NEXT: store <2 x i64> [[TMP4]], ptr addrspace(1) [[PTR0]], align 32500; SM90-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[L12]], i32 0501; SM90-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[L01]], i32 1502; SM90-NEXT: store <2 x i64> [[TMP6]], ptr addrspace(1) [[PTR2]], align 16503; SM90-NEXT: ret void504;505; SM100-LABEL: define void @int64x4(506; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {507; SM100-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr addrspace(1) [[PTR]], i64 0508; SM100-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr addrspace(1) [[PTR0]], align 32509; SM100-NEXT: [[L01:%.*]] = extractelement <4 x i64> [[TMP1]], i32 0510; SM100-NEXT: [[L12:%.*]] = extractelement <4 x i64> [[TMP1]], i32 1511; SM100-NEXT: [[L23:%.*]] = extractelement <4 x i64> [[TMP1]], i32 2512; SM100-NEXT: [[L34:%.*]] = extractelement <4 x i64> [[TMP1]], i32 3513; SM100-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[L34]], i32 0514; SM100-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[L23]], i32 1515; SM100-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[L12]], i32 2516; SM100-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> [[TMP4]], i64 [[L01]], i32 3517; SM100-NEXT: store <4 x i64> [[TMP5]], ptr addrspace(1) [[PTR0]], align 32518; SM100-NEXT: ret void519;520 521 %ptr0 = getelementptr i64, ptr addrspace(1) %ptr, i64 0522 %ptr1 = getelementptr i64, ptr addrspace(1) %ptr, i64 1523 %ptr2 = getelementptr i64, ptr addrspace(1) %ptr, i64 2524 %ptr3 = getelementptr i64, ptr addrspace(1) %ptr, i64 3525 526 %l0 = load i64, ptr addrspace(1) %ptr0, align 32527 %l1 = load i64, ptr addrspace(1) %ptr1528 %l2 = load i64, ptr addrspace(1) %ptr2, align 16529 %l3 = load i64, ptr addrspace(1) %ptr3530 531 store i64 %l3, ptr addrspace(1) %ptr0, align 32532 store i64 %l2, ptr addrspace(1) %ptr1533 store i64 %l1, ptr addrspace(1) %ptr2, align 16534 store i64 %l0, ptr addrspace(1) %ptr3535 536 ret void537}538 539define void @float32x8(ptr addrspace(1) %ptr) {540; SM90-LABEL: define void @float32x8(541; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {542; SM90-NEXT: [[PTR0:%.*]] = getelementptr float, ptr addrspace(1) [[PTR]], i64 0543; SM90-NEXT: [[PTR4:%.*]] = getelementptr float, ptr addrspace(1) [[PTR]], i64 4544; SM90-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[PTR0]], align 32545; SM90-NEXT: [[L01:%.*]] = extractelement <4 x float> [[TMP1]], i32 0546; SM90-NEXT: [[L12:%.*]] = extractelement <4 x float> [[TMP1]], i32 1547; SM90-NEXT: [[L23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2548; SM90-NEXT: [[L34:%.*]] = extractelement <4 x float> [[TMP1]], i32 3549; SM90-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[PTR4]], align 16550; SM90-NEXT: [[L45:%.*]] = extractelement <4 x float> [[TMP2]], i32 0551; SM90-NEXT: [[L56:%.*]] = extractelement <4 x float> [[TMP2]], i32 1552; SM90-NEXT: [[L67:%.*]] = extractelement <4 x float> [[TMP2]], i32 2553; SM90-NEXT: [[L78:%.*]] = extractelement <4 x float> [[TMP2]], i32 3554; SM90-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[L78]], i32 0555; SM90-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[L67]], i32 1556; SM90-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[L56]], i32 2557; SM90-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[L45]], i32 3558; SM90-NEXT: store <4 x float> [[TMP6]], ptr addrspace(1) [[PTR0]], align 32559; SM90-NEXT: [[TMP7:%.*]] = insertelement <4 x float> poison, float [[L34]], i32 0560; SM90-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[L23]], i32 1561; SM90-NEXT: [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[L12]], i32 2562; SM90-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[L01]], i32 3563; SM90-NEXT: store <4 x float> [[TMP10]], ptr addrspace(1) [[PTR4]], align 16564; SM90-NEXT: ret void565;566; SM100-LABEL: define void @float32x8(567; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {568; SM100-NEXT: [[PTR0:%.*]] = getelementptr float, ptr addrspace(1) [[PTR]], i64 0569; SM100-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[PTR0]], align 32570; SM100-NEXT: [[L01:%.*]] = extractelement <8 x float> [[TMP1]], i32 0571; SM100-NEXT: [[L12:%.*]] = extractelement <8 x float> [[TMP1]], i32 1572; SM100-NEXT: [[L23:%.*]] = extractelement <8 x float> [[TMP1]], i32 2573; SM100-NEXT: [[L34:%.*]] = extractelement <8 x float> [[TMP1]], i32 3574; SM100-NEXT: [[L45:%.*]] = extractelement <8 x float> [[TMP1]], i32 4575; SM100-NEXT: [[L56:%.*]] = extractelement <8 x float> [[TMP1]], i32 5576; SM100-NEXT: [[L67:%.*]] = extractelement <8 x float> [[TMP1]], i32 6577; SM100-NEXT: [[L78:%.*]] = extractelement <8 x float> [[TMP1]], i32 7578; SM100-NEXT: [[TMP2:%.*]] = insertelement <8 x float> poison, float [[L78]], i32 0579; SM100-NEXT: [[TMP3:%.*]] = insertelement <8 x float> [[TMP2]], float [[L67]], i32 1580; SM100-NEXT: [[TMP4:%.*]] = insertelement <8 x float> [[TMP3]], float [[L56]], i32 2581; SM100-NEXT: [[TMP5:%.*]] = insertelement <8 x float> [[TMP4]], float [[L45]], i32 3582; SM100-NEXT: [[TMP6:%.*]] = insertelement <8 x float> [[TMP5]], float [[L34]], i32 4583; SM100-NEXT: [[TMP7:%.*]] = insertelement <8 x float> [[TMP6]], float [[L23]], i32 5584; SM100-NEXT: [[TMP8:%.*]] = insertelement <8 x float> [[TMP7]], float [[L12]], i32 6585; SM100-NEXT: [[TMP9:%.*]] = insertelement <8 x float> [[TMP8]], float [[L01]], i32 7586; SM100-NEXT: store <8 x float> [[TMP9]], ptr addrspace(1) [[PTR0]], align 32587; SM100-NEXT: ret void588;589 590 %ptr0 = getelementptr float, ptr addrspace(1) %ptr, i64 0591 %ptr1 = getelementptr float, ptr addrspace(1) %ptr, i64 1592 %ptr2 = getelementptr float, ptr addrspace(1) %ptr, i64 2593 %ptr3 = getelementptr float, ptr addrspace(1) %ptr, i64 3594 %ptr4 = getelementptr float, ptr addrspace(1) %ptr, i64 4595 %ptr5 = getelementptr float, ptr addrspace(1) %ptr, i64 5596 %ptr6 = getelementptr float, ptr addrspace(1) %ptr, i64 6597 %ptr7 = getelementptr float, ptr addrspace(1) %ptr, i64 7598 599 %l0 = load float, ptr addrspace(1) %ptr0, align 32600 %l1 = load float, ptr addrspace(1) %ptr1601 %l2 = load float, ptr addrspace(1) %ptr2602 %l3 = load float, ptr addrspace(1) %ptr3603 %l4 = load float, ptr addrspace(1) %ptr4, align 16604 %l5 = load float, ptr addrspace(1) %ptr5605 %l6 = load float, ptr addrspace(1) %ptr6606 %l7 = load float, ptr addrspace(1) %ptr7607 608 store float %l7, ptr addrspace(1) %ptr0, align 32609 store float %l6, ptr addrspace(1) %ptr1610 store float %l5, ptr addrspace(1) %ptr2611 store float %l4, ptr addrspace(1) %ptr3612 store float %l3, ptr addrspace(1) %ptr4, align 16613 store float %l2, ptr addrspace(1) %ptr5614 store float %l1, ptr addrspace(1) %ptr6615 store float %l0, ptr addrspace(1) %ptr7616 617 ret void618}619 620define void @float64x4(ptr addrspace(1) %ptr) {621; SM90-LABEL: define void @float64x4(622; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {623; SM90-NEXT: [[PTR0:%.*]] = getelementptr double, ptr addrspace(1) [[PTR]], i64 0624; SM90-NEXT: [[PTR2:%.*]] = getelementptr double, ptr addrspace(1) [[PTR]], i64 2625; SM90-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr addrspace(1) [[PTR0]], align 32626; SM90-NEXT: [[L01:%.*]] = extractelement <2 x double> [[TMP1]], i32 0627; SM90-NEXT: [[L12:%.*]] = extractelement <2 x double> [[TMP1]], i32 1628; SM90-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr addrspace(1) [[PTR2]], align 16629; SM90-NEXT: [[L23:%.*]] = extractelement <2 x double> [[TMP2]], i32 0630; SM90-NEXT: [[L34:%.*]] = extractelement <2 x double> [[TMP2]], i32 1631; SM90-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[L34]], i32 0632; SM90-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[L23]], i32 1633; SM90-NEXT: store <2 x double> [[TMP4]], ptr addrspace(1) [[PTR0]], align 32634; SM90-NEXT: [[TMP5:%.*]] = insertelement <2 x double> poison, double [[L12]], i32 0635; SM90-NEXT: [[TMP6:%.*]] = insertelement <2 x double> [[TMP5]], double [[L01]], i32 1636; SM90-NEXT: store <2 x double> [[TMP6]], ptr addrspace(1) [[PTR2]], align 16637; SM90-NEXT: ret void638;639; SM100-LABEL: define void @float64x4(640; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {641; SM100-NEXT: [[PTR0:%.*]] = getelementptr double, ptr addrspace(1) [[PTR]], i64 0642; SM100-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr addrspace(1) [[PTR0]], align 32643; SM100-NEXT: [[L01:%.*]] = extractelement <4 x double> [[TMP1]], i32 0644; SM100-NEXT: [[L12:%.*]] = extractelement <4 x double> [[TMP1]], i32 1645; SM100-NEXT: [[L23:%.*]] = extractelement <4 x double> [[TMP1]], i32 2646; SM100-NEXT: [[L34:%.*]] = extractelement <4 x double> [[TMP1]], i32 3647; SM100-NEXT: [[TMP2:%.*]] = insertelement <4 x double> poison, double [[L34]], i32 0648; SM100-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[L23]], i32 1649; SM100-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[L12]], i32 2650; SM100-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[L01]], i32 3651; SM100-NEXT: store <4 x double> [[TMP5]], ptr addrspace(1) [[PTR0]], align 32652; SM100-NEXT: ret void653;654 655 %ptr0 = getelementptr double, ptr addrspace(1) %ptr, i64 0656 %ptr1 = getelementptr double, ptr addrspace(1) %ptr, i64 1657 %ptr2 = getelementptr double, ptr addrspace(1) %ptr, i64 2658 %ptr3 = getelementptr double, ptr addrspace(1) %ptr, i64 3659 660 %l0 = load double, ptr addrspace(1) %ptr0, align 32661 %l1 = load double, ptr addrspace(1) %ptr1662 %l2 = load double, ptr addrspace(1) %ptr2, align 16663 %l3 = load double, ptr addrspace(1) %ptr3664 665 store double %l3, ptr addrspace(1) %ptr0, align 32666 store double %l2, ptr addrspace(1) %ptr1667 store double %l1, ptr addrspace(1) %ptr2, align 16668 store double %l0, ptr addrspace(1) %ptr3669 670 ret void671}672 673define void @int32x8_non_global(ptr %ptr) {674; CHECK-LABEL: define void @int32x8_non_global(675; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {676; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i64 0677; CHECK-NEXT: [[PTR4:%.*]] = getelementptr i32, ptr [[PTR]], i64 4678; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[PTR0]], align 32679; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0680; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1681; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2682; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3683; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[PTR4]], align 16684; CHECK-NEXT: [[L45:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0685; CHECK-NEXT: [[L56:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1686; CHECK-NEXT: [[L67:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2687; CHECK-NEXT: [[L78:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3688; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[L78]], i32 0689; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[L67]], i32 1690; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[L56]], i32 2691; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[L45]], i32 3692; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[PTR0]], align 32693; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[L34]], i32 0694; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[L23]], i32 1695; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[L12]], i32 2696; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[L01]], i32 3697; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[PTR4]], align 16698; CHECK-NEXT: ret void699 700 %ptr0 = getelementptr i32, ptr %ptr, i64 0701 %ptr1 = getelementptr i32, ptr %ptr, i64 1702 %ptr2 = getelementptr i32, ptr %ptr, i64 2703 %ptr3 = getelementptr i32, ptr %ptr, i64 3704 %ptr4 = getelementptr i32, ptr %ptr, i64 4705 %ptr5 = getelementptr i32, ptr %ptr, i64 5706 %ptr6 = getelementptr i32, ptr %ptr, i64 6707 %ptr7 = getelementptr i32, ptr %ptr, i64 7708 709 %l0 = load i32, ptr %ptr0, align 32710 %l1 = load i32, ptr %ptr1711 %l2 = load i32, ptr %ptr2712 %l3 = load i32, ptr %ptr3713 %l4 = load i32, ptr %ptr4, align 16714 %l5 = load i32, ptr %ptr5715 %l6 = load i32, ptr %ptr6716 %l7 = load i32, ptr %ptr7717 718 store i32 %l7, ptr %ptr0, align 32719 store i32 %l6, ptr %ptr1720 store i32 %l5, ptr %ptr2721 store i32 %l4, ptr %ptr3722 store i32 %l3, ptr %ptr4, align 16723 store i32 %l2, ptr %ptr5724 store i32 %l1, ptr %ptr6725 store i32 %l0, ptr %ptr7726 727 ret void728}729