brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s3 4; Note: if you're adding tests here, also add them to5; buffer-fat-pointers-contents-legalization.ll to make sure the output of this6; transformation can codegen.7 8target triple = "amdgcn--"9 10;;; Legal types. These are natively supported, no casts should be performed.11 12define i8 @load_i8(ptr addrspace(8) inreg %buf) {13; CHECK-LABEL: define i8 @load_i8(14; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0:[0-9]+]] {15; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)16; CHECK-NEXT:    ret i8 [[RET]]17;18  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)19  %ret = load i8, ptr addrspace(7) %p20  ret i8 %ret21}22 23define void @store_i8(i8 %data, ptr addrspace(8) inreg %buf) {24; CHECK-LABEL: define void @store_i8(25; CHECK-SAME: i8 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {26; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)27; CHECK-NEXT:    ret void28;29  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)30  store i8 %data, ptr addrspace(7) %p31  ret void32}33 34define i16 @load_i16(ptr addrspace(8) inreg %buf) {35; CHECK-LABEL: define i16 @load_i16(36; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {37; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)38; CHECK-NEXT:    ret i16 [[RET]]39;40  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)41  %ret = load i16, ptr addrspace(7) %p42  ret i16 %ret43}44 45define void @store_i16(i16 %data, ptr addrspace(8) inreg %buf) {46; CHECK-LABEL: define void @store_i16(47; CHECK-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {48; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)49; CHECK-NEXT:    ret void50;51  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)52  store i16 %data, ptr addrspace(7) %p53  ret void54}55 56define i32 @load_i32(ptr addrspace(8) inreg %buf) {57; CHECK-LABEL: define i32 @load_i32(58; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {59; CHECK-NEXT:    [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)60; CHECK-NEXT:    ret i32 [[RET]]61;62  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)63  %ret = load i32, ptr addrspace(7) %p64  ret i32 %ret65}66 67define void @store_i32(i32 %data, ptr addrspace(8) inreg %buf) {68; CHECK-LABEL: define void @store_i32(69; CHECK-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {70; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)71; CHECK-NEXT:    ret void72;73  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)74  store i32 %data, ptr addrspace(7) %p75  ret void76}77 78define i64 @load_i64(ptr addrspace(8) inreg %buf) {79; CHECK-LABEL: define i64 @load_i64(80; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {81; CHECK-NEXT:    [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)82; CHECK-NEXT:    ret i64 [[RET]]83;84  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)85  %ret = load i64, ptr addrspace(7) %p86  ret i64 %ret87}88 89define void @store_i64(i64 %data, ptr addrspace(8) inreg %buf) {90; CHECK-LABEL: define void @store_i64(91; CHECK-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {92; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)93; CHECK-NEXT:    ret void94;95  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)96  store i64 %data, ptr addrspace(7) %p97  ret void98}99 100define i128 @load_i128(ptr addrspace(8) inreg %buf) {101; CHECK-LABEL: define i128 @load_i128(102; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {103; CHECK-NEXT:    [[RET:%.*]] = call i128 @llvm.amdgcn.raw.ptr.buffer.load.i128(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)104; CHECK-NEXT:    ret i128 [[RET]]105;106  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)107  %ret = load i128, ptr addrspace(7) %p108  ret i128 %ret109}110 111define void @store_i128(i128 %data, ptr addrspace(8) inreg %buf) {112; CHECK-LABEL: define void @store_i128(113; CHECK-SAME: i128 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {114; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i128(i128 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)115; CHECK-NEXT:    ret void116;117  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)118  store i128 %data, ptr addrspace(7) %p119  ret void120}121 122define <1 x i32> @load_v1i32(ptr addrspace(8) inreg %buf) {123; CHECK-LABEL: define <1 x i32> @load_v1i32(124; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {125; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)126; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <1 x i32>127; CHECK-NEXT:    ret <1 x i32> [[RET]]128;129  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)130  %ret = load <1 x i32>, ptr addrspace(7) %p131  ret <1 x i32> %ret132}133 134define void @store_v1i32(<1 x i32> %data, ptr addrspace(8) inreg %buf) {135; CHECK-LABEL: define void @store_v1i32(136; CHECK-SAME: <1 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {137; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA]] to i32138; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)139; CHECK-NEXT:    ret void140;141  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)142  store <1 x i32> %data, ptr addrspace(7) %p143  ret void144}145 146define <2 x i32> @load_v2i32(ptr addrspace(8) inreg %buf) {147; CHECK-LABEL: define <2 x i32> @load_v2i32(148; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {149; CHECK-NEXT:    [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)150; CHECK-NEXT:    ret <2 x i32> [[RET]]151;152  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)153  %ret = load <2 x i32>, ptr addrspace(7) %p154  ret <2 x i32> %ret155}156 157define void @store_v2i32(<2 x i32> %data, ptr addrspace(8) inreg %buf) {158; CHECK-LABEL: define void @store_v2i32(159; CHECK-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {160; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)161; CHECK-NEXT:    ret void162;163  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)164  store <2 x i32> %data, ptr addrspace(7) %p165  ret void166}167 168define <3 x i32> @load_v3i32(ptr addrspace(8) inreg %buf) {169; CHECK-LABEL: define <3 x i32> @load_v3i32(170; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {171; CHECK-NEXT:    [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)172; CHECK-NEXT:    ret <3 x i32> [[RET]]173;174  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)175  %ret = load <3 x i32>, ptr addrspace(7) %p176  ret <3 x i32> %ret177}178 179define void @store_v3i32(<3 x i32> %data, ptr addrspace(8) inreg %buf) {180; CHECK-LABEL: define void @store_v3i32(181; CHECK-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {182; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)183; CHECK-NEXT:    ret void184;185  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)186  store <3 x i32> %data, ptr addrspace(7) %p187  ret void188}189 190define <4 x i32> @load_v4i32(ptr addrspace(8) inreg %buf) {191; CHECK-LABEL: define <4 x i32> @load_v4i32(192; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {193; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)194; CHECK-NEXT:    ret <4 x i32> [[RET]]195;196  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)197  %ret = load <4 x i32>, ptr addrspace(7) %p198  ret <4 x i32> %ret199}200 201define void @store_v4i32(<4 x i32> %data, ptr addrspace(8) inreg %buf) {202; CHECK-LABEL: define void @store_v4i32(203; CHECK-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {204; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)205; CHECK-NEXT:    ret void206;207  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)208  store <4 x i32> %data, ptr addrspace(7) %p209  ret void210}211 212define <2 x i16> @load_v2i16(ptr addrspace(8) inreg %buf) {213; CHECK-LABEL: define <2 x i16> @load_v2i16(214; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {215; CHECK-NEXT:    [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)216; CHECK-NEXT:    ret <2 x i16> [[RET]]217;218  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)219  %ret = load <2 x i16>, ptr addrspace(7) %p220  ret <2 x i16> %ret221}222 223define void @store_v2i16(<2 x i16> %data, ptr addrspace(8) inreg %buf) {224; CHECK-LABEL: define void @store_v2i16(225; CHECK-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {226; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)227; CHECK-NEXT:    ret void228;229  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)230  store <2 x i16> %data, ptr addrspace(7) %p231  ret void232}233 234define <4 x i16> @load_v4i16(ptr addrspace(8) inreg %buf) {235; CHECK-LABEL: define <4 x i16> @load_v4i16(236; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {237; CHECK-NEXT:    [[RET:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)238; CHECK-NEXT:    ret <4 x i16> [[RET]]239;240  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)241  %ret = load <4 x i16>, ptr addrspace(7) %p242  ret <4 x i16> %ret243}244 245define void @store_v4i16(<4 x i16> %data, ptr addrspace(8) inreg %buf) {246; CHECK-LABEL: define void @store_v4i16(247; CHECK-SAME: <4 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {248; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)249; CHECK-NEXT:    ret void250;251  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)252  store <4 x i16> %data, ptr addrspace(7) %p253  ret void254}255 256define <8 x i16> @load_v8i16(ptr addrspace(8) inreg %buf) {257; CHECK-LABEL: define <8 x i16> @load_v8i16(258; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {259; CHECK-NEXT:    [[RET:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)260; CHECK-NEXT:    ret <8 x i16> [[RET]]261;262  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)263  %ret = load <8 x i16>, ptr addrspace(7) %p264  ret <8 x i16> %ret265}266 267define void @store_v8i16(<8 x i16> %data, ptr addrspace(8) inreg %buf) {268; CHECK-LABEL: define void @store_v8i16(269; CHECK-SAME: <8 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {270; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)271; CHECK-NEXT:    ret void272;273  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)274  store <8 x i16> %data, ptr addrspace(7) %p275  ret void276}277 278define <2 x i64> @load_v2i64(ptr addrspace(8) inreg %buf) {279; CHECK-LABEL: define <2 x i64> @load_v2i64(280; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {281; CHECK-NEXT:    [[RET:%.*]] = call <2 x i64> @llvm.amdgcn.raw.ptr.buffer.load.v2i64(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)282; CHECK-NEXT:    ret <2 x i64> [[RET]]283;284  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)285  %ret = load <2 x i64>, ptr addrspace(7) %p286  ret <2 x i64> %ret287}288 289define void @store_v2i64(<2 x i64> %data, ptr addrspace(8) inreg %buf) {290; CHECK-LABEL: define void @store_v2i64(291; CHECK-SAME: <2 x i64> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {292; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i64(<2 x i64> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)293; CHECK-NEXT:    ret void294;295  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)296  store <2 x i64> %data, ptr addrspace(7) %p297  ret void298}299 300define half @load_f16(ptr addrspace(8) inreg %buf) {301; CHECK-LABEL: define half @load_f16(302; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {303; CHECK-NEXT:    [[RET:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)304; CHECK-NEXT:    ret half [[RET]]305;306  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)307  %ret = load half, ptr addrspace(7) %p308  ret half %ret309}310 311define void @store_f16(half %data, ptr addrspace(8) inreg %buf) {312; CHECK-LABEL: define void @store_f16(313; CHECK-SAME: half [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {314; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)315; CHECK-NEXT:    ret void316;317  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)318  store half %data, ptr addrspace(7) %p319  ret void320}321 322define bfloat @load_bf16(ptr addrspace(8) inreg %buf) {323; CHECK-LABEL: define bfloat @load_bf16(324; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {325; CHECK-NEXT:    [[RET:%.*]] = call bfloat @llvm.amdgcn.raw.ptr.buffer.load.bf16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)326; CHECK-NEXT:    ret bfloat [[RET]]327;328  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)329  %ret = load bfloat, ptr addrspace(7) %p330  ret bfloat %ret331}332 333define void @store_bf16(bfloat %data, ptr addrspace(8) inreg %buf) {334; CHECK-LABEL: define void @store_bf16(335; CHECK-SAME: bfloat [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {336; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)337; CHECK-NEXT:    ret void338;339  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)340  store bfloat %data, ptr addrspace(7) %p341  ret void342}343 344define <2 x half> @load_v2f16(ptr addrspace(8) inreg %buf) {345; CHECK-LABEL: define <2 x half> @load_v2f16(346; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {347; CHECK-NEXT:    [[RET:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)348; CHECK-NEXT:    ret <2 x half> [[RET]]349;350  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)351  %ret = load <2 x half>, ptr addrspace(7) %p352  ret <2 x half> %ret353}354 355define void @store_v2f16(<2 x half> %data, ptr addrspace(8) inreg %buf) {356; CHECK-LABEL: define void @store_v2f16(357; CHECK-SAME: <2 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {358; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)359; CHECK-NEXT:    ret void360;361  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)362  store <2 x half> %data, ptr addrspace(7) %p363  ret void364}365 366define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {367; CHECK-LABEL: define <4 x bfloat> @load_v4bf16(368; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {369; CHECK-NEXT:    [[RET:%.*]] = call <4 x bfloat> @llvm.amdgcn.raw.ptr.buffer.load.v4bf16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)370; CHECK-NEXT:    ret <4 x bfloat> [[RET]]371;372  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)373  %ret = load <4 x bfloat>, ptr addrspace(7) %p374  ret <4 x bfloat> %ret375}376 377define void @store_v4bf16(<4 x bfloat> %data, ptr addrspace(8) inreg %buf) {378; CHECK-LABEL: define void @store_v4bf16(379; CHECK-SAME: <4 x bfloat> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {380; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)381; CHECK-NEXT:    ret void382;383  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)384  store <4 x bfloat> %data, ptr addrspace(7) %p385  ret void386}387 388define <8 x half> @load_v8f16(ptr addrspace(8) inreg %buf) {389; CHECK-LABEL: define <8 x half> @load_v8f16(390; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {391; CHECK-NEXT:    [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)392; CHECK-NEXT:    ret <8 x half> [[RET]]393;394  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)395  %ret = load <8 x half>, ptr addrspace(7) %p396  ret <8 x half> %ret397}398 399define void @store_v8f16(<8 x half> %data, ptr addrspace(8) inreg %buf) {400; CHECK-LABEL: define void @store_v8f16(401; CHECK-SAME: <8 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {402; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v8f16(<8 x half> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)403; CHECK-NEXT:    ret void404;405  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)406  store <8 x half> %data, ptr addrspace(7) %p407  ret void408}409 410define float @load_f32(ptr addrspace(8) inreg %buf) {411; CHECK-LABEL: define float @load_f32(412; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {413; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)414; CHECK-NEXT:    ret float [[RET]]415;416  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)417  %ret = load float, ptr addrspace(7) %p418  ret float %ret419}420 421define void @store_f32(float %data, ptr addrspace(8) inreg %buf) {422; CHECK-LABEL: define void @store_f32(423; CHECK-SAME: float [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {424; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)425; CHECK-NEXT:    ret void426;427  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)428  store float %data, ptr addrspace(7) %p429  ret void430}431 432define <2 x float> @load_v2f32(ptr addrspace(8) inreg %buf) {433; CHECK-LABEL: define <2 x float> @load_v2f32(434; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {435; CHECK-NEXT:    [[RET:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)436; CHECK-NEXT:    ret <2 x float> [[RET]]437;438  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)439  %ret = load <2 x float>, ptr addrspace(7) %p440  ret <2 x float> %ret441}442 443define void @store_v2f32(<2 x float> %data, ptr addrspace(8) inreg %buf) {444; CHECK-LABEL: define void @store_v2f32(445; CHECK-SAME: <2 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {446; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)447; CHECK-NEXT:    ret void448;449  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)450  store <2 x float> %data, ptr addrspace(7) %p451  ret void452}453 454define <3 x float> @load_v3f32(ptr addrspace(8) inreg %buf) {455; CHECK-LABEL: define <3 x float> @load_v3f32(456; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {457; CHECK-NEXT:    [[RET:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)458; CHECK-NEXT:    ret <3 x float> [[RET]]459;460  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)461  %ret = load <3 x float>, ptr addrspace(7) %p462  ret <3 x float> %ret463}464 465define void @store_v3f32(<3 x float> %data, ptr addrspace(8) inreg %buf) {466; CHECK-LABEL: define void @store_v3f32(467; CHECK-SAME: <3 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {468; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)469; CHECK-NEXT:    ret void470;471  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)472  store <3 x float> %data, ptr addrspace(7) %p473  ret void474}475 476define <4 x float> @load_v4f32(ptr addrspace(8) inreg %buf) {477; CHECK-LABEL: define <4 x float> @load_v4f32(478; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {479; CHECK-NEXT:    [[RET:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)480; CHECK-NEXT:    ret <4 x float> [[RET]]481;482  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)483  %ret = load <4 x float>, ptr addrspace(7) %p484  ret <4 x float> %ret485}486 487define void @store_v4f32(<4 x float> %data, ptr addrspace(8) inreg %buf) {488; CHECK-LABEL: define void @store_v4f32(489; CHECK-SAME: <4 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {490; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)491; CHECK-NEXT:    ret void492;493  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)494  store <4 x float> %data, ptr addrspace(7) %p495  ret void496}497 498define ptr addrspace(0) @load_p0(ptr addrspace(8) inreg %buf) {499; CHECK-LABEL: define ptr @load_p0(500; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {501; CHECK-NEXT:    [[RET:%.*]] = call ptr @llvm.amdgcn.raw.ptr.buffer.load.p0(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)502; CHECK-NEXT:    ret ptr [[RET]]503;504  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)505  %ret = load ptr addrspace(0), ptr addrspace(7) %p506  ret ptr addrspace(0) %ret507}508 509define void @store_p0(ptr addrspace(0) %data, ptr addrspace(8) inreg %buf) {510; CHECK-LABEL: define void @store_p0(511; CHECK-SAME: ptr [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {512; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p0(ptr [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)513; CHECK-NEXT:    ret void514;515  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)516  store ptr addrspace(0) %data, ptr addrspace(7) %p517  ret void518}519 520define ptr addrspace(1) @load_p1(ptr addrspace(8) inreg %buf) {521; CHECK-LABEL: define ptr addrspace(1) @load_p1(522; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {523; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(1) @llvm.amdgcn.raw.ptr.buffer.load.p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)524; CHECK-NEXT:    ret ptr addrspace(1) [[RET]]525;526  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)527  %ret = load ptr addrspace(1), ptr addrspace(7) %p528  ret ptr addrspace(1) %ret529}530 531define void @store_p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf) {532; CHECK-LABEL: define void @store_p1(533; CHECK-SAME: ptr addrspace(1) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {534; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p1(ptr addrspace(1) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)535; CHECK-NEXT:    ret void536;537  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)538  store ptr addrspace(1) %data, ptr addrspace(7) %p539  ret void540}541 542define ptr addrspace(2) @load_p2(ptr addrspace(8) inreg %buf) {543; CHECK-LABEL: define ptr addrspace(2) @load_p2(544; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {545; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(2) @llvm.amdgcn.raw.ptr.buffer.load.p2(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)546; CHECK-NEXT:    ret ptr addrspace(2) [[RET]]547;548  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)549  %ret = load ptr addrspace(2), ptr addrspace(7) %p550  ret ptr addrspace(2) %ret551}552 553define void @store_p2(ptr addrspace(2) %data, ptr addrspace(8) inreg %buf) {554; CHECK-LABEL: define void @store_p2(555; CHECK-SAME: ptr addrspace(2) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {556; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p2(ptr addrspace(2) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)557; CHECK-NEXT:    ret void558;559  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)560  store ptr addrspace(2) %data, ptr addrspace(7) %p561  ret void562}563 564define ptr addrspace(3) @load_p3(ptr addrspace(8) inreg %buf) {565; CHECK-LABEL: define ptr addrspace(3) @load_p3(566; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {567; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(3) @llvm.amdgcn.raw.ptr.buffer.load.p3(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)568; CHECK-NEXT:    ret ptr addrspace(3) [[RET]]569;570  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)571  %ret = load ptr addrspace(3), ptr addrspace(7) %p572  ret ptr addrspace(3) %ret573}574 575define void @store_p3(ptr addrspace(3) %data, ptr addrspace(8) inreg %buf) {576; CHECK-LABEL: define void @store_p3(577; CHECK-SAME: ptr addrspace(3) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {578; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p3(ptr addrspace(3) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)579; CHECK-NEXT:    ret void580;581  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)582  store ptr addrspace(3) %data, ptr addrspace(7) %p583  ret void584}585 586define ptr addrspace(4) @load_p4(ptr addrspace(8) inreg %buf) {587; CHECK-LABEL: define ptr addrspace(4) @load_p4(588; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {589; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(4) @llvm.amdgcn.raw.ptr.buffer.load.p4(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)590; CHECK-NEXT:    ret ptr addrspace(4) [[RET]]591;592  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)593  %ret = load ptr addrspace(4), ptr addrspace(7) %p594  ret ptr addrspace(4) %ret595}596 597define void @store_p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf) {598; CHECK-LABEL: define void @store_p4(599; CHECK-SAME: ptr addrspace(4) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {600; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p4(ptr addrspace(4) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)601; CHECK-NEXT:    ret void602;603  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)604  store ptr addrspace(4) %data, ptr addrspace(7) %p605  ret void606}607 608define ptr addrspace(5) @load_p5(ptr addrspace(8) inreg %buf) {609; CHECK-LABEL: define ptr addrspace(5) @load_p5(610; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {611; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(5) @llvm.amdgcn.raw.ptr.buffer.load.p5(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)612; CHECK-NEXT:    ret ptr addrspace(5) [[RET]]613;614  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)615  %ret = load ptr addrspace(5), ptr addrspace(7) %p616  ret ptr addrspace(5) %ret617}618 619define void @store_p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf) {620; CHECK-LABEL: define void @store_p5(621; CHECK-SAME: ptr addrspace(5) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {622; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p5(ptr addrspace(5) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)623; CHECK-NEXT:    ret void624;625  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)626  store ptr addrspace(5) %data, ptr addrspace(7) %p627  ret void628}629 630define ptr addrspace(6) @load_p6(ptr addrspace(8) inreg %buf) {631; CHECK-LABEL: define ptr addrspace(6) @load_p6(632; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {633; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(6) @llvm.amdgcn.raw.ptr.buffer.load.p6(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)634; CHECK-NEXT:    ret ptr addrspace(6) [[RET]]635;636  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)637  %ret = load ptr addrspace(6), ptr addrspace(7) %p638  ret ptr addrspace(6) %ret639}640 641define void @store_p6(ptr addrspace(6) %data, ptr addrspace(8) inreg %buf) {642; CHECK-LABEL: define void @store_p6(643; CHECK-SAME: ptr addrspace(6) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {644; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p6(ptr addrspace(6) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)645; CHECK-NEXT:    ret void646;647  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)648  store ptr addrspace(6) %data, ptr addrspace(7) %p649  ret void650}651 652define ptr addrspace(8) @load_p8(ptr addrspace(8) inreg %buf) {653; CHECK-LABEL: define ptr addrspace(8) @load_p8(654; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {655; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(8) @llvm.amdgcn.raw.ptr.buffer.load.p8(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)656; CHECK-NEXT:    ret ptr addrspace(8) [[RET]]657;658  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)659  %ret = load ptr addrspace(8), ptr addrspace(7) %p660  ret ptr addrspace(8) %ret661}662 663define void @store_p8(ptr addrspace(8) %data, ptr addrspace(8) inreg %buf) {664; CHECK-LABEL: define void @store_p8(665; CHECK-SAME: ptr addrspace(8) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {666; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p8(ptr addrspace(8) [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)667; CHECK-NEXT:    ret void668;669  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)670  store ptr addrspace(8) %data, ptr addrspace(7) %p671  ret void672}673 674define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(8) inreg %buf) {675; CHECK-LABEL: define <2 x ptr addrspace(1)> @load_v2p1(676; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {677; CHECK-NEXT:    [[RET:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)678; CHECK-NEXT:    ret <2 x ptr addrspace(1)> [[RET]]679;680  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)681  %ret = load <2 x ptr addrspace(1)>, ptr addrspace(7) %p682  ret <2 x ptr addrspace(1)> %ret683}684 685define void @store_v2p1(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {686; CHECK-LABEL: define void @store_v2p1(687; CHECK-SAME: <2 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {688; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)689; CHECK-NEXT:    ret void690;691  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)692  store <2 x ptr addrspace(1)> %data, ptr addrspace(7) %p693  ret void694}695 696define <2 x ptr addrspace(5)> @load_v2p5(ptr addrspace(8) inreg %buf) {697; CHECK-LABEL: define <2 x ptr addrspace(5)> @load_v2p5(698; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {699; CHECK-NEXT:    [[RET:%.*]] = call <2 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v2p5(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)700; CHECK-NEXT:    ret <2 x ptr addrspace(5)> [[RET]]701;702  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)703  %ret = load <2 x ptr addrspace(5)>, ptr addrspace(7) %p704  ret <2 x ptr addrspace(5)> %ret705}706 707define void @store_v2p5(<2 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {708; CHECK-LABEL: define void @store_v2p5(709; CHECK-SAME: <2 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {710; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p5(<2 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)711; CHECK-NEXT:    ret void712;713  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)714  store <2 x ptr addrspace(5)> %data, ptr addrspace(7) %p715  ret void716}717 718define <3 x ptr addrspace(5)> @load_v3p5(ptr addrspace(8) inreg %buf) {719; CHECK-LABEL: define <3 x ptr addrspace(5)> @load_v3p5(720; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {721; CHECK-NEXT:    [[RET:%.*]] = call <3 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v3p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)722; CHECK-NEXT:    ret <3 x ptr addrspace(5)> [[RET]]723;724  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)725  %ret = load <3 x ptr addrspace(5)>, ptr addrspace(7) %p726  ret <3 x ptr addrspace(5)> %ret727}728 729define void @store_v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {730; CHECK-LABEL: define void @store_v3p5(731; CHECK-SAME: <3 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {732; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3p5(<3 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)733; CHECK-NEXT:    ret void734;735  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)736  store <3 x ptr addrspace(5)> %data, ptr addrspace(7) %p737  ret void738}739 740define <4 x ptr addrspace(5)> @load_v4p5(ptr addrspace(8) inreg %buf) {741; CHECK-LABEL: define <4 x ptr addrspace(5)> @load_v4p5(742; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {743; CHECK-NEXT:    [[RET:%.*]] = call <4 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v4p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)744; CHECK-NEXT:    ret <4 x ptr addrspace(5)> [[RET]]745;746  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)747  %ret = load <4 x ptr addrspace(5)>, ptr addrspace(7) %p748  ret <4 x ptr addrspace(5)> %ret749}750 751define void @store_v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {752; CHECK-LABEL: define void @store_v4p5(753; CHECK-SAME: <4 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {754; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4p5(<4 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)755; CHECK-NEXT:    ret void756;757  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)758  store <4 x ptr addrspace(5)> %data, ptr addrspace(7) %p759  ret void760}761 762;;; 3 words in a short type. These need to be bitcast to <3 x i32> to be supported.763 764define <6 x half> @load_v6f16(ptr addrspace(8) inreg %buf) {765; CHECK-LABEL: define <6 x half> @load_v6f16(766; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {767; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)768; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <6 x half>769; CHECK-NEXT:    ret <6 x half> [[RET]]770;771  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)772  %ret = load <6 x half>, ptr addrspace(7) %p773  ret <6 x half> %ret774}775 776define void @store_v6f16(<6 x half> %data, ptr addrspace(8) inreg %buf) {777; CHECK-LABEL: define void @store_v6f16(778; CHECK-SAME: <6 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {779; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <6 x half> [[DATA]] to <3 x i32>780; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)781; CHECK-NEXT:    ret void782;783  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)784  store <6 x half> %data, ptr addrspace(7) %p785  ret void786}787 788;;; Long types (32 bit elements). Must be split into multiple operations.789 790define <5 x float> @load_v5f32(ptr addrspace(8) inreg %buf) {791; CHECK-LABEL: define <5 x float> @load_v5f32(792; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {793; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)794; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>795; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x float> poison, <5 x float> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>796; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)797; CHECK-NEXT:    [[RET:%.*]] = insertelement <5 x float> [[RET_PARTS_0]], float [[RET_OFF_16]], i64 4798; CHECK-NEXT:    ret <5 x float> [[RET]]799;800  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)801  %ret = load <5 x float>, ptr addrspace(7) %p802  ret <5 x float> %ret803}804 805define void @store_v5f32(<5 x float> %data, ptr addrspace(8) inreg %buf) {806; CHECK-LABEL: define void @store_v5f32(807; CHECK-SAME: <5 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {808; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x float> [[DATA]], <5 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>809; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)810; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x float> [[DATA]], i64 4811; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)812; CHECK-NEXT:    ret void813;814  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)815  store <5 x float> %data, ptr addrspace(7) %p816  ret void817}818 819define <6 x float> @load_v6f32(ptr addrspace(8) inreg %buf) {820; CHECK-LABEL: define <6 x float> @load_v6f32(821; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {822; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)823; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>824; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <6 x float> poison, <6 x float> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>825; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)826; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x float> [[RET_OFF_16]], <2 x float> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>827; CHECK-NEXT:    [[RET:%.*]] = shufflevector <6 x float> [[RET_PARTS_0]], <6 x float> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>828; CHECK-NEXT:    ret <6 x float> [[RET]]829;830  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)831  %ret = load <6 x float>, ptr addrspace(7) %p832  ret <6 x float> %ret833}834 835define void @store_v6f32(<6 x float> %data, ptr addrspace(8) inreg %buf) {836; CHECK-LABEL: define void @store_v6f32(837; CHECK-SAME: <6 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {838; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>839; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)840; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <2 x i32> <i32 4, i32 5>841; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)842; CHECK-NEXT:    ret void843;844  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)845  store <6 x float> %data, ptr addrspace(7) %p846  ret void847}848 849define <7 x float> @load_v7f32(ptr addrspace(8) inreg %buf) {850; CHECK-LABEL: define <7 x float> @load_v7f32(851; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {852; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)853; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison>854; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <7 x float> poison, <7 x float> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 4, i32 5, i32 6>855; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)856; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <3 x float> [[RET_OFF_16]], <3 x float> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>857; CHECK-NEXT:    [[RET:%.*]] = shufflevector <7 x float> [[RET_PARTS_0]], <7 x float> [[RET_EXT_4]], <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 8, i32 9>858; CHECK-NEXT:    ret <7 x float> [[RET]]859;860  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)861  %ret = load <7 x float>, ptr addrspace(7) %p862  ret <7 x float> %ret863}864 865define void @store_v7f32(<7 x float> %data, ptr addrspace(8) inreg %buf) {866; CHECK-LABEL: define void @store_v7f32(867; CHECK-SAME: <7 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {868; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>869; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)870; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <3 x i32> <i32 4, i32 5, i32 6>871; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)872; CHECK-NEXT:    ret void873;874  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)875  store <7 x float> %data, ptr addrspace(7) %p876  ret void877}878 879define <8 x float> @load_v8f32(ptr addrspace(8) inreg %buf) {880; CHECK-LABEL: define <8 x float> @load_v8f32(881; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {882; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)883; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>884; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <8 x float> poison, <8 x float> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>885; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)886; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x float> [[RET_OFF_16]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>887; CHECK-NEXT:    [[RET:%.*]] = shufflevector <8 x float> [[RET_PARTS_0]], <8 x float> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>888; CHECK-NEXT:    ret <8 x float> [[RET]]889;890  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)891  %ret = load <8 x float>, ptr addrspace(7) %p892  ret <8 x float> %ret893}894 895define void @store_v8f32(<8 x float> %data, ptr addrspace(8) inreg %buf) {896; CHECK-LABEL: define void @store_v8f32(897; CHECK-SAME: <8 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {898; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>899; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)900; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>901; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)902; CHECK-NEXT:    ret void903;904  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)905  store <8 x float> %data, ptr addrspace(7) %p906  ret void907}908 909define <10 x float> @load_v10f32(ptr addrspace(8) inreg %buf) {910; CHECK-LABEL: define <10 x float> @load_v10f32(911; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {912; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0)913; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>914; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <10 x float> poison, <10 x float> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>915; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)916; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x float> [[RET_OFF_16]], <4 x float> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>917; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <10 x float> [[RET_PARTS_0]], <10 x float> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>918; CHECK-NEXT:    [[RET_OFF_32:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0)919; CHECK-NEXT:    [[RET_EXT_8:%.*]] = shufflevector <2 x float> [[RET_OFF_32]], <2 x float> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>920; CHECK-NEXT:    [[RET:%.*]] = shufflevector <10 x float> [[RET_PARTS_4]], <10 x float> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>921; CHECK-NEXT:    ret <10 x float> [[RET]]922;923  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)924  %ret = load <10 x float>, ptr addrspace(7) %p925  ret <10 x float> %ret926}927 928define void @store_v10f32(<10 x float> %data, ptr addrspace(8) inreg %buf) {929; CHECK-LABEL: define void @store_v10f32(930; CHECK-SAME: <10 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {931; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>932; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0)933; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>934; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)935; CHECK-NEXT:    [[DATA_SLICE_8:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <2 x i32> <i32 8, i32 9>936; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_8]], ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0)937; CHECK-NEXT:    ret void938;939  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)940  store <10 x float> %data, ptr addrspace(7) %p941  ret void942}943 944define <6 x i32> @load_v6i32(ptr addrspace(8) inreg %buf) {945; CHECK-LABEL: define <6 x i32> @load_v6i32(946; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {947; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)948; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>949; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <6 x i32> poison, <6 x i32> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>950; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)951; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>952; CHECK-NEXT:    [[RET:%.*]] = shufflevector <6 x i32> [[RET_PARTS_0]], <6 x i32> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>953; CHECK-NEXT:    ret <6 x i32> [[RET]]954;955  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)956  %ret = load <6 x i32>, ptr addrspace(7) %p957  ret <6 x i32> %ret958}959 960define void @store_v6i32(<6 x i32> %data, ptr addrspace(8) inreg %buf) {961; CHECK-LABEL: define void @store_v6i32(962; CHECK-SAME: <6 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {963; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>964; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)965; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>966; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)967; CHECK-NEXT:    ret void968;969  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)970  store <6 x i32> %data, ptr addrspace(7) %p971  ret void972}973 974define <4 x ptr addrspace(1)> @load_v4p1(ptr addrspace(8) inreg %buf) {975; CHECK-LABEL: define <4 x ptr addrspace(1)> @load_v4p1(976; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {977; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)978; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x ptr addrspace(1)> [[RET_OFF_0]], <2 x ptr addrspace(1)> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>979; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <4 x ptr addrspace(1)> poison, <4 x ptr addrspace(1)> [[RET_EXT_0]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>980; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)981; CHECK-NEXT:    [[RET_EXT_2:%.*]] = shufflevector <2 x ptr addrspace(1)> [[RET_OFF_16]], <2 x ptr addrspace(1)> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>982; CHECK-NEXT:    [[RET:%.*]] = shufflevector <4 x ptr addrspace(1)> [[RET_PARTS_0]], <4 x ptr addrspace(1)> [[RET_EXT_2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>983; CHECK-NEXT:    ret <4 x ptr addrspace(1)> [[RET]]984;985  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)986  %ret = load <4 x ptr addrspace(1)>, ptr addrspace(7) %p987  ret <4 x ptr addrspace(1)> %ret988}989 990define void @store_v4p1(<4 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {991; CHECK-LABEL: define void @store_v4p1(992; CHECK-SAME: <4 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {993; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 0, i32 1>994; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)995; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 2, i32 3>996; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_2]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)997; CHECK-NEXT:    ret void998;999  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1000  store <4 x ptr addrspace(1)> %data, ptr addrspace(7) %p1001  ret void1002}1003 1004;;; Uneven types with 16-bit elements. Require splitting into multiple operations.1005 1006define <1 x i16> @load_v1i16(ptr addrspace(8) inreg %buf) {1007; CHECK-LABEL: define <1 x i16> @load_v1i16(1008; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1009; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1010; CHECK-NEXT:    [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <1 x i16>1011; CHECK-NEXT:    ret <1 x i16> [[RET]]1012;1013  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1014  %ret = load <1 x i16>, ptr addrspace(7) %p1015  ret <1 x i16> %ret1016}1017 1018define void @store_v1i16(<1 x i16> %data, ptr addrspace(8) inreg %buf) {1019; CHECK-LABEL: define void @store_v1i16(1020; CHECK-SAME: <1 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1021; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <1 x i16> [[DATA]] to i161022; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_STORABLE]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1023; CHECK-NEXT:    ret void1024;1025  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1026  store <1 x i16> %data, ptr addrspace(7) %p1027  ret void1028}1029 1030define <3 x i16> @load_v3i16(ptr addrspace(8) inreg %buf) {1031; CHECK-LABEL: define <3 x i16> @load_v3i16(1032; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1033; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1034; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>1035; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>1036; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1037; CHECK-NEXT:    [[RET:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 21038; CHECK-NEXT:    ret <3 x i16> [[RET]]1039;1040  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1041  %ret = load <3 x i16>, ptr addrspace(7) %p1042  ret <3 x i16> %ret1043}1044 1045define void @store_v3i16(<3 x i16> %data, ptr addrspace(8) inreg %buf) {1046; CHECK-LABEL: define void @store_v3i16(1047; CHECK-SAME: <3 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1048; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>1049; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1050; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA]], i64 21051; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1052; CHECK-NEXT:    ret void1053;1054  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1055  store <3 x i16> %data, ptr addrspace(7) %p1056  ret void1057}1058 1059define <5 x i16> @load_v5i16(ptr addrspace(8) inreg %buf) {1060; CHECK-LABEL: define <5 x i16> @load_v5i16(1061; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1062; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1063; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i16> [[RET_OFF_0]], <4 x i16> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>1064; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i16> poison, <5 x i16> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>1065; CHECK-NEXT:    [[RET_OFF_8:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)1066; CHECK-NEXT:    [[RET:%.*]] = insertelement <5 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_8]], i64 41067; CHECK-NEXT:    ret <5 x i16> [[RET]]1068;1069  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1070  %ret = load <5 x i16>, ptr addrspace(7) %p1071  ret <5 x i16> %ret1072}1073 1074define void @store_v5i16(<5 x i16> %data, ptr addrspace(8) inreg %buf) {1075; CHECK-LABEL: define void @store_v5i16(1076; CHECK-SAME: <5 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1077; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i16> [[DATA]], <5 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1078; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1079; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i16> [[DATA]], i64 41080; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)1081; CHECK-NEXT:    ret void1082;1083  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1084  store <5 x i16> %data, ptr addrspace(7) %p1085  ret void1086}1087 1088define <6 x i16> @load_v6i16(ptr addrspace(8) inreg %buf) {1089; CHECK-LABEL: define <6 x i16> @load_v6i16(1090; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1091; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1092; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <6 x i16>1093; CHECK-NEXT:    ret <6 x i16> [[RET]]1094;1095  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1096  %ret = load <6 x i16>, ptr addrspace(7) %p1097  ret <6 x i16> %ret1098}1099 1100define void @store_v6i16(<6 x i16> %data, ptr addrspace(8) inreg %buf) {1101; CHECK-LABEL: define void @store_v6i16(1102; CHECK-SAME: <6 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1103; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <6 x i16> [[DATA]] to <3 x i32>1104; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1105; CHECK-NEXT:    ret void1106;1107  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1108  store <6 x i16> %data, ptr addrspace(7) %p1109  ret void1110}1111 1112define <7 x i16> @load_v7i16(ptr addrspace(8) inreg %buf) {1113; CHECK-LABEL: define <7 x i16> @load_v7i16(1114; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1115; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1116; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast <3 x i32> [[RET_OFF_0]] to <6 x i16>1117; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <6 x i16> [[RET_OFF_0_FROM_LOADABLE]], <6 x i16> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison>1118; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <7 x i16> poison, <7 x i16> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 6>1119; CHECK-NEXT:    [[RET_OFF_12:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0)1120; CHECK-NEXT:    [[RET:%.*]] = insertelement <7 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_12]], i64 61121; CHECK-NEXT:    ret <7 x i16> [[RET]]1122;1123  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1124  %ret = load <7 x i16>, ptr addrspace(7) %p1125  ret <7 x i16> %ret1126}1127 1128define void @store_v7i16(<7 x i16> %data, ptr addrspace(8) inreg %buf) {1129; CHECK-LABEL: define void @store_v7i16(1130; CHECK-SAME: <7 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1131; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <7 x i16> [[DATA]], <7 x i16> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>1132; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <6 x i16> [[DATA_SLICE_0]] to <3 x i32>1133; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1134; CHECK-NEXT:    [[DATA_SLICE_6:%.*]] = extractelement <7 x i16> [[DATA]], i64 61135; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0)1136; CHECK-NEXT:    ret void1137;1138  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1139  store <7 x i16> %data, ptr addrspace(7) %p1140  ret void1141}1142 1143define <9 x i16> @load_v9i16(ptr addrspace(8) inreg %buf) {1144; CHECK-LABEL: define <9 x i16> @load_v9i16(1145; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1146; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)1147; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <8 x i16> [[RET_OFF_0]], <8 x i16> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>1148; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <9 x i16> poison, <9 x i16> [[RET_EXT_0]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>1149; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)1150; CHECK-NEXT:    [[RET:%.*]] = insertelement <9 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_16]], i64 81151; CHECK-NEXT:    ret <9 x i16> [[RET]]1152;1153  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1154  %ret = load <9 x i16>, ptr addrspace(7) %p1155  ret <9 x i16> %ret1156}1157 1158define void @store_v9i16(<9 x i16> %data, ptr addrspace(8) inreg %buf) {1159; CHECK-LABEL: define void @store_v9i16(1160; CHECK-SAME: <9 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1161; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <9 x i16> [[DATA]], <9 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1162; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)1163; CHECK-NEXT:    [[DATA_SLICE_8:%.*]] = extractelement <9 x i16> [[DATA]], i64 81164; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)1165; CHECK-NEXT:    ret void1166;1167  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1168  store <9 x i16> %data, ptr addrspace(7) %p1169  ret void1170}1171 1172;;; Byte vectors. Need to be1173;;; - Split into multiple operations1174;;; - Bitcast if they have a natively supported width1175 1176define <1 x i8> @load_v1i8(ptr addrspace(8) inreg %buf) {1177; CHECK-LABEL: define <1 x i8> @load_v1i8(1178; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1179; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1180; CHECK-NEXT:    [[RET:%.*]] = bitcast i8 [[RET_LOADABLE]] to <1 x i8>1181; CHECK-NEXT:    ret <1 x i8> [[RET]]1182;1183  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1184  %ret = load <1 x i8>, ptr addrspace(7) %p1185  ret <1 x i8> %ret1186}1187 1188define void @store_v1i8(<1 x i8> %data, ptr addrspace(8) inreg %buf) {1189; CHECK-LABEL: define void @store_v1i8(1190; CHECK-SAME: <1 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1191; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <1 x i8> [[DATA]] to i81192; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1193; CHECK-NEXT:    ret void1194;1195  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1196  store <1 x i8> %data, ptr addrspace(7) %p1197  ret void1198}1199 1200define <2 x i8> @load_v2i8(ptr addrspace(8) inreg %buf) {1201; CHECK-LABEL: define <2 x i8> @load_v2i8(1202; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1203; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1204; CHECK-NEXT:    [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>1205; CHECK-NEXT:    ret <2 x i8> [[RET]]1206;1207  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1208  %ret = load <2 x i8>, ptr addrspace(7) %p1209  ret <2 x i8> %ret1210}1211 1212define void @store_v2i8(<2 x i8> %data, ptr addrspace(8) inreg %buf) {1213; CHECK-LABEL: define void @store_v2i8(1214; CHECK-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1215; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i161216; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1217; CHECK-NEXT:    ret void1218;1219  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1220  store <2 x i8> %data, ptr addrspace(7) %p1221  ret void1222}1223 1224define <3 x i8> @load_v3i8(ptr addrspace(8) inreg %buf) {1225; CHECK-LABEL: define <3 x i8> @load_v3i8(1226; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1227; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1228; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i16 [[RET_OFF_0]] to <2 x i8>1229; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i8> [[RET_OFF_0_FROM_LOADABLE]], <2 x i8> poison, <3 x i32> <i32 0, i32 1, i32 poison>1230; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i8> poison, <3 x i8> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>1231; CHECK-NEXT:    [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)1232; CHECK-NEXT:    [[RET:%.*]] = insertelement <3 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_2]], i64 21233; CHECK-NEXT:    ret <3 x i8> [[RET]]1234;1235  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1236  %ret = load <3 x i8>, ptr addrspace(7) %p1237  ret <3 x i8> %ret1238}1239 1240define void @store_v3i8(<3 x i8> %data, ptr addrspace(8) inreg %buf) {1241; CHECK-LABEL: define void @store_v3i8(1242; CHECK-SAME: <3 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1243; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i8> [[DATA]], <3 x i8> poison, <2 x i32> <i32 0, i32 1>1244; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_0]] to i161245; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1246; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i8> [[DATA]], i64 21247; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)1248; CHECK-NEXT:    ret void1249;1250  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1251  store <3 x i8> %data, ptr addrspace(7) %p1252  ret void1253}1254 1255define <4 x i8> @load_v4i8(ptr addrspace(8) inreg %buf) {1256; CHECK-LABEL: define <4 x i8> @load_v4i8(1257; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1258; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1259; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>1260; CHECK-NEXT:    ret <4 x i8> [[RET]]1261;1262  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1263  %ret = load <4 x i8>, ptr addrspace(7) %p1264  ret <4 x i8> %ret1265}1266 1267define void @store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {1268; CHECK-LABEL: define void @store_v4i8(1269; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1270; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i321271; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1272; CHECK-NEXT:    ret void1273;1274  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1275  store <4 x i8> %data, ptr addrspace(7) %p1276  ret void1277}1278 1279define <5 x i8> @load_v5i8(ptr addrspace(8) inreg %buf) {1280; CHECK-LABEL: define <5 x i8> @load_v5i8(1281; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1282; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1283; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>1284; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>1285; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i8> poison, <5 x i8> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>1286; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1287; CHECK-NEXT:    [[RET:%.*]] = insertelement <5 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_4]], i64 41288; CHECK-NEXT:    ret <5 x i8> [[RET]]1289;1290  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1291  %ret = load <5 x i8>, ptr addrspace(7) %p1292  ret <5 x i8> %ret1293}1294 1295define void @store_v5i8(<5 x i8> %data, ptr addrspace(8) inreg %buf) {1296; CHECK-LABEL: define void @store_v5i8(1297; CHECK-SAME: <5 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1298; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1299; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i321300; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1301; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA]], i64 41302; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1303; CHECK-NEXT:    ret void1304;1305  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1306  store <5 x i8> %data, ptr addrspace(7) %p1307  ret void1308}1309 1310define <6 x i8> @load_v6i8(ptr addrspace(8) inreg %buf) {1311; CHECK-LABEL: define <6 x i8> @load_v6i8(1312; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1313; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1314; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>1315; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>1316; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1317; CHECK-NEXT:    [[RET_SLICE_2:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 21318; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i16> [[RET_SLICE_2]] to <6 x i8>1319; CHECK-NEXT:    ret <6 x i8> [[RET]]1320;1321  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1322  %ret = load <6 x i8>, ptr addrspace(7) %p1323  ret <6 x i8> %ret1324}1325 1326define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {1327; CHECK-LABEL: define void @store_v6i8(1328; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1329; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>1330; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>1331; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1332; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 21333; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1334; CHECK-NEXT:    ret void1335;1336  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1337  store <6 x i8> %data, ptr addrspace(7) %p1338  ret void1339}1340 1341define <7 x i8> @load_v7i8(ptr addrspace(8) inreg %buf) {1342; CHECK-LABEL: define <7 x i8> @load_v7i8(1343; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1344; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1345; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>1346; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison>1347; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <7 x i8> poison, <7 x i8> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 4, i32 5, i32 6>1348; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1349; CHECK-NEXT:    [[RET_OFF_4_FROM_LOADABLE:%.*]] = bitcast i16 [[RET_OFF_4]] to <2 x i8>1350; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x i8> [[RET_OFF_4_FROM_LOADABLE]], <2 x i8> poison, <7 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>1351; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <7 x i8> [[RET_PARTS_0]], <7 x i8> [[RET_EXT_4]], <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 8, i32 6>1352; CHECK-NEXT:    [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0)1353; CHECK-NEXT:    [[RET:%.*]] = insertelement <7 x i8> [[RET_PARTS_4]], i8 [[RET_OFF_6]], i64 61354; CHECK-NEXT:    ret <7 x i8> [[RET]]1355;1356  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1357  %ret = load <7 x i8>, ptr addrspace(7) %p1358  ret <7 x i8> %ret1359}1360 1361define void @store_v7i8(<7 x i8> %data, ptr addrspace(8) inreg %buf) {1362; CHECK-LABEL: define void @store_v7i8(1363; CHECK-SAME: <7 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1364; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1365; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i321366; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1367; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <2 x i32> <i32 4, i32 5>1368; CHECK-NEXT:    [[DATA_SLICE_4_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_4]] to i161369; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1370; CHECK-NEXT:    [[DATA_SLICE_6:%.*]] = extractelement <7 x i8> [[DATA]], i64 61371; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0)1372; CHECK-NEXT:    ret void1373;1374  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1375  store <7 x i8> %data, ptr addrspace(7) %p1376  ret void1377}1378 1379define <8 x i8> @load_v8i8(ptr addrspace(8) inreg %buf) {1380; CHECK-LABEL: define <8 x i8> @load_v8i8(1381; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1382; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1383; CHECK-NEXT:    [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <8 x i8>1384; CHECK-NEXT:    ret <8 x i8> [[RET]]1385;1386  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1387  %ret = load <8 x i8>, ptr addrspace(7) %p1388  ret <8 x i8> %ret1389}1390 1391define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {1392; CHECK-LABEL: define void @store_v8i8(1393; CHECK-SAME: <8 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1394; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <8 x i8> [[DATA]] to <2 x i32>1395; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1396; CHECK-NEXT:    ret void1397;1398  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1399  store <8 x i8> %data, ptr addrspace(7) %p1400  ret void1401}1402 1403define <12 x i8> @load_v12i8(ptr addrspace(8) inreg %buf) {1404; CHECK-LABEL: define <12 x i8> @load_v12i8(1405; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1406; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1407; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <12 x i8>1408; CHECK-NEXT:    ret <12 x i8> [[RET]]1409;1410  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1411  %ret = load <12 x i8>, ptr addrspace(7) %p1412  ret <12 x i8> %ret1413}1414 1415define void @store_v12i8(<12 x i8> %data, ptr addrspace(8) inreg %buf) {1416; CHECK-LABEL: define void @store_v12i8(1417; CHECK-SAME: <12 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1418; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <12 x i8> [[DATA]] to <3 x i32>1419; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1420; CHECK-NEXT:    ret void1421;1422  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1423  store <12 x i8> %data, ptr addrspace(7) %p1424  ret void1425}1426 1427define <16 x i8> @load_v16i8(ptr addrspace(8) inreg %buf) {1428; CHECK-LABEL: define <16 x i8> @load_v16i8(1429; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1430; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1431; CHECK-NEXT:    [[RET:%.*]] = bitcast <4 x i32> [[RET_LOADABLE]] to <16 x i8>1432; CHECK-NEXT:    ret <16 x i8> [[RET]]1433;1434  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1435  %ret = load <16 x i8>, ptr addrspace(7) %p1436  ret <16 x i8> %ret1437}1438 1439define void @store_v16i8(<16 x i8> %data, ptr addrspace(8) inreg %buf) {1440; CHECK-LABEL: define void @store_v16i8(1441; CHECK-SAME: <16 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1442; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <16 x i8> [[DATA]] to <4 x i32>1443; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)1444; CHECK-NEXT:    ret void1445;1446  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1447  store <16 x i8> %data, ptr addrspace(7) %p1448  ret void1449}1450 1451define <32 x i8> @load_v32i8(ptr addrspace(8) inreg %buf) {1452; CHECK-LABEL: define <32 x i8> @load_v32i8(1453; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1454; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)1455; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>1456; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>1457; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)1458; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>1459; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>1460; CHECK-NEXT:    [[RET:%.*]] = bitcast <8 x i32> [[RET_PARTS_4]] to <32 x i8>1461; CHECK-NEXT:    ret <32 x i8> [[RET]]1462;1463  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1464  %ret = load <32 x i8>, ptr addrspace(7) %p1465  ret <32 x i8> %ret1466}1467 1468define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {1469; CHECK-LABEL: define void @store_v32i8(1470; CHECK-SAME: <32 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1471; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <32 x i8> [[DATA]] to <8 x i32>1472; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1473; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)1474; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1475; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)1476; CHECK-NEXT:    ret void1477;1478  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1479  store <32 x i8> %data, ptr addrspace(7) %p1480  ret void1481}1482 1483;;; Arrays. Need to become vectors.1484 1485define [1 x i32] @load_a1i32(ptr addrspace(8) inreg %buf) {1486; CHECK-LABEL: define [1 x i32] @load_a1i32(1487; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1488; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1489; CHECK-NEXT:    [[RET_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_LOADABLE]] to <1 x i32>1490; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <1 x i32> [[RET_FROM_LOADABLE]], i64 01491; CHECK-NEXT:    [[RET:%.*]] = insertvalue [1 x i32] poison, i32 [[RET_ELEM_0]], 01492; CHECK-NEXT:    ret [1 x i32] [[RET]]1493;1494  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1495  %ret = load [1 x i32], ptr addrspace(7) %p1496  ret [1 x i32] %ret1497}1498 1499define void @store_a1i32([1 x i32] %data, ptr addrspace(8) inreg %buf) {1500; CHECK-LABEL: define void @store_a1i32(1501; CHECK-SAME: [1 x i32] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1502; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [1 x i32] [[DATA]], 01503; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <1 x i32> poison, i32 [[DATA_ELEM_0]], i64 01504; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA_AS_VEC_0]] to i321505; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1506; CHECK-NEXT:    ret void1507;1508  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1509  store [1 x i32] %data, ptr addrspace(7) %p1510  ret void1511}1512 1513define [2 x i32] @load_a2i32(ptr addrspace(8) inreg %buf) {1514; CHECK-LABEL: define [2 x i32] @load_a2i32(1515; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1516; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1517; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 01518; CHECK-NEXT:    [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET_ELEM_0]], 01519; CHECK-NEXT:    [[RET_ELEM_1:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 11520; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x i32] [[RET_AS_ARRAY_0]], i32 [[RET_ELEM_1]], 11521; CHECK-NEXT:    ret [2 x i32] [[RET]]1522;1523  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1524  %ret = load [2 x i32], ptr addrspace(7) %p1525  ret [2 x i32] %ret1526}1527 1528define void @store_a2i32([2 x i32] %data, ptr addrspace(8) inreg %buf) {1529; CHECK-LABEL: define void @store_a2i32(1530; CHECK-SAME: [2 x i32] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1531; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA]], 01532; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA_ELEM_0]], i64 01533; CHECK-NEXT:    [[DATA_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA]], 11534; CHECK-NEXT:    [[DATA_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA_AS_VEC_0]], i32 [[DATA_ELEM_1]], i64 11535; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1536; CHECK-NEXT:    ret void1537;1538  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1539  store [2 x i32] %data, ptr addrspace(7) %p1540  ret void1541}1542 1543define [2 x half] @load_a2f16(ptr addrspace(8) inreg %buf) {1544; CHECK-LABEL: define [2 x half] @load_a2f16(1545; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1546; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1547; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <2 x half> [[RET_LOADABLE]], i64 01548; CHECK-NEXT:    [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x half] poison, half [[RET_ELEM_0]], 01549; CHECK-NEXT:    [[RET_ELEM_1:%.*]] = extractelement <2 x half> [[RET_LOADABLE]], i64 11550; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x half] [[RET_AS_ARRAY_0]], half [[RET_ELEM_1]], 11551; CHECK-NEXT:    ret [2 x half] [[RET]]1552;1553  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1554  %ret = load [2 x half], ptr addrspace(7) %p1555  ret [2 x half] %ret1556}1557 1558define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {1559; CHECK-LABEL: define void @store_a2f16(1560; CHECK-SAME: [2 x half] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1561; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [2 x half] [[DATA]], 01562; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <2 x half> poison, half [[DATA_ELEM_0]], i64 01563; CHECK-NEXT:    [[DATA_ELEM_1:%.*]] = extractvalue [2 x half] [[DATA]], 11564; CHECK-NEXT:    [[DATA_AS_VEC_1:%.*]] = insertelement <2 x half> [[DATA_AS_VEC_0]], half [[DATA_ELEM_1]], i64 11565; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA_AS_VEC_1]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1566; CHECK-NEXT:    ret void1567;1568  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1569  store [2 x half] %data, ptr addrspace(7) %p1570  ret void1571}1572 1573define [2 x ptr addrspace(1)] @load_a2p1(ptr addrspace(8) inreg %buf) {1574; CHECK-LABEL: define [2 x ptr addrspace(1)] @load_a2p1(1575; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1576; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1577; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 01578; CHECK-NEXT:    [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x ptr addrspace(1)] poison, ptr addrspace(1) [[RET_ELEM_0]], 01579; CHECK-NEXT:    [[RET_ELEM_1:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 11580; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x ptr addrspace(1)] [[RET_AS_ARRAY_0]], ptr addrspace(1) [[RET_ELEM_1]], 11581; CHECK-NEXT:    ret [2 x ptr addrspace(1)] [[RET]]1582;1583  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1584  %ret = load [2 x ptr addrspace(1)], ptr addrspace(7) %p1585  ret [2 x ptr addrspace(1)] %ret1586}1587 1588define void @store_a2p1([2 x ptr addrspace(1)] %data, ptr addrspace(8) inreg %buf) {1589; CHECK-LABEL: define void @store_a2p1(1590; CHECK-SAME: [2 x ptr addrspace(1)] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1591; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [2 x ptr addrspace(1)] [[DATA]], 01592; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <2 x ptr addrspace(1)> poison, ptr addrspace(1) [[DATA_ELEM_0]], i64 01593; CHECK-NEXT:    [[DATA_ELEM_1:%.*]] = extractvalue [2 x ptr addrspace(1)] [[DATA]], 11594; CHECK-NEXT:    [[DATA_AS_VEC_1:%.*]] = insertelement <2 x ptr addrspace(1)> [[DATA_AS_VEC_0]], ptr addrspace(1) [[DATA_ELEM_1]], i64 11595; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_AS_VEC_1]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1596; CHECK-NEXT:    ret void1597;1598  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1599  store [2 x ptr addrspace(1)] %data, ptr addrspace(7) %p1600  ret void1601}1602 1603;;; Scalars of atypical width. Need to be cast to vectors and split.1604 1605define i40 @load_i40(ptr addrspace(8) inreg %buf) {1606; CHECK-LABEL: define i40 @load_i40(1607; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1608; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1609; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>1610; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>1611; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i8> poison, <5 x i8> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>1612; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1613; CHECK-NEXT:    [[RET_SLICE_4:%.*]] = insertelement <5 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_4]], i64 41614; CHECK-NEXT:    [[RET:%.*]] = bitcast <5 x i8> [[RET_SLICE_4]] to i401615; CHECK-NEXT:    ret i40 [[RET]]1616;1617  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1618  %ret = load i40, ptr addrspace(7) %p1619  ret i40 %ret1620}1621 1622define void @store_i40(i40 %data, ptr addrspace(8) inreg %buf) {1623; CHECK-LABEL: define void @store_i40(1624; CHECK-SAME: i40 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1625; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i40 [[DATA]] to <5 x i8>1626; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA_LEGAL]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1627; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i321628; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1629; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA_LEGAL]], i64 41630; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)1631; CHECK-NEXT:    ret void1632;1633  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1634  store i40 %data, ptr addrspace(7) %p1635  ret void1636}1637 1638define i96 @load_i96(ptr addrspace(8) inreg %buf) {1639; CHECK-LABEL: define i96 @load_i96(1640; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1641; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1642; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to i961643; CHECK-NEXT:    ret i96 [[RET]]1644;1645  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1646  %ret = load i96, ptr addrspace(7) %p1647  ret i96 %ret1648}1649 1650define void @store_i96(i96 %data, ptr addrspace(8) inreg %buf) {1651; CHECK-LABEL: define void @store_i96(1652; CHECK-SAME: i96 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1653; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i96 [[DATA]] to <3 x i32>1654; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1655; CHECK-NEXT:    ret void1656;1657  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1658  store i96 %data, ptr addrspace(7) %p1659  ret void1660}1661 1662define i160 @load_i160(ptr addrspace(8) inreg %buf) {1663; CHECK-LABEL: define i160 @load_i160(1664; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1665; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1666; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>1667; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i32> poison, <5 x i32> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>1668; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)1669; CHECK-NEXT:    [[RET_SLICE_4:%.*]] = insertelement <5 x i32> [[RET_PARTS_0]], i32 [[RET_OFF_16]], i64 41670; CHECK-NEXT:    [[RET:%.*]] = bitcast <5 x i32> [[RET_SLICE_4]] to i1601671; CHECK-NEXT:    ret i160 [[RET]]1672;1673  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1674  %ret = load i160, ptr addrspace(7) %p1675  ret i160 %ret1676}1677 1678define void @store_i160(i160 %data, ptr addrspace(8) inreg %buf) {1679; CHECK-LABEL: define void @store_i160(1680; CHECK-SAME: i160 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1681; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i160 [[DATA]] to <5 x i32>1682; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i32> [[DATA_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1683; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1684; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i32> [[DATA_LEGAL]], i64 41685; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)1686; CHECK-NEXT:    ret void1687;1688  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1689  store i160 %data, ptr addrspace(7) %p1690  ret void1691}1692 1693define i256 @load_i256(ptr addrspace(8) inreg %buf) {1694; CHECK-LABEL: define i256 @load_i256(1695; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1696; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1697; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>1698; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>1699; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)1700; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>1701; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>1702; CHECK-NEXT:    [[RET:%.*]] = bitcast <8 x i32> [[RET_PARTS_4]] to i2561703; CHECK-NEXT:    ret i256 [[RET]]1704;1705  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1706  %ret = load i256, ptr addrspace(7) %p1707  ret i256 %ret1708}1709 1710define void @store_i256(i256 %data, ptr addrspace(8) inreg %buf) {1711; CHECK-LABEL: define void @store_i256(1712; CHECK-SAME: i256 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1713; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i256 [[DATA]] to <8 x i32>1714; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1715; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)1716; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1717; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)1718; CHECK-NEXT:    ret void1719;1720  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1721  store i256 %data, ptr addrspace(7) %p1722  ret void1723}1724 1725;;; Non-byte-sized scalars. Require zero-extension.1726 1727define i7 @load_i7(ptr addrspace(8) inreg %buf) {1728; CHECK-LABEL: define i7 @load_i7(1729; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1730; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1731; CHECK-NEXT:    [[RET:%.*]] = trunc i8 [[RET_LOADABLE]] to i71732; CHECK-NEXT:    ret i7 [[RET]]1733;1734  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1735  %ret = load i7, ptr addrspace(7) %p1736  ret i7 %ret1737}1738 1739define void @store_i7(i7 %data, ptr addrspace(8) inreg %buf) {1740; CHECK-LABEL: define void @store_i7(1741; CHECK-SAME: i7 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1742; CHECK-NEXT:    [[DATA_ZEXT:%.*]] = zext i7 [[DATA]] to i81743; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1744; CHECK-NEXT:    ret void1745;1746  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1747  store i7 %data, ptr addrspace(7) %p1748  ret void1749}1750 1751define i4 @load_i4(ptr addrspace(8) inreg %buf) {1752; CHECK-LABEL: define i4 @load_i4(1753; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1754; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1755; CHECK-NEXT:    [[RET:%.*]] = trunc i8 [[RET_LOADABLE]] to i41756; CHECK-NEXT:    ret i4 [[RET]]1757;1758  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1759  %ret = load i4, ptr addrspace(7) %p1760  ret i4 %ret1761}1762 1763define void @store_i4(i4 %data, ptr addrspace(8) inreg %buf) {1764; CHECK-LABEL: define void @store_i4(1765; CHECK-SAME: i4 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1766; CHECK-NEXT:    [[DATA_ZEXT:%.*]] = zext i4 [[DATA]] to i81767; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1768; CHECK-NEXT:    ret void1769;1770  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1771  store i4 %data, ptr addrspace(7) %p1772  ret void1773}1774 1775;;; Byte-sized vectors of i4. Require casts.1776 1777define <2 x i4> @load_v2i4(ptr addrspace(8) inreg %buf) {1778; CHECK-LABEL: define <2 x i4> @load_v2i4(1779; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1780; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1781; CHECK-NEXT:    [[RET:%.*]] = bitcast i8 [[RET_LOADABLE]] to <2 x i4>1782; CHECK-NEXT:    ret <2 x i4> [[RET]]1783;1784  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1785  %ret = load <2 x i4>, ptr addrspace(7) %p1786  ret <2 x i4> %ret1787}1788 1789define void @store_v2i4(<2 x i4> %data, ptr addrspace(8) inreg %buf) {1790; CHECK-LABEL: define void @store_v2i4(1791; CHECK-SAME: <2 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1792; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <2 x i4> [[DATA]] to i81793; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)1794; CHECK-NEXT:    ret void1795;1796  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1797  store <2 x i4> %data, ptr addrspace(7) %p1798  ret void1799}1800 1801define <4 x i4> @load_v4i4(ptr addrspace(8) inreg %buf) {1802; CHECK-LABEL: define <4 x i4> @load_v4i4(1803; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1804; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1805; CHECK-NEXT:    [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <4 x i4>1806; CHECK-NEXT:    ret <4 x i4> [[RET]]1807;1808  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1809  %ret = load <4 x i4>, ptr addrspace(7) %p1810  ret <4 x i4> %ret1811}1812 1813define void @store_v4i4(<4 x i4> %data, ptr addrspace(8) inreg %buf) {1814; CHECK-LABEL: define void @store_v4i4(1815; CHECK-SAME: <4 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1816; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <4 x i4> [[DATA]] to i161817; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1818; CHECK-NEXT:    ret void1819;1820  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1821  store <4 x i4> %data, ptr addrspace(7) %p1822  ret void1823}1824 1825define <8 x i4> @load_v8i4(ptr addrspace(8) inreg %buf) {1826; CHECK-LABEL: define <8 x i4> @load_v8i4(1827; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1828; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1829; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <8 x i4>1830; CHECK-NEXT:    ret <8 x i4> [[RET]]1831;1832  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1833  %ret = load <8 x i4>, ptr addrspace(7) %p1834  ret <8 x i4> %ret1835}1836 1837define void @store_v8i4(<8 x i4> %data, ptr addrspace(8) inreg %buf) {1838; CHECK-LABEL: define void @store_v8i4(1839; CHECK-SAME: <8 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1840; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <8 x i4> [[DATA]] to i321841; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1842; CHECK-NEXT:    ret void1843;1844  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1845  store <8 x i4> %data, ptr addrspace(7) %p1846  ret void1847}1848 1849;;; Vectors of non-byte-sized integers.1850 1851define <2 x i6> @load_v2i6(ptr addrspace(8) inreg %buf) {1852; CHECK-LABEL: define <2 x i6> @load_v2i6(1853; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1854; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1855; CHECK-NEXT:    [[RET_TRUNC:%.*]] = trunc i16 [[RET_LOADABLE]] to i121856; CHECK-NEXT:    [[RET:%.*]] = bitcast i12 [[RET_TRUNC]] to <2 x i6>1857; CHECK-NEXT:    ret <2 x i6> [[RET]]1858;1859  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1860  %ret = load <2 x i6>, ptr addrspace(7) %p1861  ret <2 x i6> %ret1862}1863 1864define void @store_v2i6(<2 x i6> %data, ptr addrspace(8) inreg %buf) {1865; CHECK-LABEL: define void @store_v2i6(1866; CHECK-SAME: <2 x i6> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1867; CHECK-NEXT:    [[DATA_AS_SCALAR:%.*]] = bitcast <2 x i6> [[DATA]] to i121868; CHECK-NEXT:    [[DATA_ZEXT:%.*]] = zext i12 [[DATA_AS_SCALAR]] to i161869; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_ZEXT]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)1870; CHECK-NEXT:    ret void1871;1872  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1873  store <2 x i6> %data, ptr addrspace(7) %p1874  ret void1875}1876 1877;; Blocks of fp6 elements1878define <6 x i32> @load_v32i6(ptr addrspace(8) inreg %buf) {1879; CHECK-LABEL: define <6 x i32> @load_v32i6(1880; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1881; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)1882; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>1883; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <6 x i32> poison, <6 x i32> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>1884; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)1885; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>1886; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <6 x i32> [[RET_PARTS_0]], <6 x i32> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>1887; CHECK-NEXT:    [[RET:%.*]] = bitcast <6 x i32> [[RET_PARTS_4]] to <32 x i6>1888; CHECK-NEXT:    [[RET_CAST:%.*]] = bitcast <32 x i6> [[RET]] to <6 x i32>1889; CHECK-NEXT:    ret <6 x i32> [[RET_CAST]]1890;1891  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1892  %ret = load <32 x i6>, ptr addrspace(7) %p1893  %ret.cast = bitcast <32 x i6> %ret to <6 x i32>1894  ret <6 x i32> %ret.cast1895}1896 1897define void @store_v32i6(<6 x i32> %data.abi, ptr addrspace(8) inreg %buf) {1898; CHECK-LABEL: define void @store_v32i6(1899; CHECK-SAME: <6 x i32> [[DATA_ABI:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1900; CHECK-NEXT:    [[DATA:%.*]] = bitcast <6 x i32> [[DATA_ABI]] to <32 x i6>1901; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1902; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)1903; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>1904; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)1905; CHECK-NEXT:    ret void1906;1907  %data = bitcast <6 x i32> %data.abi to <32 x i6>1908  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1909  store <32 x i6> %data, ptr addrspace(7) %p1910  ret void1911}1912 1913;;; Modifiers1914 1915define <4 x i8> @volatile_load_v4i8(ptr addrspace(8) inreg %buf) {1916; CHECK-LABEL: define <4 x i8> @volatile_load_v4i8(1917; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1918; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648)1919; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>1920; CHECK-NEXT:    ret <4 x i8> [[RET]]1921;1922  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1923  %ret = load volatile <4 x i8>, ptr addrspace(7) %p1924  ret <4 x i8> %ret1925}1926 1927define void @volatile_store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {1928; CHECK-LABEL: define void @volatile_store_v4i8(1929; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1930; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i321931; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648)1932; CHECK-NEXT:    ret void1933;1934  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1935  store volatile <4 x i8> %data, ptr addrspace(7) %p1936  ret void1937}1938 1939define <6 x i8> @volatile_load_v6i8(ptr addrspace(8) inreg %buf) {1940; CHECK-LABEL: define <6 x i8> @volatile_load_v6i8(1941; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1942; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648)1943; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>1944; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>1945; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648)1946; CHECK-NEXT:    [[RET_SLICE_2:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 21947; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i16> [[RET_SLICE_2]] to <6 x i8>1948; CHECK-NEXT:    ret <6 x i8> [[RET]]1949;1950  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1951  %ret = load volatile <6 x i8>, ptr addrspace(7) %p1952  ret <6 x i8> %ret1953}1954 1955define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {1956; CHECK-LABEL: define void @volatile_store_v6i8(1957; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1958; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>1959; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>1960; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648)1961; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 21962; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648)1963; CHECK-NEXT:    ret void1964;1965  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1966  store volatile <6 x i8> %data, ptr addrspace(7) %p1967  ret void1968}1969 1970define [2 x [2 x i32]] @load_a2a2i32(ptr addrspace(8) inreg %buf) {1971; CHECK-LABEL: define [2 x [2 x i32]] @load_a2a2i32(1972; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1973; CHECK-NEXT:    [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)1974; CHECK-NEXT:    [[RET0_ELEM_0:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 01975; CHECK-NEXT:    [[RET0_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET0_ELEM_0]], 01976; CHECK-NEXT:    [[RET0_ELEM_1:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 11977; CHECK-NEXT:    [[RET0_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET0_AS_ARRAY_0]], i32 [[RET0_ELEM_1]], 11978; CHECK-NEXT:    [[RET0:%.*]] = insertvalue [2 x [2 x i32]] poison, [2 x i32] [[RET0_AS_ARRAY_1]], 01979; CHECK-NEXT:    [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)1980; CHECK-NEXT:    [[RET1_ELEM_0:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 01981; CHECK-NEXT:    [[RET1_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET1_ELEM_0]], 01982; CHECK-NEXT:    [[RET1_ELEM_1:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 11983; CHECK-NEXT:    [[RET1_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET1_AS_ARRAY_0]], i32 [[RET1_ELEM_1]], 11984; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x [2 x i32]] [[RET0]], [2 x i32] [[RET1_AS_ARRAY_1]], 11985; CHECK-NEXT:    ret [2 x [2 x i32]] [[RET]]1986;1987  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1988  %ret = load [2 x [2 x i32]], ptr addrspace(7) %p1989  ret [2 x [2 x i32]] %ret1990}1991 1992define void @store_a2a2i32([2 x [2 x i32]] %data, ptr addrspace(8) inreg %buf) {1993; CHECK-LABEL: define void @store_a2a2i32(1994; CHECK-SAME: [2 x [2 x i32]] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {1995; CHECK-NEXT:    [[DATA0:%.*]] = extractvalue [2 x [2 x i32]] [[DATA]], 01996; CHECK-NEXT:    [[DATA0_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA0]], 01997; CHECK-NEXT:    [[DATA0_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA0_ELEM_0]], i64 01998; CHECK-NEXT:    [[DATA0_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA0]], 11999; CHECK-NEXT:    [[DATA0_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA0_AS_VEC_0]], i32 [[DATA0_ELEM_1]], i64 12000; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)2001; CHECK-NEXT:    [[DATA1:%.*]] = extractvalue [2 x [2 x i32]] [[DATA]], 12002; CHECK-NEXT:    [[DATA1_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA1]], 02003; CHECK-NEXT:    [[DATA1_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA1_ELEM_0]], i64 02004; CHECK-NEXT:    [[DATA1_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA1]], 12005; CHECK-NEXT:    [[DATA1_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA1_AS_VEC_0]], i32 [[DATA1_ELEM_1]], i64 12006; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)2007; CHECK-NEXT:    ret void2008;2009  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2010  store [2 x [2 x i32]] %data, ptr addrspace(7) %p2011  ret void2012}2013 2014define [2 x <2 x i32>] @load_a2v2i32(ptr addrspace(8) inreg %buf) {2015; CHECK-LABEL: define [2 x <2 x i32>] @load_a2v2i32(2016; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2017; CHECK-NEXT:    [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2018; CHECK-NEXT:    [[RET0:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[RET0_OFF_0]], 02019; CHECK-NEXT:    [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)2020; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x <2 x i32>] [[RET0]], <2 x i32> [[RET1_OFF_8]], 12021; CHECK-NEXT:    ret [2 x <2 x i32>] [[RET]]2022;2023  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2024  %ret = load [2 x <2 x i32>], ptr addrspace(7) %p2025  ret [2 x <2 x i32>] %ret2026}2027 2028define void @store_a2v2i32([2 x <2 x i32>] %data, ptr addrspace(8) inreg %buf) {2029; CHECK-LABEL: define void @store_a2v2i32(2030; CHECK-SAME: [2 x <2 x i32>] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2031; CHECK-NEXT:    [[DATA0:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 02032; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2033; CHECK-NEXT:    [[DATA1:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 12034; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)2035; CHECK-NEXT:    ret void2036;2037  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2038  store [2 x <2 x i32>] %data, ptr addrspace(7) %p2039  ret void2040}2041 2042define { i32 } @load_sl_i32s(ptr addrspace(8) inreg %buf) {2043; CHECK-LABEL: define { i32 } @load_sl_i32s(2044; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2045; CHECK-NEXT:    [[RET_0_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)2046; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i32 } poison, i32 [[RET_0_OFF_0]], 02047; CHECK-NEXT:    ret { i32 } [[RET]]2048;2049  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2050  %ret = load { i32 }, ptr addrspace(7) %p2051  ret { i32 } %ret2052}2053 2054define void @store_sl_i32s({ i32 } %data, ptr addrspace(8) inreg %buf) {2055; CHECK-LABEL: define void @store_sl_i32s(2056; CHECK-SAME: { i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2057; CHECK-NEXT:    [[DATA_0:%.*]] = extractvalue { i32 } [[DATA]], 02058; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)2059; CHECK-NEXT:    ret void2060;2061  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2062  store { i32 } %data, ptr addrspace(7) %p2063  ret void2064}2065 2066define { { float } } @load_sl_sl_f32ss(ptr addrspace(8) inreg %buf) {2067; CHECK-LABEL: define { { float } } @load_sl_sl_f32ss(2068; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2069; CHECK-NEXT:    [[RET_0_0_OFF_0:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)2070; CHECK-NEXT:    [[RET:%.*]] = insertvalue { { float } } poison, float [[RET_0_0_OFF_0]], 0, 02071; CHECK-NEXT:    ret { { float } } [[RET]]2072;2073  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2074  %ret = load { { float } }, ptr addrspace(7) %p2075  ret { { float } } %ret2076}2077 2078define void @store_sl_sl_f32ss({ { float } } %data, ptr addrspace(8) inreg %buf) {2079; CHECK-LABEL: define void @store_sl_sl_f32ss(2080; CHECK-SAME: { { float } } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2081; CHECK-NEXT:    [[DATA_0_0:%.*]] = extractvalue { { float } } [[DATA]], 0, 02082; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_0_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)2083; CHECK-NEXT:    ret void2084;2085  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2086  store { { float } } %data, ptr addrspace(7) %p2087  ret void2088}2089 2090define { <2 x i32> } @load_sl_v2i32s(ptr addrspace(8) inreg %buf) {2091; CHECK-LABEL: define { <2 x i32> } @load_sl_v2i32s(2092; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2093; CHECK-NEXT:    [[RET_0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2094; CHECK-NEXT:    [[RET:%.*]] = insertvalue { <2 x i32> } poison, <2 x i32> [[RET_0_OFF_0]], 02095; CHECK-NEXT:    ret { <2 x i32> } [[RET]]2096;2097  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2098  %ret = load { <2 x i32> }, ptr addrspace(7) %p2099  ret { <2 x i32> } %ret2100}2101 2102define void @store_sl_v2i32s({ <2 x i32> } %data, ptr addrspace(8) inreg %buf) {2103; CHECK-LABEL: define void @store_sl_v2i32s(2104; CHECK-SAME: { <2 x i32> } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2105; CHECK-NEXT:    [[DATA_0:%.*]] = extractvalue { <2 x i32> } [[DATA]], 02106; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2107; CHECK-NEXT:    ret void2108;2109  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2110  store { <2 x i32> } %data, ptr addrspace(7) %p2111  ret void2112}2113 2114define { i64, i32 } @load_sl_i64i32s(ptr addrspace(8) inreg %buf) {2115; CHECK-LABEL: define { i64, i32 } @load_sl_i64i32s(2116; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2117; CHECK-NEXT:    [[RET_0_OFF_0:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2118; CHECK-NEXT:    [[RET_0:%.*]] = insertvalue { i64, i32 } poison, i64 [[RET_0_OFF_0]], 02119; CHECK-NEXT:    [[RET_1_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)2120; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i64, i32 } [[RET_0]], i32 [[RET_1_OFF_8]], 12121; CHECK-NEXT:    ret { i64, i32 } [[RET]]2122;2123  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2124  %ret = load { i64, i32 }, ptr addrspace(7) %p2125  ret { i64, i32 } %ret2126}2127 2128define void @store_sl_i64i32s({ i64, i32 } %data, ptr addrspace(8) inreg %buf) {2129; CHECK-LABEL: define void @store_sl_i64i32s(2130; CHECK-SAME: { i64, i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2131; CHECK-NEXT:    [[DATA_0:%.*]] = extractvalue { i64, i32 } [[DATA]], 02132; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2133; CHECK-NEXT:    [[DATA_1:%.*]] = extractvalue { i64, i32 } [[DATA]], 12134; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)2135; CHECK-NEXT:    ret void2136;2137  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2138  store { i64, i32 } %data, ptr addrspace(7) %p2139  ret void2140}2141 2142define [4 x i7] @load_a4i7(ptr addrspace(8) inreg %buf) {2143; CHECK-LABEL: define [4 x i7] @load_a4i7(2144; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2145; CHECK-NEXT:    [[RET0_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)2146; CHECK-NEXT:    [[RET0_TRUNC:%.*]] = trunc i8 [[RET0_OFF_0]] to i72147; CHECK-NEXT:    [[RET0:%.*]] = insertvalue [4 x i7] poison, i7 [[RET0_TRUNC]], 02148; CHECK-NEXT:    [[RET1_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)2149; CHECK-NEXT:    [[RET1_TRUNC:%.*]] = trunc i8 [[RET1_OFF_1]] to i72150; CHECK-NEXT:    [[RET1:%.*]] = insertvalue [4 x i7] [[RET0]], i7 [[RET1_TRUNC]], 12151; CHECK-NEXT:    [[RET2_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)2152; CHECK-NEXT:    [[RET2_TRUNC:%.*]] = trunc i8 [[RET2_OFF_2]] to i72153; CHECK-NEXT:    [[RET2:%.*]] = insertvalue [4 x i7] [[RET1]], i7 [[RET2_TRUNC]], 22154; CHECK-NEXT:    [[RET3_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)2155; CHECK-NEXT:    [[RET3_TRUNC:%.*]] = trunc i8 [[RET3_OFF_3]] to i72156; CHECK-NEXT:    [[RET:%.*]] = insertvalue [4 x i7] [[RET2]], i7 [[RET3_TRUNC]], 32157; CHECK-NEXT:    ret [4 x i7] [[RET]]2158;2159  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2160  %ret = load [4 x i7], ptr addrspace(7) %p2161  ret [4 x i7] %ret2162}2163 2164define void @store_a4i7([4 x i7] %data, ptr addrspace(8) inreg %buf) {2165; CHECK-LABEL: define void @store_a4i7(2166; CHECK-SAME: [4 x i7] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2167; CHECK-NEXT:    [[DATA0:%.*]] = extractvalue [4 x i7] [[DATA]], 02168; CHECK-NEXT:    [[DATA0_ZEXT:%.*]] = zext i7 [[DATA0]] to i82169; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA0_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)2170; CHECK-NEXT:    [[DATA1:%.*]] = extractvalue [4 x i7] [[DATA]], 12171; CHECK-NEXT:    [[DATA1_ZEXT:%.*]] = zext i7 [[DATA1]] to i82172; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA1_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)2173; CHECK-NEXT:    [[DATA2:%.*]] = extractvalue [4 x i7] [[DATA]], 22174; CHECK-NEXT:    [[DATA2_ZEXT:%.*]] = zext i7 [[DATA2]] to i82175; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA2_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)2176; CHECK-NEXT:    [[DATA3:%.*]] = extractvalue [4 x i7] [[DATA]], 32177; CHECK-NEXT:    [[DATA3_ZEXT:%.*]] = zext i7 [[DATA3]] to i82178; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA3_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)2179; CHECK-NEXT:    ret void2180;2181  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2182  store [4 x i7] %data, ptr addrspace(7) %p2183  ret void2184}2185 2186;;; Scalable vector. This isn't semantically meaningful but shouldn't crash.2187 2188define <vscale x 2 x i32> @load_nxv2i32(ptr addrspace(8) inreg %buf) {2189; CHECK-LABEL: define <vscale x 2 x i32> @load_nxv2i32(2190; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2191; CHECK-NEXT:    [[RET:%.*]] = call <vscale x 2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.nxv2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2192; CHECK-NEXT:    ret <vscale x 2 x i32> [[RET]]2193;2194  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2195  %ret = load <vscale x 2 x i32>, ptr addrspace(7) %p2196  ret <vscale x 2 x i32> %ret2197}2198 2199define void @store_nxv2i32(<vscale x 2 x i32> %data, ptr addrspace(8) inreg %buf) {2200; CHECK-LABEL: define void @store_nxv2i32(2201; CHECK-SAME: <vscale x 2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {2202; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.nxv2i32(<vscale x 2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)2203; CHECK-NEXT:    ret void2204;2205  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2206  store <vscale x 2 x i32> %data, ptr addrspace(7) %p2207  ret void2208}2209