182 lines · plain
1; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_35 -O0 | FileCheck %s --check-prefix PTX2; RUN: opt < %s -S -nvptx-lower-aggr-copies | FileCheck %s --check-prefix IR3; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_35 -O0 | %ptxas-verify %}4 5; Verify that the NVPTXLowerAggrCopies pass works as expected - calls to6; llvm.mem* intrinsics get lowered to loops.7 8target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"9target triple = "nvptx64-unknown-unknown"10 11declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1) #112declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1) #113declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) #114 15define ptr @memcpy_caller(ptr %dst, ptr %src, i64 %n) #0 {16entry:17 tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %n, i1 false)18 ret ptr %dst19 20; IR-LABEL: @memcpy_caller21; IR: entry:22; IR: [[Cond:%[0-9]+]] = icmp ne i64 %n, 023; IR: br i1 [[Cond]], label %loop-memcpy-expansion, label %post-loop-memcpy-expansion24 25; IR: loop-memcpy-expansion:26; IR: %loop-index = phi i64 [ 0, %entry ], [ [[IndexInc:%[0-9]+]], %loop-memcpy-expansion ]27; IR: [[SrcGep:%[0-9]+]] = getelementptr inbounds i8, ptr %src, i64 %loop-index28; IR: [[Load:%[0-9]+]] = load i8, ptr [[SrcGep]]29; IR: [[DstGep:%[0-9]+]] = getelementptr inbounds i8, ptr %dst, i64 %loop-index30; IR: store i8 [[Load]], ptr [[DstGep]]31; IR: [[IndexInc]] = add i64 %loop-index, 132; IR: [[Cond2:%[0-9]+]] = icmp ult i64 [[IndexInc]], %n33; IR: br i1 [[Cond2]], label %loop-memcpy-expansion, label %post-loop-memcpy-expansion34 35; IR-LABEL: post-loop-memcpy-expansion:36; IR: ret ptr %dst37 38; PTX-LABEL: .visible .func (.param .b64 func_retval0) memcpy_caller39; PTX: $L__BB[[LABEL:[_0-9]+]]:40; PTX: ld.b8 %rs[[REG:[0-9]+]]41; PTX: st.b8 [%rd{{[0-9]+}}], %rs[[REG]]42; PTX: add.s64 %rd[[COUNTER:[0-9]+]], %rd{{[0-9]+}}, 143; PTX: setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd44; PTX: @%p[[PRED]] bra $L__BB[[LABEL]]45 46}47 48define ptr @memcpy_volatile_caller(ptr %dst, ptr %src, i64 %n) #0 {49entry:50 tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %n, i1 true)51 ret ptr %dst52 53; IR-LABEL: @memcpy_volatile_caller54; IR: entry:55; IR: [[Cond:%[0-9]+]] = icmp ne i64 %n, 056; IR: br i1 [[Cond]], label %loop-memcpy-expansion, label %post-loop-memcpy-expansion57 58; IR: loop-memcpy-expansion:59; IR: %loop-index = phi i64 [ 0, %entry ], [ [[IndexInc:%[0-9]+]], %loop-memcpy-expansion ]60; IR: [[SrcGep:%[0-9]+]] = getelementptr inbounds i8, ptr %src, i64 %loop-index61; IR: [[Load:%[0-9]+]] = load volatile i8, ptr [[SrcGep]]62; IR: [[DstGep:%[0-9]+]] = getelementptr inbounds i8, ptr %dst, i64 %loop-index63; IR: store volatile i8 [[Load]], ptr [[DstGep]]64; IR: [[IndexInc]] = add i64 %loop-index, 165; IR: [[Cond2:%[0-9]+]] = icmp ult i64 [[IndexInc]], %n66; IR: br i1 [[Cond2]], label %loop-memcpy-expansion, label %post-loop-memcpy-expansion67 68; IR-LABEL: post-loop-memcpy-expansion:69; IR: ret ptr %dst70 71 72; PTX-LABEL: .visible .func (.param .b64 func_retval0) memcpy_volatile_caller73; PTX: $L__BB[[LABEL:[_0-9]+]]:74; PTX: ld.volatile.b8 %rs[[REG:[0-9]+]]75; PTX: st.volatile.b8 [%rd{{[0-9]+}}], %rs[[REG]]76; PTX: add.s64 %rd[[COUNTER:[0-9]+]], %rd{{[0-9]+}}, 177; PTX: setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd78; PTX: @%p[[PRED]] bra $L__BB[[LABEL]]79}80 81define ptr @memcpy_casting_caller(ptr %dst, ptr %src, i64 %n) #0 {82entry:83 tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %n, i1 false)84 ret ptr %dst85 86; Check that casts in calls to memcpy are handled properly87; IR-LABEL: @memcpy_casting_caller88; IR: getelementptr inbounds i8, ptr %src89; IR: getelementptr inbounds i8, ptr %dst90}91 92define ptr @memcpy_known_size(ptr %dst, ptr %src) {93entry:94 tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 144, i1 false)95 ret ptr %dst96 97; Check that calls with compile-time constant size are handled correctly98; IR-LABEL: @memcpy_known_size99; IR: entry:100; IR: br label %load-store-loop101; IR: load-store-loop:102; IR: %loop-index = phi i64 [ 0, %entry ], [ [[IndexInc:%[0-9]+]], %load-store-loop ]103; IR: [[SrcGep:%[0-9]+]] = getelementptr inbounds i8, ptr %src, i64 %loop-index104; IR: [[Load:%[0-9]+]] = load i8, ptr [[SrcGep]]105; IR: [[DstGep:%[0-9]+]] = getelementptr inbounds i8, ptr %dst, i64 %loop-index106; IR: store i8 [[Load]], ptr [[DstGep]]107; IR: [[IndexInc]] = add i64 %loop-index, 1108; IR: [[Cond:%[0-9]+]] = icmp ult i64 %3, 144109; IR: br i1 [[Cond]], label %load-store-loop, label %memcpy-split110}111 112define ptr @memset_caller(ptr %dst, i32 %c, i64 %n) #0 {113entry:114 %0 = trunc i32 %c to i8115 tail call void @llvm.memset.p0.i64(ptr %dst, i8 %0, i64 %n, i1 false)116 ret ptr %dst117 118; IR-LABEL: @memset_caller119; IR: [[VAL:%[0-9]+]] = trunc i32 %c to i8120; IR: [[CMPREG:%[0-9]+]] = icmp eq i64 0, %n121; IR: br i1 [[CMPREG]], label %split, label %loadstoreloop122; IR: loadstoreloop:123; IR: [[STOREPTR:%[0-9]+]] = getelementptr inbounds i8, ptr %dst, i64124; IR-NEXT: store i8 [[VAL]], ptr [[STOREPTR]]125 126; PTX-LABEL: .visible .func (.param .b64 func_retval0) memset_caller(127; PTX: ld.param.b32 %r[[C:[0-9]+]]128; PTX: cvt.u16.u32 %rs[[REG:[0-9]+]], %r[[C]];129; PTX: $L__BB[[LABEL:[_0-9]+]]:130; PTX: st.b8 [%rd{{[0-9]+}}], %rs[[REG]]131; PTX: add.s64 %rd[[COUNTER:[0-9]+]], %rd{{[0-9]+}}, 1132; PTX: setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd133; PTX: @%p[[PRED]] bra $L__BB[[LABEL]]134}135 136define ptr @volatile_memset_caller(ptr %dst, i32 %c, i64 %n) #0 {137entry:138 %0 = trunc i32 %c to i8139 tail call void @llvm.memset.p0.i64(ptr %dst, i8 %0, i64 %n, i1 true)140 ret ptr %dst141 142; IR-LABEL: @volatile_memset_caller143; IR: [[VAL:%[0-9]+]] = trunc i32 %c to i8144; IR: loadstoreloop:145; IR: [[STOREPTR:%[0-9]+]] = getelementptr inbounds i8, ptr %dst, i64146; IR-NEXT: store volatile i8 [[VAL]], ptr [[STOREPTR]]147}148 149define ptr @memmove_caller(ptr %dst, ptr %src, i64 %n) #0 {150entry:151 tail call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %n, i1 false)152 ret ptr %dst153 154; IR-LABEL: @memmove_caller155; IR: icmp ult ptr %src, %dst156; IR: [[PHIVAL:%[0-9a-zA-Z_]+]] = phi i64157; IR-NEXT: %bwd_main_index = sub i64 [[PHIVAL]], 1158; IR: [[FWDPHIVAL:%[0-9a-zA-Z_]+]] = phi i64159; IR: {{%[0-9a-zA-Z_]+}} = add i64 [[FWDPHIVAL]], 1160 161; PTX-LABEL: .visible .func (.param .b64 func_retval0) memmove_caller(162; PTX: ld.param.b64 %rd[[N:[0-9]+]]163; PTX-DAG: setp.eq.b64 %p[[NEQ0:[0-9]+]], %rd[[N]], 0164; PTX-DAG: setp.ge.u64 %p[[SRC_GT_THAN_DST:[0-9]+]], %rd{{[0-9]+}}, %rd{{[0-9]+}}165; PTX-NEXT: @%p[[SRC_GT_THAN_DST]] bra $L__BB[[FORWARD_BB:[0-9_]+]]166; -- this is the backwards copying BB167; PTX: @%p[[NEQ0]] bra $L__BB[[EXIT:[0-9_]+]]168; PTX: add.s64 %rd{{[0-9]}}, %rd{{[0-9]}}, -1169; PTX: ld.b8 %rs[[ELEMENT:[0-9]+]]170; PTX: st.b8 [%rd{{[0-9]+}}], %rs[[ELEMENT]]171; -- this is the forwards copying BB172; PTX: $L__BB[[FORWARD_BB]]:173; PTX: @%p[[NEQ0]] bra $L__BB[[EXIT]]174; PTX: ld.b8 %rs[[ELEMENT2:[0-9]+]]175; PTX: st.b8 [%rd{{[0-9]+}}], %rs[[ELEMENT2]]176; PTX: add.s64 %rd{{[0-9]+}}, %rd{{[0-9]+}}, 1177; -- exit block178; PTX: $L__BB[[EXIT]]:179; PTX-NEXT: st.param.b64 [func_retval0180; PTX-NEXT: ret181}182