582 lines · plain
1// Test hlfir.cshift simplification to hlfir.elemental:2// RUN: fir-opt --simplify-hlfir-intrinsics=allow-new-side-effects=false %s | FileCheck %s --check-prefixes=ALL,NOANSE3// RUN: fir-opt --simplify-hlfir-intrinsics=allow-new-side-effects=true %s | FileCheck %s --check-prefixes=ALL,ANSE4// RUN: fir-opt --simplify-hlfir-intrinsics -flang-inline-matmul-as-elemental %s | FileCheck %s --check-prefixes=ALL,ELEMENTAL5 6func.func @matmul_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlfir.expr<?x?xi32>) -> !hlfir.expr<?x?xi32> {7 %res = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x?xi16>, !hlfir.expr<?x?xi32>) -> !hlfir.expr<?x?xi32>8 return %res : !hlfir.expr<?x?xi32>9}10// ALL-LABEL: func.func @matmul_matrix_matrix_integer(11// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?xi16>,12// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?x?xi32>) -> !hlfir.expr<?x?xi32> {13 14// NOANSE: hlfir.matmul15 16// ANSE: %[[VAL_3:.*]] = arith.constant 1 : index17// ANSE: %[[VAL_4:.*]] = arith.constant 0 : i3218// ANSE: %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi16>) -> !fir.shape<2>19// ANSE: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 0 : index} : (!fir.shape<2>) -> index20// ANSE: %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 1 : index} : (!fir.shape<2>) -> index21// ANSE: %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xi32>) -> !fir.shape<2>22// ANSE: %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 1 : index} : (!fir.shape<2>) -> index23// ANSE: %[[VAL_10:.*]] = fir.shape %[[VAL_6]], %[[VAL_9]] : (index, index) -> !fir.shape<2>24// ANSE: %[[VAL_11:.*]] = hlfir.eval_in_mem shape %[[VAL_10]] : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {25// ANSE: ^bb0(%[[VAL_12:.*]]: !fir.ref<!fir.array<?x?xi32>>):26// ANSE: %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>27// ANSE: fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {28// ANSE: fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {29// ANSE: %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]]) : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>30// ANSE: hlfir.assign %[[VAL_4]] to %[[VAL_22]] : i32, !fir.ref<i32>31// ANSE: }32// ANSE: }33// ANSE: fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {34// ANSE: fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {35// ANSE: fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {36// ANSE: %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]]) : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>37// ANSE: %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>38// ANSE: %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xi16>, index, index) -> i1639// ANSE: %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xi32>, index, index) -> i3240// ANSE: %[[VAL_36:.*]] = fir.convert %[[VAL_34]] : (i16) -> i3241// ANSE: %[[VAL_37:.*]] = arith.muli %[[VAL_36]], %[[VAL_35]] : i3242// ANSE: %[[VAL_38:.*]] = arith.addi %[[VAL_33]], %[[VAL_37]] : i3243// ANSE: hlfir.assign %[[VAL_38]] to %[[VAL_32]] : i32, !fir.ref<i32>44// ANSE: }45// ANSE: }46// ANSE: }47// ANSE: }48// ANSE: return %[[VAL_11]] : !hlfir.expr<?x?xi32>49// ANSE: }50 51// ELEMENTAL: %[[VAL_2:.*]] = arith.constant 1 : index52// ELEMENTAL: %[[VAL_3:.*]] = arith.constant 0 : i3253// ELEMENTAL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi16>) -> !fir.shape<2>54// ELEMENTAL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index55// ELEMENTAL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index56// ELEMENTAL: %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xi32>) -> !fir.shape<2>57// ELEMENTAL: %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index58// ELEMENTAL: %[[VAL_9:.*]] = fir.shape %[[VAL_5]], %[[VAL_8]] : (index, index) -> !fir.shape<2>59// ELEMENTAL: %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {60// ELEMENTAL: ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):61// ELEMENTAL: %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_15:.*]] = %[[VAL_3]]) -> (i32) {62// ELEMENTAL: %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_14]] : (!hlfir.expr<?x?xi16>, index, index) -> i1663// ELEMENTAL: %[[VAL_17:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_14]], %[[VAL_12]] : (!hlfir.expr<?x?xi32>, index, index) -> i3264// ELEMENTAL: %[[VAL_18:.*]] = fir.convert %[[VAL_16]] : (i16) -> i3265// ELEMENTAL: %[[VAL_19:.*]] = arith.muli %[[VAL_18]], %[[VAL_17]] : i3266// ELEMENTAL: %[[VAL_20:.*]] = arith.addi %[[VAL_15]], %[[VAL_19]] : i3267// ELEMENTAL: fir.result %[[VAL_20]] : i3268// ELEMENTAL: }69// ELEMENTAL: hlfir.yield_element %[[VAL_13]] : i3270// ELEMENTAL: }71// ELEMENTAL: return %[[VAL_10]] : !hlfir.expr<?x?xi32>72// ELEMENTAL: }73 74func.func @matmul_matrix_matrix_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.expr<?x?xf16>) -> !hlfir.expr<?x?xf32> {75 %res = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?xf16>) -> !hlfir.expr<?x?xf32>76 return %res : !hlfir.expr<?x?xf32>77}78// ALL-LABEL: func.func @matmul_matrix_matrix_real(79// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?xf32>,80// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?x?xf16>) -> !hlfir.expr<?x?xf32> {81 82// NOANSE: hlfir.matmul83 84// ANSE: %[[VAL_3:.*]] = arith.constant 1 : index85// ANSE: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f3286// ANSE: %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>87// ANSE: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 0 : index} : (!fir.shape<2>) -> index88// ANSE: %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 1 : index} : (!fir.shape<2>) -> index89// ANSE: %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xf16>) -> !fir.shape<2>90// ANSE: %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 1 : index} : (!fir.shape<2>) -> index91// ANSE: %[[VAL_10:.*]] = fir.shape %[[VAL_6]], %[[VAL_9]] : (index, index) -> !fir.shape<2>92// ANSE: %[[VAL_11:.*]] = hlfir.eval_in_mem shape %[[VAL_10]] : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {93// ANSE: ^bb0(%[[VAL_12:.*]]: !fir.ref<!fir.array<?x?xf32>>):94// ANSE: %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>95// ANSE: fir.do_loop %[[VAL_14:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {96// ANSE: fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {97// ANSE: %[[VAL_22:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_15]], %[[VAL_14]]) : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>98// ANSE: hlfir.assign %[[VAL_4]] to %[[VAL_22]] : f32, !fir.ref<f32>99// ANSE: }100// ANSE: }101// ANSE: fir.do_loop %[[VAL_23:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {102// ANSE: fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {103// ANSE: fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {104// ANSE: %[[VAL_32:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_25]], %[[VAL_24]]) : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>105// ANSE: %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<f32>106// ANSE: %[[VAL_34:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_25]], %[[VAL_23]] : (!hlfir.expr<?x?xf32>, index, index) -> f32107// ANSE: %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_23]], %[[VAL_24]] : (!hlfir.expr<?x?xf16>, index, index) -> f16108// ANSE: %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (f16) -> f32109// ANSE: %[[VAL_37:.*]] = arith.mulf %[[VAL_34]], %[[VAL_36]] : f32110// ANSE: %[[VAL_38:.*]] = arith.addf %[[VAL_33]], %[[VAL_37]] : f32111// ANSE: hlfir.assign %[[VAL_38]] to %[[VAL_32]] : f32, !fir.ref<f32>112// ANSE: }113// ANSE: }114// ANSE: }115// ANSE: }116// ANSE: return %[[VAL_11]] : !hlfir.expr<?x?xf32>117// ANSE: }118 119// ELEMENTAL: %[[VAL_2:.*]] = arith.constant 1 : index120// ELEMENTAL: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32121// ELEMENTAL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>122// ELEMENTAL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index123// ELEMENTAL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index124// ELEMENTAL: %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xf16>) -> !fir.shape<2>125// ELEMENTAL: %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index126// ELEMENTAL: %[[VAL_9:.*]] = fir.shape %[[VAL_5]], %[[VAL_8]] : (index, index) -> !fir.shape<2>127// ELEMENTAL: %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {128// ELEMENTAL: ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):129// ELEMENTAL: %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_15:.*]] = %[[VAL_3]]) -> (f32) {130// ELEMENTAL: %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_14]] : (!hlfir.expr<?x?xf32>, index, index) -> f32131// ELEMENTAL: %[[VAL_17:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_14]], %[[VAL_12]] : (!hlfir.expr<?x?xf16>, index, index) -> f16132// ELEMENTAL: %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (f16) -> f32133// ELEMENTAL: %[[VAL_19:.*]] = arith.mulf %[[VAL_16]], %[[VAL_18]] : f32134// ELEMENTAL: %[[VAL_20:.*]] = arith.addf %[[VAL_15]], %[[VAL_19]] : f32135// ELEMENTAL: fir.result %[[VAL_20]] : f32136// ELEMENTAL: }137// ELEMENTAL: hlfir.yield_element %[[VAL_13]] : f32138// ELEMENTAL: }139// ELEMENTAL: return %[[VAL_10]] : !hlfir.expr<?x?xf32>140// ELEMENTAL: }141 142func.func @matmul_matrix_matrix_complex(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %arg1: !hlfir.expr<?x?xcomplex<f16>>) -> !hlfir.expr<?x?xcomplex<f32>> {143 %res = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x?xcomplex<f32>>, !hlfir.expr<?x?xcomplex<f16>>) -> !hlfir.expr<?x?xcomplex<f32>>144 return %res : !hlfir.expr<?x?xcomplex<f32>>145}146// ALL-LABEL: func.func @matmul_matrix_matrix_complex(147// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?xcomplex<f32>>,148// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?x?xcomplex<f16>>) -> !hlfir.expr<?x?xcomplex<f32>> {149 150// NOANSE: hlfir.matmul151 152// ANSE: %[[VAL_3:.*]] = arith.constant 1 : index153// ANSE: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32154// ANSE: %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>155// ANSE: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 0 : index} : (!fir.shape<2>) -> index156// ANSE: %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 1 : index} : (!fir.shape<2>) -> index157// ANSE: %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xcomplex<f16>>) -> !fir.shape<2>158// ANSE: %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 1 : index} : (!fir.shape<2>) -> index159// ANSE: %[[VAL_10:.*]] = fir.shape %[[VAL_6]], %[[VAL_9]] : (index, index) -> !fir.shape<2>160// ANSE: %[[VAL_11:.*]] = hlfir.eval_in_mem shape %[[VAL_10]] : (!fir.shape<2>) -> !hlfir.expr<?x?xcomplex<f32>> {161// ANSE: ^bb0(%[[VAL_12:.*]]: !fir.ref<!fir.array<?x?xcomplex<f32>>>):162// ANSE: %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xcomplex<f32>>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xcomplex<f32>>>163// ANSE: %[[VAL_14:.*]] = fir.undefined complex<f32>164// ANSE: %[[VAL_15:.*]] = fir.insert_value %[[VAL_14]], %[[VAL_4]], [0 : index] : (complex<f32>, f32) -> complex<f32>165// ANSE: %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>166// ANSE: fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {167// ANSE: fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {168// ANSE: %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]]) : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>169// ANSE: hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>170// ANSE: }171// ANSE: }172// ANSE: fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {173// ANSE: fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {174// ANSE: fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {175// ANSE: %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]]) : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>176// ANSE: %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>177// ANSE: %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>178// ANSE: %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xcomplex<f16>>, index, index) -> complex<f16>179// ANSE: %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (complex<f16>) -> complex<f32>180// ANSE: %[[VAL_40:.*]] = fir.mulc %[[VAL_37]], %[[VAL_39]] : complex<f32>181// ANSE: %[[VAL_41:.*]] = fir.addc %[[VAL_36]], %[[VAL_40]] : complex<f32>182// ANSE: hlfir.assign %[[VAL_41]] to %[[VAL_35]] : complex<f32>, !fir.ref<complex<f32>>183// ANSE: }184// ANSE: }185// ANSE: }186// ANSE: }187// ANSE: return %[[VAL_11]] : !hlfir.expr<?x?xcomplex<f32>>188// ANSE: }189 190// ELEMENTAL: %[[VAL_2:.*]] = arith.constant 1 : index191// ELEMENTAL: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32192// ELEMENTAL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>193// ELEMENTAL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index194// ELEMENTAL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index195// ELEMENTAL: %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xcomplex<f16>>) -> !fir.shape<2>196// ELEMENTAL: %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index197// ELEMENTAL: %[[VAL_9:.*]] = fir.shape %[[VAL_5]], %[[VAL_8]] : (index, index) -> !fir.shape<2>198// ELEMENTAL: %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xcomplex<f32>> {199// ELEMENTAL: ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):200// ELEMENTAL: %[[VAL_13:.*]] = fir.undefined complex<f32>201// ELEMENTAL: %[[VAL_14:.*]] = fir.insert_value %[[VAL_13]], %[[VAL_3]], [0 : index] : (complex<f32>, f32) -> complex<f32>202// ELEMENTAL: %[[VAL_15:.*]] = fir.insert_value %[[VAL_14]], %[[VAL_3]], [1 : index] : (complex<f32>, f32) -> complex<f32>203// ELEMENTAL: %[[VAL_16:.*]] = fir.do_loop %[[VAL_17:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_18:.*]] = %[[VAL_15]]) -> (complex<f32>) {204// ELEMENTAL: %[[VAL_19:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_17]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>205// ELEMENTAL: %[[VAL_20:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_17]], %[[VAL_12]] : (!hlfir.expr<?x?xcomplex<f16>>, index, index) -> complex<f16>206// ELEMENTAL: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (complex<f16>) -> complex<f32>207// ELEMENTAL: %[[VAL_22:.*]] = fir.mulc %[[VAL_19]], %[[VAL_21]] : complex<f32>208// ELEMENTAL: %[[VAL_23:.*]] = fir.addc %[[VAL_18]], %[[VAL_22]] : complex<f32>209// ELEMENTAL: fir.result %[[VAL_23]] : complex<f32>210// ELEMENTAL: }211// ELEMENTAL: hlfir.yield_element %[[VAL_16]] : complex<f32>212// ELEMENTAL: }213// ELEMENTAL: return %[[VAL_10]] : !hlfir.expr<?x?xcomplex<f32>>214// ELEMENTAL: }215 216func.func @matmul_matrix_matrix_complex_real(%arg0: !hlfir.expr<?x?xcomplex<f32>>, %arg1: !hlfir.expr<?x?xf16>) -> !hlfir.expr<?x?xcomplex<f32>> {217 %res = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x?xcomplex<f32>>, !hlfir.expr<?x?xf16>) -> !hlfir.expr<?x?xcomplex<f32>>218 return %res : !hlfir.expr<?x?xcomplex<f32>>219}220// ALL-LABEL: func.func @matmul_matrix_matrix_complex_real(221// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?xcomplex<f32>>,222// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?x?xf16>) -> !hlfir.expr<?x?xcomplex<f32>> {223 224// NOANSE: hlfir.matmul225 226// ANSE: %[[VAL_3:.*]] = arith.constant 1 : index227// ANSE: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32228// ANSE: %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>229// ANSE: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 0 : index} : (!fir.shape<2>) -> index230// ANSE: %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 1 : index} : (!fir.shape<2>) -> index231// ANSE: %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xf16>) -> !fir.shape<2>232// ANSE: %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 1 : index} : (!fir.shape<2>) -> index233// ANSE: %[[VAL_10:.*]] = fir.shape %[[VAL_6]], %[[VAL_9]] : (index, index) -> !fir.shape<2>234// ANSE: %[[VAL_11:.*]] = hlfir.eval_in_mem shape %[[VAL_10]] : (!fir.shape<2>) -> !hlfir.expr<?x?xcomplex<f32>> {235// ANSE: ^bb0(%[[VAL_12:.*]]: !fir.ref<!fir.array<?x?xcomplex<f32>>>):236// ANSE: %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?xcomplex<f32>>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xcomplex<f32>>>237// ANSE: %[[VAL_14:.*]] = fir.undefined complex<f32>238// ANSE: %[[VAL_15:.*]] = fir.insert_value %[[VAL_14]], %[[VAL_4]], [0 : index] : (complex<f32>, f32) -> complex<f32>239// ANSE: %[[VAL_16:.*]] = fir.insert_value %[[VAL_15]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>240// ANSE: fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {241// ANSE: fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {242// ANSE: %[[VAL_25:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_18]], %[[VAL_17]]) : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>243// ANSE: hlfir.assign %[[VAL_16]] to %[[VAL_25]] : complex<f32>, !fir.ref<complex<f32>>244// ANSE: }245// ANSE: }246// ANSE: fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {247// ANSE: fir.do_loop %[[VAL_27:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] {248// ANSE: fir.do_loop %[[VAL_28:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {249// ANSE: %[[VAL_35:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_28]], %[[VAL_27]]) : (!fir.box<!fir.array<?x?xcomplex<f32>>>, index, index) -> !fir.ref<complex<f32>>250// ANSE: %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f32>>251// ANSE: %[[VAL_37:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_28]], %[[VAL_26]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>252// ANSE: %[[VAL_38:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_26]], %[[VAL_27]] : (!hlfir.expr<?x?xf16>, index, index) -> f16253// ANSE: %[[VAL_39:.*]] = fir.undefined complex<f32>254// ANSE: %[[VAL_40:.*]] = fir.insert_value %[[VAL_39]], %[[VAL_4]], [0 : index] : (complex<f32>, f32) -> complex<f32>255// ANSE: %[[VAL_41:.*]] = fir.insert_value %[[VAL_40]], %[[VAL_4]], [1 : index] : (complex<f32>, f32) -> complex<f32>256// ANSE: %[[VAL_42:.*]] = fir.convert %[[VAL_38]] : (f16) -> f32257// ANSE: %[[VAL_43:.*]] = fir.insert_value %[[VAL_41]], %[[VAL_42]], [0 : index] : (complex<f32>, f32) -> complex<f32>258// ANSE: %[[VAL_44:.*]] = fir.mulc %[[VAL_37]], %[[VAL_43]] : complex<f32>259// ANSE: %[[VAL_45:.*]] = fir.addc %[[VAL_36]], %[[VAL_44]] : complex<f32>260// ANSE: hlfir.assign %[[VAL_45]] to %[[VAL_35]] : complex<f32>, !fir.ref<complex<f32>>261// ANSE: }262// ANSE: }263// ANSE: }264// ANSE: }265// ANSE: return %[[VAL_11]] : !hlfir.expr<?x?xcomplex<f32>>266// ANSE: }267 268// ELEMENTAL: %[[VAL_2:.*]] = arith.constant 1 : index269// ELEMENTAL: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32270// ELEMENTAL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xcomplex<f32>>) -> !fir.shape<2>271// ELEMENTAL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index272// ELEMENTAL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index273// ELEMENTAL: %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xf16>) -> !fir.shape<2>274// ELEMENTAL: %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index275// ELEMENTAL: %[[VAL_9:.*]] = fir.shape %[[VAL_5]], %[[VAL_8]] : (index, index) -> !fir.shape<2>276// ELEMENTAL: %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xcomplex<f32>> {277// ELEMENTAL: ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):278// ELEMENTAL: %[[VAL_13:.*]] = fir.undefined complex<f32>279// ELEMENTAL: %[[VAL_14:.*]] = fir.insert_value %[[VAL_13]], %[[VAL_3]], [0 : index] : (complex<f32>, f32) -> complex<f32>280// ELEMENTAL: %[[VAL_15:.*]] = fir.insert_value %[[VAL_14]], %[[VAL_3]], [1 : index] : (complex<f32>, f32) -> complex<f32>281// ELEMENTAL: %[[VAL_16:.*]] = fir.do_loop %[[VAL_17:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_18:.*]] = %[[VAL_15]]) -> (complex<f32>) {282// ELEMENTAL: %[[VAL_19:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_17]] : (!hlfir.expr<?x?xcomplex<f32>>, index, index) -> complex<f32>283// ELEMENTAL: %[[VAL_20:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_17]], %[[VAL_12]] : (!hlfir.expr<?x?xf16>, index, index) -> f16284// ELEMENTAL: %[[VAL_21:.*]] = fir.undefined complex<f32>285// ELEMENTAL: %[[VAL_22:.*]] = fir.insert_value %[[VAL_21]], %[[VAL_3]], [0 : index] : (complex<f32>, f32) -> complex<f32>286// ELEMENTAL: %[[VAL_23:.*]] = fir.insert_value %[[VAL_22]], %[[VAL_3]], [1 : index] : (complex<f32>, f32) -> complex<f32>287// ELEMENTAL: %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (f16) -> f32288// ELEMENTAL: %[[VAL_25:.*]] = fir.insert_value %[[VAL_23]], %[[VAL_24]], [0 : index] : (complex<f32>, f32) -> complex<f32>289// ELEMENTAL: %[[VAL_26:.*]] = fir.mulc %[[VAL_19]], %[[VAL_25]] : complex<f32>290// ELEMENTAL: %[[VAL_27:.*]] = fir.addc %[[VAL_18]], %[[VAL_26]] : complex<f32>291// ELEMENTAL: fir.result %[[VAL_27]] : complex<f32>292// ELEMENTAL: }293// ELEMENTAL: hlfir.yield_element %[[VAL_16]] : complex<f32>294// ELEMENTAL: }295// ELEMENTAL: return %[[VAL_10]] : !hlfir.expr<?x?xcomplex<f32>>296// ELEMENTAL: }297 298func.func @matmul_matrix_matrix_logical(%arg0: !hlfir.expr<?x?x!fir.logical<1>>, %arg1: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?x?x!fir.logical<4>> {299 %res = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x?x!fir.logical<1>>, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?x?x!fir.logical<4>>300 return %res : !hlfir.expr<?x?x!fir.logical<4>>301}302// ALL-LABEL: func.func @matmul_matrix_matrix_logical(303// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?x!fir.logical<1>>,304// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?x?x!fir.logical<4>> {305 306// NOANSE: hlfir.matmul307 308// ANSE: %[[VAL_3:.*]] = arith.constant 1 : index309// ANSE: %[[VAL_4:.*]] = arith.constant false310// ANSE: %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<1>>) -> !fir.shape<2>311// ANSE: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 0 : index} : (!fir.shape<2>) -> index312// ANSE: %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 1 : index} : (!fir.shape<2>) -> index313// ANSE: %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>314// ANSE: %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 1 : index} : (!fir.shape<2>) -> index315// ANSE: %[[VAL_10:.*]] = fir.shape %[[VAL_6]], %[[VAL_9]] : (index, index) -> !fir.shape<2>316// ANSE: %[[VAL_11:.*]] = hlfir.eval_in_mem shape %[[VAL_10]] : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<4>> {317// ANSE: ^bb0(%[[VAL_12:.*]]: !fir.ref<!fir.array<?x?x!fir.logical<4>>>):318// ANSE: %[[VAL_13:.*]] = fir.embox %[[VAL_12]](%[[VAL_10]]) : (!fir.ref<!fir.array<?x?x!fir.logical<4>>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?x!fir.logical<4>>>319// ANSE: %[[VAL_14:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>320// ANSE: fir.do_loop %[[VAL_15:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {321// ANSE: fir.do_loop %[[VAL_16:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {322// ANSE: %[[VAL_23:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_16]], %[[VAL_15]]) : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>323// ANSE: hlfir.assign %[[VAL_14]] to %[[VAL_23]] : !fir.logical<4>, !fir.ref<!fir.logical<4>>324// ANSE: }325// ANSE: }326// ANSE: fir.do_loop %[[VAL_24:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] unordered {327// ANSE: fir.do_loop %[[VAL_25:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered {328// ANSE: fir.do_loop %[[VAL_26:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {329// ANSE: %[[VAL_33:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_26]], %[[VAL_25]]) : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>330// ANSE: %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<!fir.logical<4>>331// ANSE: %[[VAL_35:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_26]], %[[VAL_24]] : (!hlfir.expr<?x?x!fir.logical<1>>, index, index) -> !fir.logical<1>332// ANSE: %[[VAL_36:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_24]], %[[VAL_25]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>333// ANSE: %[[VAL_37:.*]] = fir.convert %[[VAL_34]] : (!fir.logical<4>) -> i1334// ANSE: %[[VAL_38:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<1>) -> i1335// ANSE: %[[VAL_39:.*]] = fir.convert %[[VAL_36]] : (!fir.logical<4>) -> i1336// ANSE: %[[VAL_40:.*]] = arith.andi %[[VAL_38]], %[[VAL_39]] : i1337// ANSE: %[[VAL_41:.*]] = arith.ori %[[VAL_37]], %[[VAL_40]] : i1338// ANSE: %[[VAL_42:.*]] = fir.convert %[[VAL_41]] : (i1) -> !fir.logical<4>339// ANSE: hlfir.assign %[[VAL_42]] to %[[VAL_33]] : !fir.logical<4>, !fir.ref<!fir.logical<4>>340// ANSE: }341// ANSE: }342// ANSE: }343// ANSE: }344// ANSE: return %[[VAL_11]] : !hlfir.expr<?x?x!fir.logical<4>>345// ANSE: }346 347// ELEMENTAL: %[[VAL_2:.*]] = arith.constant 1 : index348// ELEMENTAL: %[[VAL_3:.*]] = arith.constant false349// ELEMENTAL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<1>>) -> !fir.shape<2>350// ELEMENTAL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index351// ELEMENTAL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index352// ELEMENTAL: %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>353// ELEMENTAL: %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index354// ELEMENTAL: %[[VAL_9:.*]] = fir.shape %[[VAL_5]], %[[VAL_8]] : (index, index) -> !fir.shape<2>355// ELEMENTAL: %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<4>> {356// ELEMENTAL: ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):357// ELEMENTAL: %[[VAL_13:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>358// ELEMENTAL: %[[VAL_14:.*]] = fir.do_loop %[[VAL_15:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_13]]) -> (!fir.logical<4>) {359// ELEMENTAL: %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_15]] : (!hlfir.expr<?x?x!fir.logical<1>>, index, index) -> !fir.logical<1>360// ELEMENTAL: %[[VAL_18:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_15]], %[[VAL_12]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>361// ELEMENTAL: %[[VAL_19:.*]] = fir.convert %[[VAL_16]] : (!fir.logical<4>) -> i1362// ELEMENTAL: %[[VAL_20:.*]] = fir.convert %[[VAL_17]] : (!fir.logical<1>) -> i1363// ELEMENTAL: %[[VAL_21:.*]] = fir.convert %[[VAL_18]] : (!fir.logical<4>) -> i1364// ELEMENTAL: %[[VAL_22:.*]] = arith.andi %[[VAL_20]], %[[VAL_21]] : i1365// ELEMENTAL: %[[VAL_23:.*]] = arith.ori %[[VAL_19]], %[[VAL_22]] : i1366// ELEMENTAL: %[[VAL_24:.*]] = fir.convert %[[VAL_23]] : (i1) -> !fir.logical<4>367// ELEMENTAL: fir.result %[[VAL_24]] : !fir.logical<4>368// ELEMENTAL: }369// ELEMENTAL: hlfir.yield_element %[[VAL_14]] : !fir.logical<4>370// ELEMENTAL: }371// ELEMENTAL: return %[[VAL_10]] : !hlfir.expr<?x?x!fir.logical<4>>372// ELEMENTAL: }373 374func.func @matmul_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.expr<?xf16>) -> !hlfir.expr<?xf32> {375 %res = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x?xf32>, !hlfir.expr<?xf16>) -> !hlfir.expr<?xf32>376 return %res : !hlfir.expr<?xf32>377}378// ALL-LABEL: func.func @matmul_matrix_vector_real(379// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?xf32>,380// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?xf16>) -> !hlfir.expr<?xf32> {381 382// NOANSE: hlfir.matmul383 384// ANSE: %[[VAL_3:.*]] = arith.constant 1 : index385// ANSE: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32386// ANSE: %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>387// ANSE: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 0 : index} : (!fir.shape<2>) -> index388// ANSE: %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 1 : index} : (!fir.shape<2>) -> index389// ANSE: %[[VAL_8:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>390// ANSE: %[[VAL_9:.*]] = hlfir.eval_in_mem shape %[[VAL_8]] : (!fir.shape<1>) -> !hlfir.expr<?xf32> {391// ANSE: ^bb0(%[[VAL_10:.*]]: !fir.ref<!fir.array<?xf32>>):392// ANSE: %[[VAL_11:.*]] = fir.embox %[[VAL_10]](%[[VAL_8]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>393// ANSE: fir.do_loop %[[VAL_12:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] unordered {394// ANSE: %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_12]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>395// ANSE: hlfir.assign %[[VAL_4]] to %[[VAL_16]] : f32, !fir.ref<f32>396// ANSE: }397// ANSE: fir.do_loop %[[VAL_17:.*]] = %[[VAL_3]] to %[[VAL_7]] step %[[VAL_3]] {398// ANSE: fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {399// ANSE: %[[VAL_22:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_18]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>400// ANSE: %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<f32>401// ANSE: %[[VAL_24:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]], %[[VAL_17]] : (!hlfir.expr<?x?xf32>, index, index) -> f32402// ANSE: %[[VAL_25:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_17]] : (!hlfir.expr<?xf16>, index) -> f16403// ANSE: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (f16) -> f32404// ANSE: %[[VAL_27:.*]] = arith.mulf %[[VAL_24]], %[[VAL_26]] : f32405// ANSE: %[[VAL_28:.*]] = arith.addf %[[VAL_23]], %[[VAL_27]] : f32406// ANSE: hlfir.assign %[[VAL_28]] to %[[VAL_22]] : f32, !fir.ref<f32>407// ANSE: }408// ANSE: }409// ANSE: }410// ANSE: return %[[VAL_9]] : !hlfir.expr<?xf32>411// ANSE: }412 413// ELEMENTAL: %[[VAL_2:.*]] = arith.constant 1 : index414// ELEMENTAL: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32415// ELEMENTAL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>416// ELEMENTAL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index417// ELEMENTAL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index418// ELEMENTAL: %[[VAL_7:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>419// ELEMENTAL: %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {420// ELEMENTAL: ^bb0(%[[VAL_9:.*]]: index):421// ELEMENTAL: %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f32) {422// ELEMENTAL: %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_11]] : (!hlfir.expr<?x?xf32>, index, index) -> f32423// ELEMENTAL: %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?xf16>, index) -> f16424// ELEMENTAL: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (f16) -> f32425// ELEMENTAL: %[[VAL_16:.*]] = arith.mulf %[[VAL_13]], %[[VAL_15]] : f32426// ELEMENTAL: %[[VAL_17:.*]] = arith.addf %[[VAL_12]], %[[VAL_16]] : f32427// ELEMENTAL: fir.result %[[VAL_17]] : f32428// ELEMENTAL: }429// ELEMENTAL: hlfir.yield_element %[[VAL_10]] : f32430// ELEMENTAL: }431// ELEMENTAL: return %[[VAL_8]] : !hlfir.expr<?xf32>432// ELEMENTAL: }433 434func.func @matmul_vector_matrix_real(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.expr<?x?xf16>) -> !hlfir.expr<?xf32> {435 %res = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?xf32>, !hlfir.expr<?x?xf16>) -> !hlfir.expr<?xf32>436 return %res : !hlfir.expr<?xf32>437}438// ALL-LABEL: func.func @matmul_vector_matrix_real(439// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?xf32>,440// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?x?xf16>) -> !hlfir.expr<?xf32> {441 442// NOANSE: hlfir.matmul443 444// ANSE: %[[VAL_3:.*]] = arith.constant 1 : index445// ANSE: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32446// ANSE: %[[VAL_5:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>447// ANSE: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_5]] {dim = 0 : index} : (!fir.shape<1>) -> index448// ANSE: %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xf16>) -> !fir.shape<2>449// ANSE: %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index450// ANSE: %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>451// ANSE: %[[VAL_10:.*]] = hlfir.eval_in_mem shape %[[VAL_9]] : (!fir.shape<1>) -> !hlfir.expr<?xf32> {452// ANSE: ^bb0(%[[VAL_11:.*]]: !fir.ref<!fir.array<?xf32>>):453// ANSE: %[[VAL_12:.*]] = fir.embox %[[VAL_11]](%[[VAL_9]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>454// ANSE: fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] unordered {455// ANSE: %[[VAL_17:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_13]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>456// ANSE: hlfir.assign %[[VAL_4]] to %[[VAL_17]] : f32, !fir.ref<f32>457// ANSE: }458// ANSE: fir.do_loop %[[VAL_18:.*]] = %[[VAL_3]] to %[[VAL_6]] step %[[VAL_3]] {459// ANSE: fir.do_loop %[[VAL_19:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] {460// ANSE: %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]] (%[[VAL_19]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>461// ANSE: %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f32>462// ANSE: %[[VAL_25:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]] : (!hlfir.expr<?xf32>, index) -> f32463// ANSE: %[[VAL_26:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_18]], %[[VAL_19]] : (!hlfir.expr<?x?xf16>, index, index) -> f16464// ANSE: %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (f16) -> f32465// ANSE: %[[VAL_28:.*]] = arith.mulf %[[VAL_25]], %[[VAL_27]] : f32466// ANSE: %[[VAL_29:.*]] = arith.addf %[[VAL_24]], %[[VAL_28]] : f32467// ANSE: hlfir.assign %[[VAL_29]] to %[[VAL_23]] : f32, !fir.ref<f32>468// ANSE: }469// ANSE: }470// ANSE: }471// ANSE: return %[[VAL_10]] : !hlfir.expr<?xf32>472// ANSE: }473 474// ELEMENTAL: %[[VAL_2:.*]] = arith.constant 1 : index475// ELEMENTAL: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32476// ELEMENTAL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>477// ELEMENTAL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<1>) -> index478// ELEMENTAL: %[[VAL_6:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xf16>) -> !fir.shape<2>479// ELEMENTAL: %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_6]] {dim = 1 : index} : (!fir.shape<2>) -> index480// ELEMENTAL: %[[VAL_8:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>481// ELEMENTAL: %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {482// ELEMENTAL: ^bb0(%[[VAL_10:.*]]: index):483// ELEMENTAL: %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_13:.*]] = %[[VAL_3]]) -> (f32) {484// ELEMENTAL: %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_12]] : (!hlfir.expr<?xf32>, index) -> f32485// ELEMENTAL: %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?xf16>, index, index) -> f16486// ELEMENTAL: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (f16) -> f32487// ELEMENTAL: %[[VAL_17:.*]] = arith.mulf %[[VAL_14]], %[[VAL_16]] : f32488// ELEMENTAL: %[[VAL_18:.*]] = arith.addf %[[VAL_13]], %[[VAL_17]] : f32489// ELEMENTAL: fir.result %[[VAL_18]] : f32490// ELEMENTAL: }491// ELEMENTAL: hlfir.yield_element %[[VAL_11]] : f32492// ELEMENTAL: }493// ELEMENTAL: return %[[VAL_9]] : !hlfir.expr<?xf32>494// ELEMENTAL: }495 496func.func @matmul_transpose_matrix_matrix_integer(%arg0: !hlfir.expr<?x?xi16>, %arg1: !hlfir.expr<?x?xi32>) -> !hlfir.expr<?x?xi32> {497 %res = hlfir.matmul_transpose %arg0 %arg1 : (!hlfir.expr<?x?xi16>, !hlfir.expr<?x?xi32>) -> !hlfir.expr<?x?xi32>498 return %res : !hlfir.expr<?x?xi32>499}500// ALL-LABEL: func.func @matmul_transpose_matrix_matrix_integer(501// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?xi16>,502// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?x?xi32>) -> !hlfir.expr<?x?xi32> {503// ALL: %[[VAL_2:.*]] = arith.constant 1 : index504// ALL: %[[VAL_3:.*]] = arith.constant 0 : i32505// ALL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi16>) -> !fir.shape<2>506// ALL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index507// ALL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index508// ALL: %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?x?xi32>) -> !fir.shape<2>509// ALL: %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index510// ALL: %[[VAL_9:.*]] = fir.shape %[[VAL_6]], %[[VAL_8]] : (index, index) -> !fir.shape<2>511// ALL: %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {512// ALL: ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):513// ALL: %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_15:.*]] = %[[VAL_3]]) -> (i32) {514// ALL: %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_14]], %[[VAL_11]] : (!hlfir.expr<?x?xi16>, index, index) -> i16515// ALL: %[[VAL_17:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_14]], %[[VAL_12]] : (!hlfir.expr<?x?xi32>, index, index) -> i32516// ALL: %[[VAL_18:.*]] = fir.convert %[[VAL_16]] : (i16) -> i32517// ALL: %[[VAL_19:.*]] = arith.muli %[[VAL_18]], %[[VAL_17]] : i32518// ALL: %[[VAL_20:.*]] = arith.addi %[[VAL_15]], %[[VAL_19]] : i32519// ALL: fir.result %[[VAL_20]] : i32520// ALL: }521// ALL: hlfir.yield_element %[[VAL_13]] : i32522// ALL: }523// ALL: return %[[VAL_10]] : !hlfir.expr<?x?xi32>524// ALL: }525 526func.func @matmul_transpose_matrix_vector_real(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.expr<?xf16>) -> !hlfir.expr<?xf32> {527 %res = hlfir.matmul_transpose %arg0 %arg1 : (!hlfir.expr<?x?xf32>, !hlfir.expr<?xf16>) -> !hlfir.expr<?xf32>528 return %res : !hlfir.expr<?xf32>529}530// ALL-LABEL: func.func @matmul_transpose_matrix_vector_real(531// ALL-SAME: %[[VAL_0:.*]]: !hlfir.expr<?x?xf32>,532// ALL-SAME: %[[VAL_1:.*]]: !hlfir.expr<?xf16>) -> !hlfir.expr<?xf32> {533// ALL: %[[VAL_2:.*]] = arith.constant 1 : index534// ALL: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32535// ALL: %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>536// ALL: %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index537// ALL: %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index538// ALL: %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>539// ALL: %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {540// ALL: ^bb0(%[[VAL_9:.*]]: index):541// ALL: %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f32) {542// ALL: %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf32>, index, index) -> f32543// ALL: %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?xf16>, index) -> f16544// ALL: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (f16) -> f32545// ALL: %[[VAL_16:.*]] = arith.mulf %[[VAL_13]], %[[VAL_15]] : f32546// ALL: %[[VAL_17:.*]] = arith.addf %[[VAL_12]], %[[VAL_16]] : f32547// ALL: fir.result %[[VAL_17]] : f32548// ALL: }549// ALL: hlfir.yield_element %[[VAL_10]] : f32550// ALL: }551// ALL: return %[[VAL_8]] : !hlfir.expr<?xf32>552// ALL: }553 554// Check that the inner-product loop uses the best known extent555// of the input matrices:556func.func @matmul_matrix_matrix_deduce_bounds(%arg0: !hlfir.expr<?x10xi16>, %arg1: !hlfir.expr<?x?xi32>, %arg2: !hlfir.expr<10x?xi16>) -> (!hlfir.expr<?x?xi32>, !hlfir.expr<?x?xi32>) {557 %res1 = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x10xi16>, !hlfir.expr<?x?xi32>) -> !hlfir.expr<?x?xi32>558 %res2 = hlfir.matmul %arg1 %arg2 : (!hlfir.expr<?x?xi32>, !hlfir.expr<10x?xi16>) -> !hlfir.expr<?x?xi32>559 return %res1, %res2 : !hlfir.expr<?x?xi32>, !hlfir.expr<?x?xi32>560}561// ALL-LABEL: func.func @matmul_matrix_matrix_deduce_bounds(562 563// ANSE: %[[VAL_6:.*]] = arith.constant 10 : index564// ANSE: hlfir.eval_in_mem shape {{.*}}565// ANSE: fir.do_loop566// ANSE: fir.do_loop567// ANSE: fir.do_loop %{{.*}} = %{{.*}} to %[[VAL_6]]568// ANSE: fir.do_loop569// ANSE: fir.do_loop570// ANSE: hlfir.eval_in_mem shape {{.*}}571// ANSE: fir.do_loop572// ANSE: fir.do_loop573// ANSE: fir.do_loop %{{.*}} = %{{.*}} to %[[VAL_6]]574// ANSE: fir.do_loop575// ANSE: fir.do_loop576 577// ELEMENTAL: %[[VAL_5:.*]] = arith.constant 10 : index578// ELEMENTAL: hlfir.elemental %{{.*}}579// ELEMENTAL: fir.do_loop %{{.*}} = %{{.*}} to %[[VAL_5]]580// ELEMENTAL: hlfir.elemental %{{.*}}581// ELEMENTAL: fir.do_loop %{{.*}} = %{{.*}} to %[[VAL_5]]582