380 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.4 */5 6#include <linux/dim.h>7#include <linux/rtnetlink.h>8 9/*10 * Net DIM profiles:11 * There are different set of profiles for each CQ period mode.12 * There are different set of profiles for RX/TX CQs.13 * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES14 */15#define NET_DIM_RX_EQE_PROFILES { \16 {.usec = 1, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \17 {.usec = 8, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \18 {.usec = 64, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \19 {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \20 {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,} \21}22 23#define NET_DIM_RX_CQE_PROFILES { \24 {.usec = 2, .pkts = 256,}, \25 {.usec = 8, .pkts = 128,}, \26 {.usec = 16, .pkts = 64,}, \27 {.usec = 32, .pkts = 64,}, \28 {.usec = 64, .pkts = 64,} \29}30 31#define NET_DIM_TX_EQE_PROFILES { \32 {.usec = 1, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \33 {.usec = 8, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \34 {.usec = 32, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \35 {.usec = 64, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \36 {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,} \37}38 39#define NET_DIM_TX_CQE_PROFILES { \40 {.usec = 5, .pkts = 128,}, \41 {.usec = 8, .pkts = 64,}, \42 {.usec = 16, .pkts = 32,}, \43 {.usec = 32, .pkts = 32,}, \44 {.usec = 64, .pkts = 32,} \45}46 47static const struct dim_cq_moder48rx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {49 NET_DIM_RX_EQE_PROFILES,50 NET_DIM_RX_CQE_PROFILES,51};52 53static const struct dim_cq_moder54tx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {55 NET_DIM_TX_EQE_PROFILES,56 NET_DIM_TX_CQE_PROFILES,57};58 59struct dim_cq_moder60net_dim_get_rx_moderation(u8 cq_period_mode, int ix)61{62 struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];63 64 cq_moder.cq_period_mode = cq_period_mode;65 return cq_moder;66}67EXPORT_SYMBOL(net_dim_get_rx_moderation);68 69struct dim_cq_moder70net_dim_get_def_rx_moderation(u8 cq_period_mode)71{72 u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?73 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;74 75 return net_dim_get_rx_moderation(cq_period_mode, profile_ix);76}77EXPORT_SYMBOL(net_dim_get_def_rx_moderation);78 79struct dim_cq_moder80net_dim_get_tx_moderation(u8 cq_period_mode, int ix)81{82 struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];83 84 cq_moder.cq_period_mode = cq_period_mode;85 return cq_moder;86}87EXPORT_SYMBOL(net_dim_get_tx_moderation);88 89struct dim_cq_moder90net_dim_get_def_tx_moderation(u8 cq_period_mode)91{92 u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?93 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;94 95 return net_dim_get_tx_moderation(cq_period_mode, profile_ix);96}97EXPORT_SYMBOL(net_dim_get_def_tx_moderation);98 99int net_dim_init_irq_moder(struct net_device *dev, u8 profile_flags,100 u8 coal_flags, u8 rx_mode, u8 tx_mode,101 void (*rx_dim_work)(struct work_struct *work),102 void (*tx_dim_work)(struct work_struct *work))103{104 struct dim_cq_moder *rxp = NULL, *txp;105 struct dim_irq_moder *moder;106 int len;107 108 dev->irq_moder = kzalloc(sizeof(*dev->irq_moder), GFP_KERNEL);109 if (!dev->irq_moder)110 return -ENOMEM;111 112 moder = dev->irq_moder;113 len = NET_DIM_PARAMS_NUM_PROFILES * sizeof(*moder->rx_profile);114 115 moder->coal_flags = coal_flags;116 moder->profile_flags = profile_flags;117 118 if (profile_flags & DIM_PROFILE_RX) {119 moder->rx_dim_work = rx_dim_work;120 moder->dim_rx_mode = rx_mode;121 rxp = kmemdup(rx_profile[rx_mode], len, GFP_KERNEL);122 if (!rxp)123 goto free_moder;124 125 rcu_assign_pointer(moder->rx_profile, rxp);126 }127 128 if (profile_flags & DIM_PROFILE_TX) {129 moder->tx_dim_work = tx_dim_work;130 moder->dim_tx_mode = tx_mode;131 txp = kmemdup(tx_profile[tx_mode], len, GFP_KERNEL);132 if (!txp)133 goto free_rxp;134 135 rcu_assign_pointer(moder->tx_profile, txp);136 }137 138 return 0;139 140free_rxp:141 kfree(rxp);142free_moder:143 kfree(moder);144 return -ENOMEM;145}146EXPORT_SYMBOL(net_dim_init_irq_moder);147 148/* RTNL lock is held. */149void net_dim_free_irq_moder(struct net_device *dev)150{151 struct dim_cq_moder *rxp, *txp;152 153 if (!dev->irq_moder)154 return;155 156 rxp = rtnl_dereference(dev->irq_moder->rx_profile);157 txp = rtnl_dereference(dev->irq_moder->tx_profile);158 159 rcu_assign_pointer(dev->irq_moder->rx_profile, NULL);160 rcu_assign_pointer(dev->irq_moder->tx_profile, NULL);161 162 kfree_rcu(rxp, rcu);163 kfree_rcu(txp, rcu);164 kfree(dev->irq_moder);165}166EXPORT_SYMBOL(net_dim_free_irq_moder);167 168void net_dim_setting(struct net_device *dev, struct dim *dim, bool is_tx)169{170 struct dim_irq_moder *irq_moder = dev->irq_moder;171 172 if (!irq_moder)173 return;174 175 if (is_tx) {176 INIT_WORK(&dim->work, irq_moder->tx_dim_work);177 dim->mode = READ_ONCE(irq_moder->dim_tx_mode);178 return;179 }180 181 INIT_WORK(&dim->work, irq_moder->rx_dim_work);182 dim->mode = READ_ONCE(irq_moder->dim_rx_mode);183}184EXPORT_SYMBOL(net_dim_setting);185 186void net_dim_work_cancel(struct dim *dim)187{188 cancel_work_sync(&dim->work);189}190EXPORT_SYMBOL(net_dim_work_cancel);191 192struct dim_cq_moder net_dim_get_rx_irq_moder(struct net_device *dev,193 struct dim *dim)194{195 struct dim_cq_moder res, *profile;196 197 rcu_read_lock();198 profile = rcu_dereference(dev->irq_moder->rx_profile);199 res = profile[dim->profile_ix];200 rcu_read_unlock();201 202 res.cq_period_mode = dim->mode;203 204 return res;205}206EXPORT_SYMBOL(net_dim_get_rx_irq_moder);207 208struct dim_cq_moder net_dim_get_tx_irq_moder(struct net_device *dev,209 struct dim *dim)210{211 struct dim_cq_moder res, *profile;212 213 rcu_read_lock();214 profile = rcu_dereference(dev->irq_moder->tx_profile);215 res = profile[dim->profile_ix];216 rcu_read_unlock();217 218 res.cq_period_mode = dim->mode;219 220 return res;221}222EXPORT_SYMBOL(net_dim_get_tx_irq_moder);223 224void net_dim_set_rx_mode(struct net_device *dev, u8 rx_mode)225{226 WRITE_ONCE(dev->irq_moder->dim_rx_mode, rx_mode);227}228EXPORT_SYMBOL(net_dim_set_rx_mode);229 230void net_dim_set_tx_mode(struct net_device *dev, u8 tx_mode)231{232 WRITE_ONCE(dev->irq_moder->dim_tx_mode, tx_mode);233}234EXPORT_SYMBOL(net_dim_set_tx_mode);235 236static int net_dim_step(struct dim *dim)237{238 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))239 return DIM_TOO_TIRED;240 241 switch (dim->tune_state) {242 case DIM_PARKING_ON_TOP:243 case DIM_PARKING_TIRED:244 break;245 case DIM_GOING_RIGHT:246 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))247 return DIM_ON_EDGE;248 dim->profile_ix++;249 dim->steps_right++;250 break;251 case DIM_GOING_LEFT:252 if (dim->profile_ix == 0)253 return DIM_ON_EDGE;254 dim->profile_ix--;255 dim->steps_left++;256 break;257 }258 259 dim->tired++;260 return DIM_STEPPED;261}262 263static void net_dim_exit_parking(struct dim *dim)264{265 dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT;266 net_dim_step(dim);267}268 269static int net_dim_stats_compare(struct dim_stats *curr,270 struct dim_stats *prev)271{272 if (!prev->bpms)273 return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME;274 275 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))276 return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER :277 DIM_STATS_WORSE;278 279 if (!prev->ppms)280 return curr->ppms ? DIM_STATS_BETTER :281 DIM_STATS_SAME;282 283 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))284 return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER :285 DIM_STATS_WORSE;286 287 if (!prev->epms)288 return DIM_STATS_SAME;289 290 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))291 return (curr->epms < prev->epms) ? DIM_STATS_BETTER :292 DIM_STATS_WORSE;293 294 return DIM_STATS_SAME;295}296 297static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim)298{299 int prev_state = dim->tune_state;300 int prev_ix = dim->profile_ix;301 int stats_res;302 int step_res;303 304 switch (dim->tune_state) {305 case DIM_PARKING_ON_TOP:306 stats_res = net_dim_stats_compare(curr_stats,307 &dim->prev_stats);308 if (stats_res != DIM_STATS_SAME)309 net_dim_exit_parking(dim);310 break;311 312 case DIM_PARKING_TIRED:313 dim->tired--;314 if (!dim->tired)315 net_dim_exit_parking(dim);316 break;317 318 case DIM_GOING_RIGHT:319 case DIM_GOING_LEFT:320 stats_res = net_dim_stats_compare(curr_stats,321 &dim->prev_stats);322 if (stats_res != DIM_STATS_BETTER)323 dim_turn(dim);324 325 if (dim_on_top(dim)) {326 dim_park_on_top(dim);327 break;328 }329 330 step_res = net_dim_step(dim);331 switch (step_res) {332 case DIM_ON_EDGE:333 dim_park_on_top(dim);334 break;335 case DIM_TOO_TIRED:336 dim_park_tired(dim);337 break;338 }339 340 break;341 }342 343 if (prev_state != DIM_PARKING_ON_TOP ||344 dim->tune_state != DIM_PARKING_ON_TOP)345 dim->prev_stats = *curr_stats;346 347 return dim->profile_ix != prev_ix;348}349 350void net_dim(struct dim *dim, struct dim_sample end_sample)351{352 struct dim_stats curr_stats;353 u16 nevents;354 355 switch (dim->state) {356 case DIM_MEASURE_IN_PROGRESS:357 nevents = BIT_GAP(BITS_PER_TYPE(u16),358 end_sample.event_ctr,359 dim->start_sample.event_ctr);360 if (nevents < DIM_NEVENTS)361 break;362 if (!dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats))363 break;364 if (net_dim_decision(&curr_stats, dim)) {365 dim->state = DIM_APPLY_NEW_PROFILE;366 schedule_work(&dim->work);367 break;368 }369 fallthrough;370 case DIM_START_MEASURE:371 dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr,372 end_sample.byte_ctr, &dim->start_sample);373 dim->state = DIM_MEASURE_IN_PROGRESS;374 break;375 case DIM_APPLY_NEW_PROFILE:376 break;377 }378}379EXPORT_SYMBOL(net_dim);380