466 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 Intel Corporation */3 4#include "igc.h"5#include "igc_hw.h"6#include "igc_tsn.h"7 8static bool is_any_launchtime(struct igc_adapter *adapter)9{10 int i;11 12 for (i = 0; i < adapter->num_tx_queues; i++) {13 struct igc_ring *ring = adapter->tx_ring[i];14 15 if (ring->launchtime_enable)16 return true;17 }18 19 return false;20}21 22static bool is_cbs_enabled(struct igc_adapter *adapter)23{24 int i;25 26 for (i = 0; i < adapter->num_tx_queues; i++) {27 struct igc_ring *ring = adapter->tx_ring[i];28 29 if (ring->cbs_enable)30 return true;31 }32 33 return false;34}35 36static unsigned int igc_tsn_new_flags(struct igc_adapter *adapter)37{38 unsigned int new_flags = adapter->flags & ~IGC_FLAG_TSN_ANY_ENABLED;39 40 if (adapter->taprio_offload_enable)41 new_flags |= IGC_FLAG_TSN_QBV_ENABLED;42 43 if (is_any_launchtime(adapter))44 new_flags |= IGC_FLAG_TSN_QBV_ENABLED;45 46 if (is_cbs_enabled(adapter))47 new_flags |= IGC_FLAG_TSN_QAV_ENABLED;48 49 if (adapter->strict_priority_enable)50 new_flags |= IGC_FLAG_TSN_LEGACY_ENABLED;51 52 return new_flags;53}54 55static bool igc_tsn_is_tx_mode_in_tsn(struct igc_adapter *adapter)56{57 struct igc_hw *hw = &adapter->hw;58 59 return !!(rd32(IGC_TQAVCTRL) & IGC_TQAVCTRL_TRANSMIT_MODE_TSN);60}61 62void igc_tsn_adjust_txtime_offset(struct igc_adapter *adapter)63{64 struct igc_hw *hw = &adapter->hw;65 u16 txoffset;66 67 if (!igc_tsn_is_tx_mode_in_tsn(adapter))68 return;69 70 switch (adapter->link_speed) {71 case SPEED_10:72 txoffset = IGC_TXOFFSET_SPEED_10;73 break;74 case SPEED_100:75 txoffset = IGC_TXOFFSET_SPEED_100;76 break;77 case SPEED_1000:78 txoffset = IGC_TXOFFSET_SPEED_1000;79 break;80 case SPEED_2500:81 txoffset = IGC_TXOFFSET_SPEED_2500;82 break;83 default:84 txoffset = 0;85 break;86 }87 88 wr32(IGC_GTXOFFSET, txoffset);89}90 91static void igc_tsn_restore_retx_default(struct igc_adapter *adapter)92{93 struct igc_hw *hw = &adapter->hw;94 u32 retxctl;95 96 retxctl = rd32(IGC_RETX_CTL) & IGC_RETX_CTL_WATERMARK_MASK;97 wr32(IGC_RETX_CTL, retxctl);98}99 100bool igc_tsn_is_taprio_activated_by_user(struct igc_adapter *adapter)101{102 struct igc_hw *hw = &adapter->hw;103 104 return (rd32(IGC_BASET_H) || rd32(IGC_BASET_L)) &&105 adapter->taprio_offload_enable;106}107 108static void igc_tsn_tx_arb(struct igc_adapter *adapter, u16 *queue_per_tc)109{110 struct igc_hw *hw = &adapter->hw;111 u32 txarb;112 113 txarb = rd32(IGC_TXARB);114 115 txarb &= ~(IGC_TXARB_TXQ_PRIO_0_MASK |116 IGC_TXARB_TXQ_PRIO_1_MASK |117 IGC_TXARB_TXQ_PRIO_2_MASK |118 IGC_TXARB_TXQ_PRIO_3_MASK);119 120 txarb |= IGC_TXARB_TXQ_PRIO_0(queue_per_tc[3]);121 txarb |= IGC_TXARB_TXQ_PRIO_1(queue_per_tc[2]);122 txarb |= IGC_TXARB_TXQ_PRIO_2(queue_per_tc[1]);123 txarb |= IGC_TXARB_TXQ_PRIO_3(queue_per_tc[0]);124 125 wr32(IGC_TXARB, txarb);126}127 128/* Returns the TSN specific registers to their default values after129 * the adapter is reset.130 */131static int igc_tsn_disable_offload(struct igc_adapter *adapter)132{133 u16 queue_per_tc[4] = { 3, 2, 1, 0 };134 struct igc_hw *hw = &adapter->hw;135 u32 tqavctrl;136 int i;137 138 wr32(IGC_GTXOFFSET, 0);139 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);140 wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_DEFAULT);141 142 if (igc_is_device_id_i226(hw))143 igc_tsn_restore_retx_default(adapter);144 145 tqavctrl = rd32(IGC_TQAVCTRL);146 tqavctrl &= ~(IGC_TQAVCTRL_TRANSMIT_MODE_TSN |147 IGC_TQAVCTRL_ENHANCED_QAV | IGC_TQAVCTRL_FUTSCDDIS);148 149 wr32(IGC_TQAVCTRL, tqavctrl);150 151 for (i = 0; i < adapter->num_tx_queues; i++) {152 wr32(IGC_TXQCTL(i), 0);153 wr32(IGC_STQT(i), 0);154 wr32(IGC_ENDQT(i), NSEC_PER_SEC);155 }156 157 wr32(IGC_QBVCYCLET_S, 0);158 wr32(IGC_QBVCYCLET, NSEC_PER_SEC);159 160 /* Reset mqprio TC configuration. */161 netdev_reset_tc(adapter->netdev);162 163 /* Restore the default Tx arbitration: Priority 0 has the highest164 * priority and is assigned to queue 0 and so on and so forth.165 */166 igc_tsn_tx_arb(adapter, queue_per_tc);167 168 adapter->flags &= ~IGC_FLAG_TSN_QBV_ENABLED;169 adapter->flags &= ~IGC_FLAG_TSN_LEGACY_ENABLED;170 171 return 0;172}173 174/* To partially fix i226 HW errata, reduce MAC internal buffering from 192 Bytes175 * to 88 Bytes by setting RETX_CTL register using the recommendation from:176 * a) Ethernet Controller I225/I226 Specification Update Rev 2.1177 * Item 9: TSN: Packet Transmission Might Cross the Qbv Window178 * b) I225/6 SW User Manual Rev 1.2.4: Section 8.11.5 Retry Buffer Control179 */180static void igc_tsn_set_retx_qbvfullthreshold(struct igc_adapter *adapter)181{182 struct igc_hw *hw = &adapter->hw;183 u32 retxctl, watermark;184 185 retxctl = rd32(IGC_RETX_CTL);186 watermark = retxctl & IGC_RETX_CTL_WATERMARK_MASK;187 /* Set QBVFULLTH value using watermark and set QBVFULLEN */188 retxctl |= (watermark << IGC_RETX_CTL_QBVFULLTH_SHIFT) |189 IGC_RETX_CTL_QBVFULLEN;190 wr32(IGC_RETX_CTL, retxctl);191}192 193static int igc_tsn_enable_offload(struct igc_adapter *adapter)194{195 struct igc_hw *hw = &adapter->hw;196 u32 tqavctrl, baset_l, baset_h;197 u32 sec, nsec, cycle;198 ktime_t base_time, systim;199 int i;200 201 wr32(IGC_TSAUXC, 0);202 wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_TSN);203 wr32(IGC_TXPBS, IGC_TXPBSIZE_TSN);204 205 if (igc_is_device_id_i226(hw))206 igc_tsn_set_retx_qbvfullthreshold(adapter);207 208 if (adapter->strict_priority_enable) {209 int err;210 211 err = netdev_set_num_tc(adapter->netdev, adapter->num_tc);212 if (err)213 return err;214 215 for (i = 0; i < adapter->num_tc; i++) {216 err = netdev_set_tc_queue(adapter->netdev, i, 1,217 adapter->queue_per_tc[i]);218 if (err)219 return err;220 }221 222 /* In case the card is configured with less than four queues. */223 for (; i < IGC_MAX_TX_QUEUES; i++)224 adapter->queue_per_tc[i] = i;225 226 /* Configure queue priorities according to the user provided227 * mapping.228 */229 igc_tsn_tx_arb(adapter, adapter->queue_per_tc);230 231 /* Enable legacy TSN mode which will do strict priority without232 * any other TSN features.233 */234 tqavctrl = rd32(IGC_TQAVCTRL);235 tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN;236 tqavctrl &= ~IGC_TQAVCTRL_ENHANCED_QAV;237 wr32(IGC_TQAVCTRL, tqavctrl);238 239 return 0;240 }241 242 for (i = 0; i < adapter->num_tx_queues; i++) {243 struct igc_ring *ring = adapter->tx_ring[i];244 u32 txqctl = 0;245 u16 cbs_value;246 u32 tqavcc;247 248 wr32(IGC_STQT(i), ring->start_time);249 wr32(IGC_ENDQT(i), ring->end_time);250 251 if (adapter->taprio_offload_enable) {252 /* If taprio_offload_enable is set we are in "taprio"253 * mode and we need to be strict about the254 * cycles: only transmit a packet if it can be255 * completed during that cycle.256 *257 * If taprio_offload_enable is NOT true when258 * enabling TSN offload, the cycle should have259 * no external effects, but is only used internally260 * to adapt the base time register after a second261 * has passed.262 *263 * Enabling strict mode in this case would264 * unnecessarily prevent the transmission of265 * certain packets (i.e. at the boundary of a266 * second) and thus interfere with the launchtime267 * feature that promises transmission at a268 * certain point in time.269 */270 txqctl |= IGC_TXQCTL_STRICT_CYCLE |271 IGC_TXQCTL_STRICT_END;272 }273 274 if (ring->launchtime_enable)275 txqctl |= IGC_TXQCTL_QUEUE_MODE_LAUNCHT;276 277 /* Skip configuring CBS for Q2 and Q3 */278 if (i > 1)279 goto skip_cbs;280 281 if (ring->cbs_enable) {282 if (i == 0)283 txqctl |= IGC_TXQCTL_QAV_SEL_CBS0;284 else285 txqctl |= IGC_TXQCTL_QAV_SEL_CBS1;286 287 /* According to i225 datasheet section 7.5.2.7, we288 * should set the 'idleSlope' field from TQAVCC289 * register following the equation:290 *291 * value = link-speed 0x7736 * BW * 0.2292 * ---------- * ----------------- (E1)293 * 100Mbps 2.5294 *295 * Note that 'link-speed' is in Mbps.296 *297 * 'BW' is the percentage bandwidth out of full298 * link speed which can be found with the299 * following equation. Note that idleSlope here300 * is the parameter from this function301 * which is in kbps.302 *303 * BW = idleSlope304 * ----------------- (E2)305 * link-speed * 1000306 *307 * That said, we can come up with a generic308 * equation to calculate the value we should set309 * it TQAVCC register by replacing 'BW' in E1 by E2.310 * The resulting equation is:311 *312 * value = link-speed * 0x7736 * idleSlope * 0.2313 * ------------------------------------- (E3)314 * 100 * 2.5 * link-speed * 1000315 *316 * 'link-speed' is present in both sides of the317 * fraction so it is canceled out. The final318 * equation is the following:319 *320 * value = idleSlope * 61036321 * ----------------- (E4)322 * 2500000323 *324 * NOTE: For i225, given the above, we can see325 * that idleslope is represented in326 * 40.959433 kbps units by the value at327 * the TQAVCC register (2.5Gbps / 61036),328 * which reduces the granularity for329 * idleslope increments.330 *331 * In i225 controller, the sendSlope and loCredit332 * parameters from CBS are not configurable333 * by software so we don't do any334 * 'controller configuration' in respect to335 * these parameters.336 */337 cbs_value = DIV_ROUND_UP_ULL(ring->idleslope338 * 61036ULL, 2500000);339 340 tqavcc = rd32(IGC_TQAVCC(i));341 tqavcc &= ~IGC_TQAVCC_IDLESLOPE_MASK;342 tqavcc |= cbs_value | IGC_TQAVCC_KEEP_CREDITS;343 wr32(IGC_TQAVCC(i), tqavcc);344 345 wr32(IGC_TQAVHC(i),346 0x80000000 + ring->hicredit * 0x7736);347 } else {348 /* Disable any CBS for the queue */349 txqctl &= ~(IGC_TXQCTL_QAV_SEL_MASK);350 351 /* Set idleSlope to zero. */352 tqavcc = rd32(IGC_TQAVCC(i));353 tqavcc &= ~(IGC_TQAVCC_IDLESLOPE_MASK |354 IGC_TQAVCC_KEEP_CREDITS);355 wr32(IGC_TQAVCC(i), tqavcc);356 357 /* Set hiCredit to zero. */358 wr32(IGC_TQAVHC(i), 0);359 }360skip_cbs:361 wr32(IGC_TXQCTL(i), txqctl);362 }363 364 tqavctrl = rd32(IGC_TQAVCTRL) & ~IGC_TQAVCTRL_FUTSCDDIS;365 366 tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN | IGC_TQAVCTRL_ENHANCED_QAV;367 368 adapter->qbv_count++;369 370 cycle = adapter->cycle_time;371 base_time = adapter->base_time;372 373 nsec = rd32(IGC_SYSTIML);374 sec = rd32(IGC_SYSTIMH);375 376 systim = ktime_set(sec, nsec);377 if (ktime_compare(systim, base_time) > 0) {378 s64 n = div64_s64(ktime_sub_ns(systim, base_time), cycle);379 380 base_time = ktime_add_ns(base_time, (n + 1) * cycle);381 } else {382 if (igc_is_device_id_i226(hw)) {383 ktime_t adjust_time, expires_time;384 385 /* According to datasheet section 7.5.2.9.3.3, FutScdDis bit386 * has to be configured before the cycle time and base time.387 * Tx won't hang if a GCL is already running,388 * so in this case we don't need to set FutScdDis.389 */390 if (!(rd32(IGC_BASET_H) || rd32(IGC_BASET_L)))391 tqavctrl |= IGC_TQAVCTRL_FUTSCDDIS;392 393 nsec = rd32(IGC_SYSTIML);394 sec = rd32(IGC_SYSTIMH);395 systim = ktime_set(sec, nsec);396 397 adjust_time = adapter->base_time;398 expires_time = ktime_sub_ns(adjust_time, systim);399 hrtimer_start(&adapter->hrtimer, expires_time, HRTIMER_MODE_REL);400 }401 }402 403 wr32(IGC_TQAVCTRL, tqavctrl);404 405 wr32(IGC_QBVCYCLET_S, cycle);406 wr32(IGC_QBVCYCLET, cycle);407 408 baset_h = div_s64_rem(base_time, NSEC_PER_SEC, &baset_l);409 wr32(IGC_BASET_H, baset_h);410 411 /* In i226, Future base time is only supported when FutScdDis bit412 * is enabled and only active for re-configuration.413 * In this case, initialize the base time with zero to create414 * "re-configuration" scenario then only set the desired base time.415 */416 if (tqavctrl & IGC_TQAVCTRL_FUTSCDDIS)417 wr32(IGC_BASET_L, 0);418 wr32(IGC_BASET_L, baset_l);419 420 return 0;421}422 423int igc_tsn_reset(struct igc_adapter *adapter)424{425 unsigned int new_flags;426 int err = 0;427 428 new_flags = igc_tsn_new_flags(adapter);429 430 if (!(new_flags & IGC_FLAG_TSN_ANY_ENABLED))431 return igc_tsn_disable_offload(adapter);432 433 err = igc_tsn_enable_offload(adapter);434 if (err < 0)435 return err;436 437 adapter->flags = new_flags;438 439 return err;440}441 442static bool igc_tsn_will_tx_mode_change(struct igc_adapter *adapter)443{444 bool any_tsn_enabled = !!(igc_tsn_new_flags(adapter) &445 IGC_FLAG_TSN_ANY_ENABLED);446 447 return (any_tsn_enabled && !igc_tsn_is_tx_mode_in_tsn(adapter)) ||448 (!any_tsn_enabled && igc_tsn_is_tx_mode_in_tsn(adapter));449}450 451int igc_tsn_offload_apply(struct igc_adapter *adapter)452{453 /* Per I225/6 HW Design Section 7.5.2.1 guideline, if tx mode change454 * from legacy->tsn or tsn->legacy, then reset adapter is needed.455 */456 if (netif_running(adapter->netdev) &&457 igc_tsn_will_tx_mode_change(adapter)) {458 schedule_work(&adapter->reset_task);459 return 0;460 }461 462 igc_tsn_reset(adapter);463 464 return 0;465}466