543 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * This file is based on code from OCTEON SDK by Cavium Networks.4 *5 * Copyright (c) 2003-2010 Cavium Networks6 */7 8#include <linux/module.h>9#include <linux/kernel.h>10#include <linux/cache.h>11#include <linux/cpumask.h>12#include <linux/netdevice.h>13#include <linux/etherdevice.h>14#include <linux/ip.h>15#include <linux/string.h>16#include <linux/prefetch.h>17#include <linux/ratelimit.h>18#include <linux/smp.h>19#include <linux/interrupt.h>20#include <net/dst.h>21#ifdef CONFIG_XFRM22#include <linux/xfrm.h>23#include <net/xfrm.h>24#endif /* CONFIG_XFRM */25 26#include "octeon-ethernet.h"27#include "ethernet-defines.h"28#include "ethernet-mem.h"29#include "ethernet-rx.h"30#include "ethernet-util.h"31 32static atomic_t oct_rx_ready = ATOMIC_INIT(0);33 34static struct oct_rx_group {35 int irq;36 int group;37 struct napi_struct napi;38} oct_rx_group[16];39 40/**41 * cvm_oct_do_interrupt - interrupt handler.42 * @irq: Interrupt number.43 * @napi_id: Cookie to identify the NAPI instance.44 *45 * The interrupt occurs whenever the POW has packets in our group.46 *47 */48static irqreturn_t cvm_oct_do_interrupt(int irq, void *napi_id)49{50 /* Disable the IRQ and start napi_poll. */51 disable_irq_nosync(irq);52 napi_schedule(napi_id);53 54 return IRQ_HANDLED;55}56 57/**58 * cvm_oct_check_rcv_error - process receive errors59 * @work: Work queue entry pointing to the packet.60 *61 * Returns Non-zero if the packet can be dropped, zero otherwise.62 */63static inline int cvm_oct_check_rcv_error(struct cvmx_wqe *work)64{65 int port;66 67 if (octeon_has_feature(OCTEON_FEATURE_PKND))68 port = work->word0.pip.cn68xx.pknd;69 else70 port = work->word1.cn38xx.ipprt;71 72 if ((work->word2.snoip.err_code == 10) && (work->word1.len <= 64))73 /*74 * Ignore length errors on min size packets. Some75 * equipment incorrectly pads packets to 64+4FCS76 * instead of 60+4FCS. Note these packets still get77 * counted as frame errors.78 */79 return 0;80 81 if (work->word2.snoip.err_code == 5 ||82 work->word2.snoip.err_code == 7) {83 /*84 * We received a packet with either an alignment error85 * or a FCS error. This may be signalling that we are86 * running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK]87 * off. If this is the case we need to parse the88 * packet to determine if we can remove a non spec89 * preamble and generate a correct packet.90 */91 int interface = cvmx_helper_get_interface_num(port);92 int index = cvmx_helper_get_interface_index_num(port);93 union cvmx_gmxx_rxx_frm_ctl gmxx_rxx_frm_ctl;94 95 gmxx_rxx_frm_ctl.u64 =96 cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface));97 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) {98 u8 *ptr =99 cvmx_phys_to_ptr(work->packet_ptr.s.addr);100 int i = 0;101 102 while (i < work->word1.len - 1) {103 if (*ptr != 0x55)104 break;105 ptr++;106 i++;107 }108 109 if (*ptr == 0xd5) {110 /* Port received 0xd5 preamble */111 work->packet_ptr.s.addr += i + 1;112 work->word1.len -= i + 5;113 return 0;114 }115 116 if ((*ptr & 0xf) == 0xd) {117 /* Port received 0xd preamble */118 work->packet_ptr.s.addr += i;119 work->word1.len -= i + 4;120 for (i = 0; i < work->word1.len; i++) {121 *ptr =122 ((*ptr & 0xf0) >> 4) |123 ((*(ptr + 1) & 0xf) << 4);124 ptr++;125 }126 return 0;127 }128 129 printk_ratelimited("Port %d unknown preamble, packet dropped\n",130 port);131 cvm_oct_free_work(work);132 return 1;133 }134 }135 136 printk_ratelimited("Port %d receive error code %d, packet dropped\n",137 port, work->word2.snoip.err_code);138 cvm_oct_free_work(work);139 return 1;140}141 142static void copy_segments_to_skb(struct cvmx_wqe *work, struct sk_buff *skb)143{144 int segments = work->word2.s.bufs;145 union cvmx_buf_ptr segment_ptr = work->packet_ptr;146 int len = work->word1.len;147 int segment_size;148 149 while (segments--) {150 union cvmx_buf_ptr next_ptr;151 152 next_ptr = *(union cvmx_buf_ptr *)153 cvmx_phys_to_ptr(segment_ptr.s.addr - 8);154 155 /*156 * Octeon Errata PKI-100: The segment size is wrong.157 *158 * Until it is fixed, calculate the segment size based on159 * the packet pool buffer size.160 * When it is fixed, the following line should be replaced161 * with this one:162 * int segment_size = segment_ptr.s.size;163 */164 segment_size =165 CVMX_FPA_PACKET_POOL_SIZE -166 (segment_ptr.s.addr -167 (((segment_ptr.s.addr >> 7) -168 segment_ptr.s.back) << 7));169 170 /* Don't copy more than what is left in the packet */171 if (segment_size > len)172 segment_size = len;173 174 /* Copy the data into the packet */175 skb_put_data(skb, cvmx_phys_to_ptr(segment_ptr.s.addr),176 segment_size);177 len -= segment_size;178 segment_ptr = next_ptr;179 }180}181 182static int cvm_oct_poll(struct oct_rx_group *rx_group, int budget)183{184 const int coreid = cvmx_get_core_num();185 u64 old_group_mask;186 u64 old_scratch;187 int rx_count = 0;188 int did_work_request = 0;189 int packet_not_copied;190 191 /* Prefetch cvm_oct_device since we know we need it soon */192 prefetch(cvm_oct_device);193 194 if (USE_ASYNC_IOBDMA) {195 /* Save scratch in case userspace is using it */196 CVMX_SYNCIOBDMA;197 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH);198 }199 200 /* Only allow work for our group (and preserve priorities) */201 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {202 old_group_mask = cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid));203 cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid),204 BIT(rx_group->group));205 cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */206 } else {207 old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));208 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),209 (old_group_mask & ~0xFFFFull) |210 BIT(rx_group->group));211 }212 213 if (USE_ASYNC_IOBDMA) {214 cvmx_pow_work_request_async(CVMX_SCR_SCRATCH, CVMX_POW_NO_WAIT);215 did_work_request = 1;216 }217 218 while (rx_count < budget) {219 struct sk_buff *skb = NULL;220 struct sk_buff **pskb = NULL;221 int skb_in_hw;222 struct cvmx_wqe *work;223 int port;224 225 if (USE_ASYNC_IOBDMA && did_work_request)226 work = cvmx_pow_work_response_async(CVMX_SCR_SCRATCH);227 else228 work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT);229 230 prefetch(work);231 did_work_request = 0;232 if (!work) {233 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {234 cvmx_write_csr(CVMX_SSO_WQ_IQ_DIS,235 BIT(rx_group->group));236 cvmx_write_csr(CVMX_SSO_WQ_INT,237 BIT(rx_group->group));238 } else {239 union cvmx_pow_wq_int wq_int;240 241 wq_int.u64 = 0;242 wq_int.s.iq_dis = BIT(rx_group->group);243 wq_int.s.wq_int = BIT(rx_group->group);244 cvmx_write_csr(CVMX_POW_WQ_INT, wq_int.u64);245 }246 break;247 }248 pskb = (struct sk_buff **)249 (cvm_oct_get_buffer_ptr(work->packet_ptr) -250 sizeof(void *));251 prefetch(pskb);252 253 if (USE_ASYNC_IOBDMA && rx_count < (budget - 1)) {254 cvmx_pow_work_request_async_nocheck(CVMX_SCR_SCRATCH,255 CVMX_POW_NO_WAIT);256 did_work_request = 1;257 }258 rx_count++;259 260 skb_in_hw = work->word2.s.bufs == 1;261 if (likely(skb_in_hw)) {262 skb = *pskb;263 prefetch(&skb->head);264 prefetch(&skb->len);265 }266 267 if (octeon_has_feature(OCTEON_FEATURE_PKND))268 port = work->word0.pip.cn68xx.pknd;269 else270 port = work->word1.cn38xx.ipprt;271 272 prefetch(cvm_oct_device[port]);273 274 /* Immediately throw away all packets with receive errors */275 if (unlikely(work->word2.snoip.rcv_error)) {276 if (cvm_oct_check_rcv_error(work))277 continue;278 }279 280 /*281 * We can only use the zero copy path if skbuffs are282 * in the FPA pool and the packet fits in a single283 * buffer.284 */285 if (likely(skb_in_hw)) {286 skb->data = skb->head + work->packet_ptr.s.addr -287 cvmx_ptr_to_phys(skb->head);288 prefetch(skb->data);289 skb->len = work->word1.len;290 skb_set_tail_pointer(skb, skb->len);291 packet_not_copied = 1;292 } else {293 /*294 * We have to copy the packet. First allocate295 * an skbuff for it.296 */297 skb = dev_alloc_skb(work->word1.len);298 if (!skb) {299 cvm_oct_free_work(work);300 continue;301 }302 303 /*304 * Check if we've received a packet that was305 * entirely stored in the work entry.306 */307 if (unlikely(work->word2.s.bufs == 0)) {308 u8 *ptr = work->packet_data;309 310 if (likely(!work->word2.s.not_IP)) {311 /*312 * The beginning of the packet313 * moves for IP packets.314 */315 if (work->word2.s.is_v6)316 ptr += 2;317 else318 ptr += 6;319 }320 skb_put_data(skb, ptr, work->word1.len);321 /* No packet buffers to free */322 } else {323 copy_segments_to_skb(work, skb);324 }325 packet_not_copied = 0;326 }327 if (likely((port < TOTAL_NUMBER_OF_PORTS) &&328 cvm_oct_device[port])) {329 struct net_device *dev = cvm_oct_device[port];330 331 /*332 * Only accept packets for devices that are333 * currently up.334 */335 if (likely(dev->flags & IFF_UP)) {336 skb->protocol = eth_type_trans(skb, dev);337 skb->dev = dev;338 339 if (unlikely(work->word2.s.not_IP ||340 work->word2.s.IP_exc ||341 work->word2.s.L4_error ||342 !work->word2.s.tcp_or_udp))343 skb->ip_summed = CHECKSUM_NONE;344 else345 skb->ip_summed = CHECKSUM_UNNECESSARY;346 347 /* Increment RX stats for virtual ports */348 if (port >= CVMX_PIP_NUM_INPUT_PORTS) {349 dev->stats.rx_packets++;350 dev->stats.rx_bytes += skb->len;351 }352 netif_receive_skb(skb);353 } else {354 /*355 * Drop any packet received for a device that356 * isn't up.357 */358 dev->stats.rx_dropped++;359 dev_kfree_skb_irq(skb);360 }361 } else {362 /*363 * Drop any packet received for a device that364 * doesn't exist.365 */366 printk_ratelimited("Port %d not controlled by Linux, packet dropped\n",367 port);368 dev_kfree_skb_irq(skb);369 }370 /*371 * Check to see if the skbuff and work share the same372 * packet buffer.373 */374 if (likely(packet_not_copied)) {375 /*376 * This buffer needs to be replaced, increment377 * the number of buffers we need to free by378 * one.379 */380 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,381 1);382 383 cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 1);384 } else {385 cvm_oct_free_work(work);386 }387 }388 /* Restore the original POW group mask */389 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {390 cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid), old_group_mask);391 cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */392 } else {393 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask);394 }395 396 if (USE_ASYNC_IOBDMA) {397 /* Restore the scratch area */398 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch);399 }400 cvm_oct_rx_refill_pool(0);401 402 return rx_count;403}404 405/**406 * cvm_oct_napi_poll - the NAPI poll function.407 * @napi: The NAPI instance.408 * @budget: Maximum number of packets to receive.409 *410 * Returns the number of packets processed.411 */412static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)413{414 struct oct_rx_group *rx_group = container_of(napi, struct oct_rx_group,415 napi);416 int rx_count;417 418 rx_count = cvm_oct_poll(rx_group, budget);419 420 if (rx_count < budget) {421 /* No more work */422 napi_complete_done(napi, rx_count);423 enable_irq(rx_group->irq);424 }425 return rx_count;426}427 428#ifdef CONFIG_NET_POLL_CONTROLLER429/**430 * cvm_oct_poll_controller - poll for receive packets431 * device.432 *433 * @dev: Device to poll. Unused434 */435void cvm_oct_poll_controller(struct net_device *dev)436{437 int i;438 439 if (!atomic_read(&oct_rx_ready))440 return;441 442 for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) {443 if (!(pow_receive_groups & BIT(i)))444 continue;445 446 cvm_oct_poll(&oct_rx_group[i], 16);447 }448}449#endif450 451void cvm_oct_rx_initialize(void)452{453 int i;454 struct net_device *dev_for_napi = NULL;455 456 for (i = 0; i < TOTAL_NUMBER_OF_PORTS; i++) {457 if (cvm_oct_device[i]) {458 dev_for_napi = cvm_oct_device[i];459 break;460 }461 }462 463 if (!dev_for_napi)464 panic("No net_devices were allocated.");465 466 for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) {467 int ret;468 469 if (!(pow_receive_groups & BIT(i)))470 continue;471 472 netif_napi_add_weight(dev_for_napi, &oct_rx_group[i].napi,473 cvm_oct_napi_poll, rx_napi_weight);474 napi_enable(&oct_rx_group[i].napi);475 476 oct_rx_group[i].irq = OCTEON_IRQ_WORKQ0 + i;477 oct_rx_group[i].group = i;478 479 /* Register an IRQ handler to receive POW interrupts */480 ret = request_irq(oct_rx_group[i].irq, cvm_oct_do_interrupt, 0,481 "Ethernet", &oct_rx_group[i].napi);482 if (ret)483 panic("Could not acquire Ethernet IRQ %d\n",484 oct_rx_group[i].irq);485 486 disable_irq_nosync(oct_rx_group[i].irq);487 488 /* Enable POW interrupt when our port has at least one packet */489 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {490 union cvmx_sso_wq_int_thrx int_thr;491 union cvmx_pow_wq_int_pc int_pc;492 493 int_thr.u64 = 0;494 int_thr.s.tc_en = 1;495 int_thr.s.tc_thr = 1;496 cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), int_thr.u64);497 498 int_pc.u64 = 0;499 int_pc.s.pc_thr = 5;500 cvmx_write_csr(CVMX_SSO_WQ_INT_PC, int_pc.u64);501 } else {502 union cvmx_pow_wq_int_thrx int_thr;503 union cvmx_pow_wq_int_pc int_pc;504 505 int_thr.u64 = 0;506 int_thr.s.tc_en = 1;507 int_thr.s.tc_thr = 1;508 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), int_thr.u64);509 510 int_pc.u64 = 0;511 int_pc.s.pc_thr = 5;512 cvmx_write_csr(CVMX_POW_WQ_INT_PC, int_pc.u64);513 }514 515 /* Schedule NAPI now. This will indirectly enable the516 * interrupt.517 */518 napi_schedule(&oct_rx_group[i].napi);519 }520 atomic_inc(&oct_rx_ready);521}522 523void cvm_oct_rx_shutdown(void)524{525 int i;526 527 for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) {528 if (!(pow_receive_groups & BIT(i)))529 continue;530 531 /* Disable POW interrupt */532 if (OCTEON_IS_MODEL(OCTEON_CN68XX))533 cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), 0);534 else535 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0);536 537 /* Free the interrupt handler */538 free_irq(oct_rx_group[i].irq, cvm_oct_device);539 540 netif_napi_del(&oct_rx_group[i].napi);541 }542}543