969 lines · c
1/**********************************************************************2 * Author: Cavium, Inc.3 *4 * Contact: support@cavium.com5 * Please include "LiquidIO" in the subject.6 *7 * Copyright (c) 2003-2016 Cavium, Inc.8 *9 * This file is free software; you can redistribute it and/or modify10 * it under the terms of the GNU General Public License, Version 2, as11 * published by the Free Software Foundation.12 *13 * This file is distributed in the hope that it will be useful, but14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or16 * NONINFRINGEMENT. See the GNU General Public License for more details.17 ***********************************************************************/18#include <linux/pci.h>19#include <linux/netdevice.h>20#include <linux/vmalloc.h>21#include "liquidio_common.h"22#include "octeon_droq.h"23#include "octeon_iq.h"24#include "response_manager.h"25#include "octeon_device.h"26#include "octeon_main.h"27#include "octeon_network.h"28#include "cn66xx_regs.h"29#include "cn66xx_device.h"30#include "cn23xx_pf_device.h"31#include "cn23xx_vf_device.h"32 33struct __dispatch {34 struct list_head list;35 struct octeon_recv_info *rinfo;36 octeon_dispatch_fn_t disp_fn;37};38 39/** Get the argument that the user set when registering dispatch40 * function for a given opcode/subcode.41 * @param octeon_dev - the octeon device pointer.42 * @param opcode - the opcode for which the dispatch argument43 * is to be checked.44 * @param subcode - the subcode for which the dispatch argument45 * is to be checked.46 * @return Success: void * (argument to the dispatch function)47 * @return Failure: NULL48 *49 */50void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,51 u16 opcode, u16 subcode)52{53 int idx;54 struct list_head *dispatch;55 void *fn_arg = NULL;56 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);57 58 idx = combined_opcode & OCTEON_OPCODE_MASK;59 60 spin_lock_bh(&octeon_dev->dispatch.lock);61 62 if (octeon_dev->dispatch.count == 0) {63 spin_unlock_bh(&octeon_dev->dispatch.lock);64 return NULL;65 }66 67 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {68 fn_arg = octeon_dev->dispatch.dlist[idx].arg;69 } else {70 list_for_each(dispatch,71 &octeon_dev->dispatch.dlist[idx].list) {72 if (((struct octeon_dispatch *)dispatch)->opcode ==73 combined_opcode) {74 fn_arg = ((struct octeon_dispatch *)75 dispatch)->arg;76 break;77 }78 }79 }80 81 spin_unlock_bh(&octeon_dev->dispatch.lock);82 return fn_arg;83}84 85/** Check for packets on Droq. This function should be called with lock held.86 * @param droq - Droq on which count is checked.87 * @return Returns packet count.88 */89u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)90{91 u32 pkt_count = 0;92 u32 last_count;93 94 pkt_count = readl(droq->pkts_sent_reg);95 96 last_count = pkt_count - droq->pkt_count;97 droq->pkt_count = pkt_count;98 99 /* we shall write to cnts at napi irq enable or end of droq tasklet */100 if (last_count)101 atomic_add(last_count, &droq->pkts_pending);102 103 return last_count;104}105EXPORT_SYMBOL_GPL(octeon_droq_check_hw_for_pkts);106 107static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)108{109 u32 count = 0;110 111 /* max_empty_descs is the max. no. of descs that can have no buffers.112 * If the empty desc count goes beyond this value, we cannot safely113 * read in a 64K packet sent by Octeon114 * (64K is max pkt size from Octeon)115 */116 droq->max_empty_descs = 0;117 118 do {119 droq->max_empty_descs++;120 count += droq->buffer_size;121 } while (count < (64 * 1024));122 123 droq->max_empty_descs = droq->max_count - droq->max_empty_descs;124}125 126static void octeon_droq_reset_indices(struct octeon_droq *droq)127{128 droq->read_idx = 0;129 droq->write_idx = 0;130 droq->refill_idx = 0;131 droq->refill_count = 0;132 atomic_set(&droq->pkts_pending, 0);133}134 135static void136octeon_droq_destroy_ring_buffers(struct octeon_device *oct,137 struct octeon_droq *droq)138{139 u32 i;140 struct octeon_skb_page_info *pg_info;141 142 for (i = 0; i < droq->max_count; i++) {143 pg_info = &droq->recv_buf_list[i].pg_info;144 if (!pg_info)145 continue;146 147 if (pg_info->dma)148 lio_unmap_ring(oct->pci_dev,149 (u64)pg_info->dma);150 pg_info->dma = 0;151 152 if (pg_info->page)153 recv_buffer_destroy(droq->recv_buf_list[i].buffer,154 pg_info);155 156 droq->recv_buf_list[i].buffer = NULL;157 }158 159 octeon_droq_reset_indices(droq);160}161 162static int163octeon_droq_setup_ring_buffers(struct octeon_device *oct,164 struct octeon_droq *droq)165{166 u32 i;167 void *buf;168 struct octeon_droq_desc *desc_ring = droq->desc_ring;169 170 for (i = 0; i < droq->max_count; i++) {171 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);172 173 if (!buf) {174 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",175 __func__);176 droq->stats.rx_alloc_failure++;177 return -ENOMEM;178 }179 180 droq->recv_buf_list[i].buffer = buf;181 droq->recv_buf_list[i].data = get_rbd(buf);182 desc_ring[i].info_ptr = 0;183 desc_ring[i].buffer_ptr =184 lio_map_ring(droq->recv_buf_list[i].buffer);185 }186 187 octeon_droq_reset_indices(droq);188 189 octeon_droq_compute_max_packet_bufs(droq);190 191 return 0;192}193 194int octeon_delete_droq(struct octeon_device *oct, u32 q_no)195{196 struct octeon_droq *droq = oct->droq[q_no];197 198 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);199 200 octeon_droq_destroy_ring_buffers(oct, droq);201 vfree(droq->recv_buf_list);202 203 if (droq->desc_ring)204 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),205 droq->desc_ring, droq->desc_ring_dma);206 207 memset(droq, 0, OCT_DROQ_SIZE);208 oct->io_qmask.oq &= ~(1ULL << q_no);209 vfree(oct->droq[q_no]);210 oct->droq[q_no] = NULL;211 oct->num_oqs--;212 213 return 0;214}215EXPORT_SYMBOL_GPL(octeon_delete_droq);216 217int octeon_init_droq(struct octeon_device *oct,218 u32 q_no,219 u32 num_descs,220 u32 desc_size,221 void *app_ctx)222{223 struct octeon_droq *droq;224 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;225 u32 c_pkts_per_intr = 0, c_refill_threshold = 0;226 int numa_node = dev_to_node(&oct->pci_dev->dev);227 228 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);229 230 droq = oct->droq[q_no];231 memset(droq, 0, OCT_DROQ_SIZE);232 233 droq->oct_dev = oct;234 droq->q_no = q_no;235 if (app_ctx)236 droq->app_ctx = app_ctx;237 else238 droq->app_ctx = (void *)(size_t)q_no;239 240 c_num_descs = num_descs;241 c_buf_size = desc_size;242 if (OCTEON_CN6XXX(oct)) {243 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);244 245 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);246 c_refill_threshold =247 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);248 } else if (OCTEON_CN23XX_PF(oct)) {249 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);250 251 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);252 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);253 } else if (OCTEON_CN23XX_VF(oct)) {254 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf);255 256 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);257 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);258 } else {259 return 1;260 }261 262 droq->max_count = c_num_descs;263 droq->buffer_size = c_buf_size;264 265 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;266 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,267 (dma_addr_t *)&droq->desc_ring_dma);268 269 if (!droq->desc_ring) {270 dev_err(&oct->pci_dev->dev,271 "Output queue %d ring alloc failed\n", q_no);272 return 1;273 }274 275 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",276 q_no, droq->desc_ring, droq->desc_ring_dma);277 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,278 droq->max_count);279 280 droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE),281 numa_node);282 if (!droq->recv_buf_list)283 droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE));284 if (!droq->recv_buf_list) {285 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");286 goto init_droq_fail;287 }288 289 if (octeon_droq_setup_ring_buffers(oct, droq))290 goto init_droq_fail;291 292 droq->pkts_per_intr = c_pkts_per_intr;293 droq->refill_threshold = c_refill_threshold;294 295 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",296 droq->max_empty_descs);297 298 INIT_LIST_HEAD(&droq->dispatch_list);299 300 /* For 56xx Pass1, this function won't be called, so no checks. */301 oct->fn_list.setup_oq_regs(oct, q_no);302 303 oct->io_qmask.oq |= BIT_ULL(q_no);304 305 return 0;306 307init_droq_fail:308 octeon_delete_droq(oct, q_no);309 return 1;310}311 312/* octeon_create_recv_info313 * Parameters:314 * octeon_dev - pointer to the octeon device structure315 * droq - droq in which the packet arrived.316 * buf_cnt - no. of buffers used by the packet.317 * idx - index in the descriptor for the first buffer in the packet.318 * Description:319 * Allocates a recv_info_t and copies the buffer addresses for packet data320 * into the recv_pkt space which starts at an 8B offset from recv_info_t.321 * Flags the descriptors for refill later. If available descriptors go322 * below the threshold to receive a 64K pkt, new buffers are first allocated323 * before the recv_pkt_t is created.324 * This routine will be called in interrupt context.325 * Returns:326 * Success: Pointer to recv_info_t327 * Failure: NULL.328 */329static inline struct octeon_recv_info *octeon_create_recv_info(330 struct octeon_device *octeon_dev,331 struct octeon_droq *droq,332 u32 buf_cnt,333 u32 idx)334{335 struct octeon_droq_info *info;336 struct octeon_recv_pkt *recv_pkt;337 struct octeon_recv_info *recv_info;338 u32 i, bytes_left;339 struct octeon_skb_page_info *pg_info;340 341 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data;342 343 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));344 if (!recv_info)345 return NULL;346 347 recv_pkt = recv_info->recv_pkt;348 recv_pkt->rh = info->rh;349 recv_pkt->length = (u32)info->length;350 recv_pkt->buffer_count = (u16)buf_cnt;351 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;352 353 i = 0;354 bytes_left = (u32)info->length;355 356 while (buf_cnt) {357 {358 pg_info = &droq->recv_buf_list[idx].pg_info;359 360 lio_unmap_ring(octeon_dev->pci_dev,361 (u64)pg_info->dma);362 pg_info->page = NULL;363 pg_info->dma = 0;364 }365 366 recv_pkt->buffer_size[i] =367 (bytes_left >=368 droq->buffer_size) ? droq->buffer_size : bytes_left;369 370 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;371 droq->recv_buf_list[idx].buffer = NULL;372 373 idx = incr_index(idx, 1, droq->max_count);374 bytes_left -= droq->buffer_size;375 i++;376 buf_cnt--;377 }378 379 return recv_info;380}381 382/* If we were not able to refill all buffers, try to move around383 * the buffers that were not dispatched.384 */385static inline u32386octeon_droq_refill_pullup_descs(struct octeon_droq *droq,387 struct octeon_droq_desc *desc_ring)388{389 u32 desc_refilled = 0;390 391 u32 refill_index = droq->refill_idx;392 393 while (refill_index != droq->read_idx) {394 if (droq->recv_buf_list[refill_index].buffer) {395 droq->recv_buf_list[droq->refill_idx].buffer =396 droq->recv_buf_list[refill_index].buffer;397 droq->recv_buf_list[droq->refill_idx].data =398 droq->recv_buf_list[refill_index].data;399 desc_ring[droq->refill_idx].buffer_ptr =400 desc_ring[refill_index].buffer_ptr;401 droq->recv_buf_list[refill_index].buffer = NULL;402 desc_ring[refill_index].buffer_ptr = 0;403 do {404 droq->refill_idx = incr_index(droq->refill_idx,405 1,406 droq->max_count);407 desc_refilled++;408 droq->refill_count--;409 } while (droq->recv_buf_list[droq->refill_idx].buffer);410 }411 refill_index = incr_index(refill_index, 1, droq->max_count);412 } /* while */413 return desc_refilled;414}415 416/* octeon_droq_refill417 * Parameters:418 * droq - droq in which descriptors require new buffers.419 * Description:420 * Called during normal DROQ processing in interrupt mode or by the poll421 * thread to refill the descriptors from which buffers were dispatched422 * to upper layers. Attempts to allocate new buffers. If that fails, moves423 * up buffers (that were not dispatched) to form a contiguous ring.424 * Returns:425 * No of descriptors refilled.426 */427static u32428octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)429{430 struct octeon_droq_desc *desc_ring;431 void *buf = NULL;432 u8 *data;433 u32 desc_refilled = 0;434 struct octeon_skb_page_info *pg_info;435 436 desc_ring = droq->desc_ring;437 438 while (droq->refill_count && (desc_refilled < droq->max_count)) {439 /* If a valid buffer exists (happens if there is no dispatch),440 * reuse the buffer, else allocate.441 */442 if (!droq->recv_buf_list[droq->refill_idx].buffer) {443 pg_info =444 &droq->recv_buf_list[droq->refill_idx].pg_info;445 /* Either recycle the existing pages or go for446 * new page alloc447 */448 if (pg_info->page)449 buf = recv_buffer_reuse(octeon_dev, pg_info);450 else451 buf = recv_buffer_alloc(octeon_dev, pg_info);452 /* If a buffer could not be allocated, no point in453 * continuing454 */455 if (!buf) {456 droq->stats.rx_alloc_failure++;457 break;458 }459 droq->recv_buf_list[droq->refill_idx].buffer =460 buf;461 data = get_rbd(buf);462 } else {463 data = get_rbd(droq->recv_buf_list464 [droq->refill_idx].buffer);465 }466 467 droq->recv_buf_list[droq->refill_idx].data = data;468 469 desc_ring[droq->refill_idx].buffer_ptr =470 lio_map_ring(droq->recv_buf_list[471 droq->refill_idx].buffer);472 473 droq->refill_idx = incr_index(droq->refill_idx, 1,474 droq->max_count);475 desc_refilled++;476 droq->refill_count--;477 }478 479 if (droq->refill_count)480 desc_refilled +=481 octeon_droq_refill_pullup_descs(droq, desc_ring);482 483 /* if droq->refill_count484 * The refill count would not change in pass two. We only moved buffers485 * to close the gap in the ring, but we would still have the same no. of486 * buffers to refill.487 */488 return desc_refilled;489}490 491/** check if we can allocate packets to get out of oom.492 * @param droq - Droq being checked.493 * @return 1 if fails to refill minimum494 */495int octeon_retry_droq_refill(struct octeon_droq *droq)496{497 struct octeon_device *oct = droq->oct_dev;498 int desc_refilled, reschedule = 1;499 u32 pkts_credit;500 501 pkts_credit = readl(droq->pkts_credit_reg);502 desc_refilled = octeon_droq_refill(oct, droq);503 if (desc_refilled) {504 /* Flush the droq descriptor data to memory to be sure505 * that when we update the credits the data in memory506 * is accurate.507 */508 wmb();509 writel(desc_refilled, droq->pkts_credit_reg);510 511 if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP)512 reschedule = 0;513 }514 515 return reschedule;516}517 518static inline u32519octeon_droq_get_bufcount(u32 buf_size, u32 total_len)520{521 return DIV_ROUND_UP(total_len, buf_size);522}523 524static int525octeon_droq_dispatch_pkt(struct octeon_device *oct,526 struct octeon_droq *droq,527 union octeon_rh *rh,528 struct octeon_droq_info *info)529{530 u32 cnt;531 octeon_dispatch_fn_t disp_fn;532 struct octeon_recv_info *rinfo;533 534 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);535 536 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,537 (u16)rh->r.subcode);538 if (disp_fn) {539 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);540 if (rinfo) {541 struct __dispatch *rdisp = rinfo->rsvd;542 543 rdisp->rinfo = rinfo;544 rdisp->disp_fn = disp_fn;545 rinfo->recv_pkt->rh = *rh;546 list_add_tail(&rdisp->list,547 &droq->dispatch_list);548 } else {549 droq->stats.dropped_nomem++;550 }551 } else {552 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",553 (unsigned int)rh->r.opcode,554 (unsigned int)rh->r.subcode);555 droq->stats.dropped_nodispatch++;556 }557 558 return cnt;559}560 561static inline void octeon_droq_drop_packets(struct octeon_device *oct,562 struct octeon_droq *droq,563 u32 cnt)564{565 u32 i = 0, buf_cnt;566 struct octeon_droq_info *info;567 568 for (i = 0; i < cnt; i++) {569 info = (struct octeon_droq_info *)570 droq->recv_buf_list[droq->read_idx].data;571 octeon_swap_8B_data((u64 *)info, 2);572 573 if (info->length) {574 info->length += OCTNET_FRM_LENGTH_SIZE;575 droq->stats.bytes_received += info->length;576 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,577 (u32)info->length);578 } else {579 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");580 buf_cnt = 1;581 }582 583 droq->read_idx = incr_index(droq->read_idx, buf_cnt,584 droq->max_count);585 droq->refill_count += buf_cnt;586 }587}588 589static u32590octeon_droq_fast_process_packets(struct octeon_device *oct,591 struct octeon_droq *droq,592 u32 pkts_to_process)593{594 u32 pkt, total_len = 0, pkt_count, retval;595 struct octeon_droq_info *info;596 union octeon_rh *rh;597 598 pkt_count = pkts_to_process;599 600 for (pkt = 0; pkt < pkt_count; pkt++) {601 u32 pkt_len = 0;602 struct sk_buff *nicbuf = NULL;603 struct octeon_skb_page_info *pg_info;604 void *buf;605 606 info = (struct octeon_droq_info *)607 droq->recv_buf_list[droq->read_idx].data;608 octeon_swap_8B_data((u64 *)info, 2);609 610 if (!info->length) {611 dev_err(&oct->pci_dev->dev,612 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",613 droq->q_no, droq->read_idx, pkt_count);614 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,615 (u8 *)info,616 OCT_DROQ_INFO_SIZE);617 break;618 }619 620 /* Len of resp hdr in included in the received data len. */621 rh = &info->rh;622 623 info->length += OCTNET_FRM_LENGTH_SIZE;624 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64));625 total_len += (u32)info->length;626 if (opcode_slow_path(rh)) {627 u32 buf_cnt;628 629 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);630 droq->read_idx = incr_index(droq->read_idx,631 buf_cnt, droq->max_count);632 droq->refill_count += buf_cnt;633 } else {634 if (info->length <= droq->buffer_size) {635 pkt_len = (u32)info->length;636 nicbuf = droq->recv_buf_list[637 droq->read_idx].buffer;638 pg_info = &droq->recv_buf_list[639 droq->read_idx].pg_info;640 if (recv_buffer_recycle(oct, pg_info))641 pg_info->page = NULL;642 droq->recv_buf_list[droq->read_idx].buffer =643 NULL;644 645 droq->read_idx = incr_index(droq->read_idx, 1,646 droq->max_count);647 droq->refill_count++;648 } else {649 nicbuf = octeon_fast_packet_alloc((u32)650 info->length);651 pkt_len = 0;652 /* nicbuf allocation can fail. We'll handle it653 * inside the loop.654 */655 while (pkt_len < info->length) {656 int cpy_len, idx = droq->read_idx;657 658 cpy_len = ((pkt_len + droq->buffer_size)659 > info->length) ?660 ((u32)info->length - pkt_len) :661 droq->buffer_size;662 663 if (nicbuf) {664 octeon_fast_packet_next(droq,665 nicbuf,666 cpy_len,667 idx);668 buf = droq->recv_buf_list[669 idx].buffer;670 recv_buffer_fast_free(buf);671 droq->recv_buf_list[idx].buffer672 = NULL;673 } else {674 droq->stats.rx_alloc_failure++;675 }676 677 pkt_len += cpy_len;678 droq->read_idx =679 incr_index(droq->read_idx, 1,680 droq->max_count);681 droq->refill_count++;682 }683 }684 685 if (nicbuf) {686 if (droq->ops.fptr) {687 droq->ops.fptr(oct->octeon_id,688 nicbuf, pkt_len,689 rh, &droq->napi,690 droq->ops.farg);691 } else {692 recv_buffer_free(nicbuf);693 }694 }695 }696 697 if (droq->refill_count >= droq->refill_threshold) {698 int desc_refilled = octeon_droq_refill(oct, droq);699 700 if (desc_refilled) {701 /* Flush the droq descriptor data to memory to702 * be sure that when we update the credits the703 * data in memory is accurate.704 */705 wmb();706 writel(desc_refilled, droq->pkts_credit_reg);707 }708 }709 } /* for (each packet)... */710 711 /* Increment refill_count by the number of buffers processed. */712 droq->stats.pkts_received += pkt;713 droq->stats.bytes_received += total_len;714 715 retval = pkt;716 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {717 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));718 719 droq->stats.dropped_toomany += (pkts_to_process - pkt);720 retval = pkts_to_process;721 }722 723 atomic_sub(retval, &droq->pkts_pending);724 725 if (droq->refill_count >= droq->refill_threshold &&726 readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) {727 octeon_droq_check_hw_for_pkts(droq);728 729 /* Make sure there are no pkts_pending */730 if (!atomic_read(&droq->pkts_pending))731 octeon_schedule_rxq_oom_work(oct, droq);732 }733 734 return retval;735}736 737int738octeon_droq_process_packets(struct octeon_device *oct,739 struct octeon_droq *droq,740 u32 budget)741{742 u32 pkt_count = 0;743 struct list_head *tmp, *tmp2;744 745 octeon_droq_check_hw_for_pkts(droq);746 pkt_count = atomic_read(&droq->pkts_pending);747 748 if (!pkt_count)749 return 0;750 751 if (pkt_count > budget)752 pkt_count = budget;753 754 octeon_droq_fast_process_packets(oct, droq, pkt_count);755 756 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {757 struct __dispatch *rdisp = (struct __dispatch *)tmp;758 759 list_del(tmp);760 rdisp->disp_fn(rdisp->rinfo,761 octeon_get_dispatch_arg762 (oct,763 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,764 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));765 }766 767 /* If there are packets pending. schedule tasklet again */768 if (atomic_read(&droq->pkts_pending))769 return 1;770 771 return 0;772}773EXPORT_SYMBOL_GPL(octeon_droq_process_packets);774 775/*776 * Utility function to poll for packets. check_hw_for_packets must be777 * called before calling this routine.778 */779 780int781octeon_droq_process_poll_pkts(struct octeon_device *oct,782 struct octeon_droq *droq, u32 budget)783{784 struct list_head *tmp, *tmp2;785 u32 pkts_available = 0, pkts_processed = 0;786 u32 total_pkts_processed = 0;787 788 if (budget > droq->max_count)789 budget = droq->max_count;790 791 while (total_pkts_processed < budget) {792 octeon_droq_check_hw_for_pkts(droq);793 794 pkts_available = min((budget - total_pkts_processed),795 (u32)(atomic_read(&droq->pkts_pending)));796 797 if (pkts_available == 0)798 break;799 800 pkts_processed =801 octeon_droq_fast_process_packets(oct, droq,802 pkts_available);803 804 total_pkts_processed += pkts_processed;805 }806 807 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {808 struct __dispatch *rdisp = (struct __dispatch *)tmp;809 810 list_del(tmp);811 rdisp->disp_fn(rdisp->rinfo,812 octeon_get_dispatch_arg813 (oct,814 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,815 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));816 }817 818 return total_pkts_processed;819}820 821/* Enable Pkt Interrupt */822int823octeon_enable_irq(struct octeon_device *oct, u32 q_no)824{825 switch (oct->chip_id) {826 case OCTEON_CN66XX:827 case OCTEON_CN68XX: {828 struct octeon_cn6xxx *cn6xxx =829 (struct octeon_cn6xxx *)oct->chip;830 unsigned long flags;831 u32 value;832 833 spin_lock_irqsave834 (&cn6xxx->lock_for_droq_int_enb_reg, flags);835 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);836 value |= (1 << q_no);837 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value);838 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);839 value |= (1 << q_no);840 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value);841 842 /* don't bother flushing the enables */843 844 spin_unlock_irqrestore845 (&cn6xxx->lock_for_droq_int_enb_reg, flags);846 }847 break;848 case OCTEON_CN23XX_PF_VID:849 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);850 break;851 852 case OCTEON_CN23XX_VF_VID:853 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);854 break;855 default:856 dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__);857 return 1;858 }859 860 return 0;861}862 863int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,864 struct octeon_droq_ops *ops)865{866 struct octeon_config *oct_cfg = NULL;867 struct octeon_droq *droq;868 869 oct_cfg = octeon_get_conf(oct);870 871 if (!oct_cfg)872 return -EINVAL;873 874 if (!(ops)) {875 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",876 __func__);877 return -EINVAL;878 }879 880 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {881 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",882 __func__, q_no, (oct->num_oqs - 1));883 return -EINVAL;884 }885 886 droq = oct->droq[q_no];887 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));888 889 return 0;890}891 892int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)893{894 struct octeon_config *oct_cfg = NULL;895 struct octeon_droq *droq;896 897 oct_cfg = octeon_get_conf(oct);898 899 if (!oct_cfg)900 return -EINVAL;901 902 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {903 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",904 __func__, q_no, oct->num_oqs - 1);905 return -EINVAL;906 }907 908 droq = oct->droq[q_no];909 910 if (!droq) {911 dev_info(&oct->pci_dev->dev,912 "Droq id (%d) not available.\n", q_no);913 return 0;914 }915 916 droq->ops.fptr = NULL;917 droq->ops.farg = NULL;918 droq->ops.drop_on_max = 0;919 920 return 0;921}922EXPORT_SYMBOL_GPL(octeon_unregister_droq_ops);923 924int octeon_create_droq(struct octeon_device *oct,925 u32 q_no, u32 num_descs,926 u32 desc_size, void *app_ctx)927{928 struct octeon_droq *droq;929 int numa_node = dev_to_node(&oct->pci_dev->dev);930 931 if (oct->droq[q_no]) {932 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",933 q_no);934 return 1;935 }936 937 /* Allocate the DS for the new droq. */938 droq = vmalloc_node(sizeof(*droq), numa_node);939 if (!droq)940 droq = vmalloc(sizeof(*droq));941 if (!droq)942 return -1;943 944 memset(droq, 0, sizeof(struct octeon_droq));945 946 /*Disable the pkt o/p for this Q */947 octeon_set_droq_pkt_op(oct, q_no, 0);948 oct->droq[q_no] = droq;949 950 /* Initialize the Droq */951 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {952 vfree(oct->droq[q_no]);953 oct->droq[q_no] = NULL;954 return -1;955 }956 957 oct->num_oqs++;958 959 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,960 oct->num_oqs);961 962 /* Global Droq register settings */963 964 /* As of now not required, as setting are done for all 32 Droqs at965 * the same time.966 */967 return 0;968}969