469 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright (c) 2014-2015 Hisilicon Limited.4 */5 6#include <linux/dma-mapping.h>7#include <linux/interrupt.h>8#include <linux/of.h>9#include <linux/skbuff.h>10#include <linux/slab.h>11#include "hnae.h"12 13#define cls_to_ae_dev(dev) container_of(dev, struct hnae_ae_dev, cls_dev)14 15static const struct class hnae_class = {16 .name = "hnae",17};18 19static void20hnae_list_add(spinlock_t *lock, struct list_head *node, struct list_head *head)21{22 unsigned long flags;23 24 spin_lock_irqsave(lock, flags);25 list_add_tail_rcu(node, head);26 spin_unlock_irqrestore(lock, flags);27}28 29static void hnae_list_del(spinlock_t *lock, struct list_head *node)30{31 unsigned long flags;32 33 spin_lock_irqsave(lock, flags);34 list_del_rcu(node);35 spin_unlock_irqrestore(lock, flags);36}37 38static int hnae_alloc_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)39{40 unsigned int order = hnae_page_order(ring);41 struct page *p = dev_alloc_pages(order);42 43 if (!p)44 return -ENOMEM;45 46 cb->priv = p;47 cb->page_offset = 0;48 cb->reuse_flag = 0;49 cb->buf = page_address(p);50 cb->length = hnae_page_size(ring);51 cb->type = DESC_TYPE_PAGE;52 53 return 0;54}55 56static void hnae_free_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)57{58 if (unlikely(!cb->priv))59 return;60 61 if (cb->type == DESC_TYPE_SKB)62 dev_kfree_skb_any((struct sk_buff *)cb->priv);63 else if (unlikely(is_rx_ring(ring)))64 put_page((struct page *)cb->priv);65 66 cb->priv = NULL;67}68 69static int hnae_map_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)70{71 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,72 cb->length, ring_to_dma_dir(ring));73 74 if (dma_mapping_error(ring_to_dev(ring), cb->dma))75 return -EIO;76 77 return 0;78}79 80static void hnae_unmap_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)81{82 if (cb->type == DESC_TYPE_SKB)83 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,84 ring_to_dma_dir(ring));85 else if (cb->length)86 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,87 ring_to_dma_dir(ring));88}89 90static struct hnae_buf_ops hnae_bops = {91 .alloc_buffer = hnae_alloc_buffer,92 .free_buffer = hnae_free_buffer,93 .map_buffer = hnae_map_buffer,94 .unmap_buffer = hnae_unmap_buffer,95};96 97static int __ae_match(struct device *dev, const void *data)98{99 struct hnae_ae_dev *hdev = cls_to_ae_dev(dev);100 101 if (dev_of_node(hdev->dev))102 return (data == &hdev->dev->of_node->fwnode);103 else if (is_acpi_node(hdev->dev->fwnode))104 return (data == hdev->dev->fwnode);105 106 dev_err(dev, "__ae_match cannot read cfg data from OF or acpi\n");107 return 0;108}109 110static struct hnae_ae_dev *find_ae(const struct fwnode_handle *fwnode)111{112 struct device *dev;113 114 WARN_ON(!fwnode);115 116 dev = class_find_device(&hnae_class, NULL, fwnode, __ae_match);117 118 return dev ? cls_to_ae_dev(dev) : NULL;119}120 121static void hnae_free_buffers(struct hnae_ring *ring)122{123 int i;124 125 for (i = 0; i < ring->desc_num; i++)126 hnae_free_buffer_detach(ring, i);127}128 129/* Allocate memory for raw pkg, and map with dma */130static int hnae_alloc_buffers(struct hnae_ring *ring)131{132 int i, j, ret;133 134 for (i = 0; i < ring->desc_num; i++) {135 ret = hnae_alloc_buffer_attach(ring, i);136 if (ret)137 goto out_buffer_fail;138 }139 140 return 0;141 142out_buffer_fail:143 for (j = i - 1; j >= 0; j--)144 hnae_free_buffer_detach(ring, j);145 return ret;146}147 148/* free desc along with its attached buffer */149static void hnae_free_desc(struct hnae_ring *ring)150{151 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,152 ring->desc_num * sizeof(ring->desc[0]),153 ring_to_dma_dir(ring));154 ring->desc_dma_addr = 0;155 kfree(ring->desc);156 ring->desc = NULL;157}158 159/* alloc desc, without buffer attached */160static int hnae_alloc_desc(struct hnae_ring *ring)161{162 int size = ring->desc_num * sizeof(ring->desc[0]);163 164 ring->desc = kzalloc(size, GFP_KERNEL);165 if (!ring->desc)166 return -ENOMEM;167 168 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring),169 ring->desc, size, ring_to_dma_dir(ring));170 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {171 ring->desc_dma_addr = 0;172 kfree(ring->desc);173 ring->desc = NULL;174 return -ENOMEM;175 }176 177 return 0;178}179 180/* fini ring, also free the buffer for the ring */181static void hnae_fini_ring(struct hnae_ring *ring)182{183 if (is_rx_ring(ring))184 hnae_free_buffers(ring);185 186 hnae_free_desc(ring);187 kfree(ring->desc_cb);188 ring->desc_cb = NULL;189 ring->next_to_clean = 0;190 ring->next_to_use = 0;191}192 193/* init ring, and with buffer for rx ring */194static int195hnae_init_ring(struct hnae_queue *q, struct hnae_ring *ring, int flags)196{197 int ret;198 199 if (ring->desc_num <= 0 || ring->buf_size <= 0)200 return -EINVAL;201 202 ring->q = q;203 ring->flags = flags;204 ring->coal_param = q->handle->coal_param;205 assert(!ring->desc && !ring->desc_cb && !ring->desc_dma_addr);206 207 /* not matter for tx or rx ring, the ntc and ntc start from 0 */208 assert(ring->next_to_use == 0);209 assert(ring->next_to_clean == 0);210 211 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),212 GFP_KERNEL);213 if (!ring->desc_cb) {214 ret = -ENOMEM;215 goto out;216 }217 218 ret = hnae_alloc_desc(ring);219 if (ret)220 goto out_with_desc_cb;221 222 if (is_rx_ring(ring)) {223 ret = hnae_alloc_buffers(ring);224 if (ret)225 goto out_with_desc;226 }227 228 return 0;229 230out_with_desc:231 hnae_free_desc(ring);232out_with_desc_cb:233 kfree(ring->desc_cb);234 ring->desc_cb = NULL;235out:236 return ret;237}238 239static int hnae_init_queue(struct hnae_handle *h, struct hnae_queue *q,240 struct hnae_ae_dev *dev)241{242 int ret;243 244 q->dev = dev;245 q->handle = h;246 247 ret = hnae_init_ring(q, &q->tx_ring, q->tx_ring.flags | RINGF_DIR);248 if (ret)249 goto out;250 251 ret = hnae_init_ring(q, &q->rx_ring, q->rx_ring.flags & ~RINGF_DIR);252 if (ret)253 goto out_with_tx_ring;254 255 if (dev->ops->init_queue)256 dev->ops->init_queue(q);257 258 return 0;259 260out_with_tx_ring:261 hnae_fini_ring(&q->tx_ring);262out:263 return ret;264}265 266static void hnae_fini_queue(struct hnae_queue *q)267{268 if (q->dev->ops->fini_queue)269 q->dev->ops->fini_queue(q);270 271 hnae_fini_ring(&q->tx_ring);272 hnae_fini_ring(&q->rx_ring);273}274 275/*276 * ae_chain - define ae chain head277 */278static RAW_NOTIFIER_HEAD(ae_chain);279 280int hnae_register_notifier(struct notifier_block *nb)281{282 return raw_notifier_chain_register(&ae_chain, nb);283}284EXPORT_SYMBOL(hnae_register_notifier);285 286void hnae_unregister_notifier(struct notifier_block *nb)287{288 if (raw_notifier_chain_unregister(&ae_chain, nb))289 dev_err(NULL, "notifier chain unregister fail\n");290}291EXPORT_SYMBOL(hnae_unregister_notifier);292 293int hnae_reinit_handle(struct hnae_handle *handle)294{295 int i, j;296 int ret;297 298 for (i = 0; i < handle->q_num; i++) /* free ring*/299 hnae_fini_queue(handle->qs[i]);300 301 if (handle->dev->ops->reset)302 handle->dev->ops->reset(handle);303 304 for (i = 0; i < handle->q_num; i++) {/* reinit ring*/305 ret = hnae_init_queue(handle, handle->qs[i], handle->dev);306 if (ret)307 goto out_when_init_queue;308 }309 return 0;310out_when_init_queue:311 for (j = i - 1; j >= 0; j--)312 hnae_fini_queue(handle->qs[j]);313 return ret;314}315EXPORT_SYMBOL(hnae_reinit_handle);316 317/* hnae_get_handle - get a handle from the AE318 * @owner_dev: the dev use this handle319 * @ae_id: the id of the ae to be used320 * @ae_opts: the options set for the handle321 * @bops: the callbacks for buffer management322 *323 * return handle ptr or ERR_PTR324 */325struct hnae_handle *hnae_get_handle(struct device *owner_dev,326 const struct fwnode_handle *fwnode,327 u32 port_id,328 struct hnae_buf_ops *bops)329{330 struct hnae_ae_dev *dev;331 struct hnae_handle *handle;332 int i, j;333 int ret;334 335 dev = find_ae(fwnode);336 if (!dev)337 return ERR_PTR(-ENODEV);338 339 handle = dev->ops->get_handle(dev, port_id);340 if (IS_ERR(handle)) {341 put_device(&dev->cls_dev);342 return handle;343 }344 345 handle->dev = dev;346 handle->owner_dev = owner_dev;347 handle->bops = bops ? bops : &hnae_bops;348 handle->eport_id = port_id;349 350 for (i = 0; i < handle->q_num; i++) {351 ret = hnae_init_queue(handle, handle->qs[i], dev);352 if (ret)353 goto out_when_init_queue;354 }355 356 __module_get(dev->owner);357 358 hnae_list_add(&dev->lock, &handle->node, &dev->handle_list);359 360 return handle;361 362out_when_init_queue:363 for (j = i - 1; j >= 0; j--)364 hnae_fini_queue(handle->qs[j]);365 366 put_device(&dev->cls_dev);367 368 return ERR_PTR(-ENOMEM);369}370EXPORT_SYMBOL(hnae_get_handle);371 372void hnae_put_handle(struct hnae_handle *h)373{374 struct hnae_ae_dev *dev = h->dev;375 int i;376 377 for (i = 0; i < h->q_num; i++)378 hnae_fini_queue(h->qs[i]);379 380 if (h->dev->ops->reset)381 h->dev->ops->reset(h);382 383 hnae_list_del(&dev->lock, &h->node);384 385 if (dev->ops->put_handle)386 dev->ops->put_handle(h);387 388 module_put(dev->owner);389 390 put_device(&dev->cls_dev);391}392EXPORT_SYMBOL(hnae_put_handle);393 394static void hnae_release(struct device *dev)395{396}397 398/**399 * hnae_ae_register - register a AE engine to hnae framework400 * @hdev: the hnae ae engine device401 * @owner: the module who provides this dev402 * NOTE: the duplicated name will not be checked403 */404int hnae_ae_register(struct hnae_ae_dev *hdev, struct module *owner)405{406 static atomic_t id = ATOMIC_INIT(-1);407 int ret;408 409 if (!hdev->dev)410 return -ENODEV;411 412 if (!hdev->ops || !hdev->ops->get_handle ||413 !hdev->ops->toggle_ring_irq ||414 !hdev->ops->get_status || !hdev->ops->adjust_link)415 return -EINVAL;416 417 hdev->owner = owner;418 hdev->id = (int)atomic_inc_return(&id);419 hdev->cls_dev.parent = hdev->dev;420 hdev->cls_dev.class = &hnae_class;421 hdev->cls_dev.release = hnae_release;422 (void)dev_set_name(&hdev->cls_dev, "hnae%d", hdev->id);423 ret = device_register(&hdev->cls_dev);424 if (ret) {425 put_device(&hdev->cls_dev);426 return ret;427 }428 429 INIT_LIST_HEAD(&hdev->handle_list);430 spin_lock_init(&hdev->lock);431 432 ret = raw_notifier_call_chain(&ae_chain, HNAE_AE_REGISTER, NULL);433 if (ret)434 dev_dbg(hdev->dev,435 "has not notifier for AE: %s\n", hdev->name);436 437 return 0;438}439EXPORT_SYMBOL(hnae_ae_register);440 441/**442 * hnae_ae_unregister - unregisters a HNAE AE engine443 * @hdev: the device to unregister444 */445void hnae_ae_unregister(struct hnae_ae_dev *hdev)446{447 device_unregister(&hdev->cls_dev);448}449EXPORT_SYMBOL(hnae_ae_unregister);450 451static int __init hnae_init(void)452{453 return class_register(&hnae_class);454}455 456static void __exit hnae_exit(void)457{458 class_unregister(&hnae_class);459}460 461subsys_initcall(hnae_init);462module_exit(hnae_exit);463 464MODULE_AUTHOR("Hisilicon, Inc.");465MODULE_LICENSE("GPL");466MODULE_DESCRIPTION("Hisilicon Network Acceleration Engine Framework");467 468/* vi: set tw=78 noet: */469