1084 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Virtio-based remote processor messaging bus4 *5 * Copyright (C) 2011 Texas Instruments, Inc.6 * Copyright (C) 2011 Google, Inc.7 *8 * Ohad Ben-Cohen <ohad@wizery.com>9 * Brian Swetland <swetland@google.com>10 */11 12#define pr_fmt(fmt) "%s: " fmt, __func__13 14#include <linux/dma-mapping.h>15#include <linux/idr.h>16#include <linux/jiffies.h>17#include <linux/kernel.h>18#include <linux/module.h>19#include <linux/mutex.h>20#include <linux/rpmsg.h>21#include <linux/rpmsg/byteorder.h>22#include <linux/rpmsg/ns.h>23#include <linux/scatterlist.h>24#include <linux/slab.h>25#include <linux/sched.h>26#include <linux/virtio.h>27#include <linux/virtio_ids.h>28#include <linux/virtio_config.h>29#include <linux/wait.h>30 31#include "rpmsg_internal.h"32 33/**34 * struct virtproc_info - virtual remote processor state35 * @vdev: the virtio device36 * @rvq: rx virtqueue37 * @svq: tx virtqueue38 * @rbufs: kernel address of rx buffers39 * @sbufs: kernel address of tx buffers40 * @num_bufs: total number of buffers for rx and tx41 * @buf_size: size of one rx or tx buffer42 * @last_sbuf: index of last tx buffer used43 * @bufs_dma: dma base addr of the buffers44 * @tx_lock: protects svq, sbufs and sleepers, to allow concurrent senders.45 * sending a message might require waking up a dozing remote46 * processor, which involves sleeping, hence the mutex.47 * @endpoints: idr of local endpoints, allows fast retrieval48 * @endpoints_lock: lock of the endpoints set49 * @sendq: wait queue of sending contexts waiting for a tx buffers50 * @sleepers: number of senders that are waiting for a tx buffer51 *52 * This structure stores the rpmsg state of a given virtio remote processor53 * device (there might be several virtio proc devices for each physical54 * remote processor).55 */56struct virtproc_info {57 struct virtio_device *vdev;58 struct virtqueue *rvq, *svq;59 void *rbufs, *sbufs;60 unsigned int num_bufs;61 unsigned int buf_size;62 int last_sbuf;63 dma_addr_t bufs_dma;64 struct mutex tx_lock;65 struct idr endpoints;66 struct mutex endpoints_lock;67 wait_queue_head_t sendq;68 atomic_t sleepers;69};70 71/* The feature bitmap for virtio rpmsg */72#define VIRTIO_RPMSG_F_NS 0 /* RP supports name service notifications */73 74/**75 * struct rpmsg_hdr - common header for all rpmsg messages76 * @src: source address77 * @dst: destination address78 * @reserved: reserved for future use79 * @len: length of payload (in bytes)80 * @flags: message flags81 * @data: @len bytes of message payload data82 *83 * Every message sent(/received) on the rpmsg bus begins with this header.84 */85struct rpmsg_hdr {86 __rpmsg32 src;87 __rpmsg32 dst;88 __rpmsg32 reserved;89 __rpmsg16 len;90 __rpmsg16 flags;91 u8 data[];92} __packed;93 94 95/**96 * struct virtio_rpmsg_channel - rpmsg channel descriptor97 * @rpdev: the rpmsg channel device98 * @vrp: the virtio remote processor device this channel belongs to99 *100 * This structure stores the channel that links the rpmsg device to the virtio101 * remote processor device.102 */103struct virtio_rpmsg_channel {104 struct rpmsg_device rpdev;105 106 struct virtproc_info *vrp;107};108 109#define to_virtio_rpmsg_channel(_rpdev) \110 container_of(_rpdev, struct virtio_rpmsg_channel, rpdev)111 112/*113 * We're allocating buffers of 512 bytes each for communications. The114 * number of buffers will be computed from the number of buffers supported115 * by the vring, upto a maximum of 512 buffers (256 in each direction).116 *117 * Each buffer will have 16 bytes for the msg header and 496 bytes for118 * the payload.119 *120 * This will utilize a maximum total space of 256KB for the buffers.121 *122 * We might also want to add support for user-provided buffers in time.123 * This will allow bigger buffer size flexibility, and can also be used124 * to achieve zero-copy messaging.125 *126 * Note that these numbers are purely a decision of this driver - we127 * can change this without changing anything in the firmware of the remote128 * processor.129 */130#define MAX_RPMSG_NUM_BUFS (512)131#define MAX_RPMSG_BUF_SIZE (512)132 133/*134 * Local addresses are dynamically allocated on-demand.135 * We do not dynamically assign addresses from the low 1024 range,136 * in order to reserve that address range for predefined services.137 */138#define RPMSG_RESERVED_ADDRESSES (1024)139 140static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept);141static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len);142static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len,143 u32 dst);144static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src,145 u32 dst, void *data, int len);146static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len);147static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data,148 int len, u32 dst);149static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src,150 u32 dst, void *data, int len);151static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept);152static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,153 struct rpmsg_channel_info *chinfo);154 155static const struct rpmsg_endpoint_ops virtio_endpoint_ops = {156 .destroy_ept = virtio_rpmsg_destroy_ept,157 .send = virtio_rpmsg_send,158 .sendto = virtio_rpmsg_sendto,159 .send_offchannel = virtio_rpmsg_send_offchannel,160 .trysend = virtio_rpmsg_trysend,161 .trysendto = virtio_rpmsg_trysendto,162 .trysend_offchannel = virtio_rpmsg_trysend_offchannel,163 .get_mtu = virtio_rpmsg_get_mtu,164};165 166/**167 * rpmsg_sg_init - initialize scatterlist according to cpu address location168 * @sg: scatterlist to fill169 * @cpu_addr: virtual address of the buffer170 * @len: buffer length171 *172 * An internal function filling scatterlist according to virtual address173 * location (in vmalloc or in kernel).174 */175static void176rpmsg_sg_init(struct scatterlist *sg, void *cpu_addr, unsigned int len)177{178 if (is_vmalloc_addr(cpu_addr)) {179 sg_init_table(sg, 1);180 sg_set_page(sg, vmalloc_to_page(cpu_addr), len,181 offset_in_page(cpu_addr));182 } else {183 WARN_ON(!virt_addr_valid(cpu_addr));184 sg_init_one(sg, cpu_addr, len);185 }186}187 188/**189 * __ept_release() - deallocate an rpmsg endpoint190 * @kref: the ept's reference count191 *192 * This function deallocates an ept, and is invoked when its @kref refcount193 * drops to zero.194 *195 * Never invoke this function directly!196 */197static void __ept_release(struct kref *kref)198{199 struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint,200 refcount);201 /*202 * At this point no one holds a reference to ept anymore,203 * so we can directly free it204 */205 kfree(ept);206}207 208/* for more info, see below documentation of rpmsg_create_ept() */209static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,210 struct rpmsg_device *rpdev,211 rpmsg_rx_cb_t cb,212 void *priv, u32 addr)213{214 int id_min, id_max, id;215 struct rpmsg_endpoint *ept;216 struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;217 218 ept = kzalloc(sizeof(*ept), GFP_KERNEL);219 if (!ept)220 return NULL;221 222 kref_init(&ept->refcount);223 mutex_init(&ept->cb_lock);224 225 ept->rpdev = rpdev;226 ept->cb = cb;227 ept->priv = priv;228 ept->ops = &virtio_endpoint_ops;229 230 /* do we need to allocate a local address ? */231 if (addr == RPMSG_ADDR_ANY) {232 id_min = RPMSG_RESERVED_ADDRESSES;233 id_max = 0;234 } else {235 id_min = addr;236 id_max = addr + 1;237 }238 239 mutex_lock(&vrp->endpoints_lock);240 241 /* bind the endpoint to an rpmsg address (and allocate one if needed) */242 id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);243 if (id < 0) {244 dev_err(dev, "idr_alloc failed: %d\n", id);245 goto free_ept;246 }247 ept->addr = id;248 249 mutex_unlock(&vrp->endpoints_lock);250 251 return ept;252 253free_ept:254 mutex_unlock(&vrp->endpoints_lock);255 kref_put(&ept->refcount, __ept_release);256 return NULL;257}258 259static struct rpmsg_device *virtio_rpmsg_create_channel(struct rpmsg_device *rpdev,260 struct rpmsg_channel_info *chinfo)261{262 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);263 struct virtproc_info *vrp = vch->vrp;264 265 return __rpmsg_create_channel(vrp, chinfo);266}267 268static int virtio_rpmsg_release_channel(struct rpmsg_device *rpdev,269 struct rpmsg_channel_info *chinfo)270{271 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);272 struct virtproc_info *vrp = vch->vrp;273 274 return rpmsg_unregister_device(&vrp->vdev->dev, chinfo);275}276 277static struct rpmsg_endpoint *virtio_rpmsg_create_ept(struct rpmsg_device *rpdev,278 rpmsg_rx_cb_t cb,279 void *priv,280 struct rpmsg_channel_info chinfo)281{282 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);283 284 return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);285}286 287/**288 * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint289 * @vrp: virtproc which owns this ept290 * @ept: endpoing to destroy291 *292 * An internal function which destroy an ept without assuming it is293 * bound to an rpmsg channel. This is needed for handling the internal294 * name service endpoint, which isn't bound to an rpmsg channel.295 * See also __rpmsg_create_ept().296 */297static void298__rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept)299{300 /* make sure new inbound messages can't find this ept anymore */301 mutex_lock(&vrp->endpoints_lock);302 idr_remove(&vrp->endpoints, ept->addr);303 mutex_unlock(&vrp->endpoints_lock);304 305 /* make sure in-flight inbound messages won't invoke cb anymore */306 mutex_lock(&ept->cb_lock);307 ept->cb = NULL;308 mutex_unlock(&ept->cb_lock);309 310 kref_put(&ept->refcount, __ept_release);311}312 313static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept)314{315 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev);316 317 __rpmsg_destroy_ept(vch->vrp, ept);318}319 320static int virtio_rpmsg_announce_create(struct rpmsg_device *rpdev)321{322 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);323 struct virtproc_info *vrp = vch->vrp;324 struct device *dev = &rpdev->dev;325 int err = 0;326 327 /* need to tell remote processor's name service about this channel ? */328 if (rpdev->announce && rpdev->ept &&329 virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {330 struct rpmsg_ns_msg nsm;331 332 strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name));333 nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr);334 nsm.flags = cpu_to_rpmsg32(rpdev, RPMSG_NS_CREATE);335 336 err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);337 if (err)338 dev_err(dev, "failed to announce service %d\n", err);339 }340 341 return err;342}343 344static int virtio_rpmsg_announce_destroy(struct rpmsg_device *rpdev)345{346 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);347 struct virtproc_info *vrp = vch->vrp;348 struct device *dev = &rpdev->dev;349 int err = 0;350 351 /* tell remote processor's name service we're removing this channel */352 if (rpdev->announce && rpdev->ept &&353 virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {354 struct rpmsg_ns_msg nsm;355 356 strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name));357 nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr);358 nsm.flags = cpu_to_rpmsg32(rpdev, RPMSG_NS_DESTROY);359 360 err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);361 if (err)362 dev_err(dev, "failed to announce service %d\n", err);363 }364 365 return err;366}367 368static const struct rpmsg_device_ops virtio_rpmsg_ops = {369 .create_channel = virtio_rpmsg_create_channel,370 .release_channel = virtio_rpmsg_release_channel,371 .create_ept = virtio_rpmsg_create_ept,372 .announce_create = virtio_rpmsg_announce_create,373 .announce_destroy = virtio_rpmsg_announce_destroy,374};375 376static void virtio_rpmsg_release_device(struct device *dev)377{378 struct rpmsg_device *rpdev = to_rpmsg_device(dev);379 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);380 381 kfree(rpdev->driver_override);382 kfree(vch);383}384 385/*386 * create an rpmsg channel using its name and address info.387 * this function will be used to create both static and dynamic388 * channels.389 */390static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,391 struct rpmsg_channel_info *chinfo)392{393 struct virtio_rpmsg_channel *vch;394 struct rpmsg_device *rpdev;395 struct device *tmp, *dev = &vrp->vdev->dev;396 int ret;397 398 /* make sure a similar channel doesn't already exist */399 tmp = rpmsg_find_device(dev, chinfo);400 if (tmp) {401 /* decrement the matched device's refcount back */402 put_device(tmp);403 dev_err(dev, "channel %s:%x:%x already exist\n",404 chinfo->name, chinfo->src, chinfo->dst);405 return NULL;406 }407 408 vch = kzalloc(sizeof(*vch), GFP_KERNEL);409 if (!vch)410 return NULL;411 412 /* Link the channel to our vrp */413 vch->vrp = vrp;414 415 /* Assign public information to the rpmsg_device */416 rpdev = &vch->rpdev;417 rpdev->src = chinfo->src;418 rpdev->dst = chinfo->dst;419 rpdev->ops = &virtio_rpmsg_ops;420 rpdev->little_endian = virtio_is_little_endian(vrp->vdev);421 422 /*423 * rpmsg server channels has predefined local address (for now),424 * and their existence needs to be announced remotely425 */426 rpdev->announce = rpdev->src != RPMSG_ADDR_ANY;427 428 strscpy(rpdev->id.name, chinfo->name, sizeof(rpdev->id.name));429 430 rpdev->dev.parent = &vrp->vdev->dev;431 rpdev->dev.release = virtio_rpmsg_release_device;432 ret = rpmsg_register_device(rpdev);433 if (ret)434 return NULL;435 436 return rpdev;437}438 439/* super simple buffer "allocator" that is just enough for now */440static void *get_a_tx_buf(struct virtproc_info *vrp)441{442 unsigned int len;443 void *ret;444 445 /* support multiple concurrent senders */446 mutex_lock(&vrp->tx_lock);447 448 /*449 * either pick the next unused tx buffer450 * (half of our buffers are used for sending messages)451 */452 if (vrp->last_sbuf < vrp->num_bufs / 2)453 ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;454 /* or recycle a used one */455 else456 ret = virtqueue_get_buf(vrp->svq, &len);457 458 mutex_unlock(&vrp->tx_lock);459 460 return ret;461}462 463/**464 * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed465 * @vrp: virtual remote processor state466 *467 * This function is called before a sender is blocked, waiting for468 * a tx buffer to become available.469 *470 * If we already have blocking senders, this function merely increases471 * the "sleepers" reference count, and exits.472 *473 * Otherwise, if this is the first sender to block, we also enable474 * virtio's tx callbacks, so we'd be immediately notified when a tx475 * buffer is consumed (we rely on virtio's tx callback in order476 * to wake up sleeping senders as soon as a tx buffer is used by the477 * remote processor).478 */479static void rpmsg_upref_sleepers(struct virtproc_info *vrp)480{481 /* support multiple concurrent senders */482 mutex_lock(&vrp->tx_lock);483 484 /* are we the first sleeping context waiting for tx buffers ? */485 if (atomic_inc_return(&vrp->sleepers) == 1)486 /* enable "tx-complete" interrupts before dozing off */487 virtqueue_enable_cb(vrp->svq);488 489 mutex_unlock(&vrp->tx_lock);490}491 492/**493 * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed494 * @vrp: virtual remote processor state495 *496 * This function is called after a sender, that waited for a tx buffer497 * to become available, is unblocked.498 *499 * If we still have blocking senders, this function merely decreases500 * the "sleepers" reference count, and exits.501 *502 * Otherwise, if there are no more blocking senders, we also disable503 * virtio's tx callbacks, to avoid the overhead incurred with handling504 * those (now redundant) interrupts.505 */506static void rpmsg_downref_sleepers(struct virtproc_info *vrp)507{508 /* support multiple concurrent senders */509 mutex_lock(&vrp->tx_lock);510 511 /* are we the last sleeping context waiting for tx buffers ? */512 if (atomic_dec_and_test(&vrp->sleepers))513 /* disable "tx-complete" interrupts */514 virtqueue_disable_cb(vrp->svq);515 516 mutex_unlock(&vrp->tx_lock);517}518 519/**520 * rpmsg_send_offchannel_raw() - send a message across to the remote processor521 * @rpdev: the rpmsg channel522 * @src: source address523 * @dst: destination address524 * @data: payload of message525 * @len: length of payload526 * @wait: indicates whether caller should block in case no TX buffers available527 *528 * This function is the base implementation for all of the rpmsg sending API.529 *530 * It will send @data of length @len to @dst, and say it's from @src. The531 * message will be sent to the remote processor which the @rpdev channel532 * belongs to.533 *534 * The message is sent using one of the TX buffers that are available for535 * communication with this remote processor.536 *537 * If @wait is true, the caller will be blocked until either a TX buffer is538 * available, or 15 seconds elapses (we don't want callers to539 * sleep indefinitely due to misbehaving remote processors), and in that540 * case -ERESTARTSYS is returned. The number '15' itself was picked541 * arbitrarily; there's little point in asking drivers to provide a timeout542 * value themselves.543 *544 * Otherwise, if @wait is false, and there are no TX buffers available,545 * the function will immediately fail, and -ENOMEM will be returned.546 *547 * Normally drivers shouldn't use this function directly; instead, drivers548 * should use the appropriate rpmsg_{try}send{to, _offchannel} API549 * (see include/linux/rpmsg.h).550 *551 * Return: 0 on success and an appropriate error value on failure.552 */553static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev,554 u32 src, u32 dst,555 void *data, int len, bool wait)556{557 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);558 struct virtproc_info *vrp = vch->vrp;559 struct device *dev = &rpdev->dev;560 struct scatterlist sg;561 struct rpmsg_hdr *msg;562 int err;563 564 /* bcasting isn't allowed */565 if (src == RPMSG_ADDR_ANY || dst == RPMSG_ADDR_ANY) {566 dev_err(dev, "invalid addr (src 0x%x, dst 0x%x)\n", src, dst);567 return -EINVAL;568 }569 570 /*571 * We currently use fixed-sized buffers, and therefore the payload572 * length is limited.573 *574 * One of the possible improvements here is either to support575 * user-provided buffers (and then we can also support zero-copy576 * messaging), or to improve the buffer allocator, to support577 * variable-length buffer sizes.578 */579 if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {580 dev_err(dev, "message is too big (%d)\n", len);581 return -EMSGSIZE;582 }583 584 /* grab a buffer */585 msg = get_a_tx_buf(vrp);586 if (!msg && !wait)587 return -ENOMEM;588 589 /* no free buffer ? wait for one (but bail after 15 seconds) */590 while (!msg) {591 /* enable "tx-complete" interrupts, if not already enabled */592 rpmsg_upref_sleepers(vrp);593 594 /*595 * sleep until a free buffer is available or 15 secs elapse.596 * the timeout period is not configurable because there's597 * little point in asking drivers to specify that.598 * if later this happens to be required, it'd be easy to add.599 */600 err = wait_event_interruptible_timeout(vrp->sendq,601 (msg = get_a_tx_buf(vrp)),602 msecs_to_jiffies(15000));603 604 /* disable "tx-complete" interrupts if we're the last sleeper */605 rpmsg_downref_sleepers(vrp);606 607 /* timeout ? */608 if (!err) {609 dev_err(dev, "timeout waiting for a tx buffer\n");610 return -ERESTARTSYS;611 }612 }613 614 msg->len = cpu_to_rpmsg16(rpdev, len);615 msg->flags = 0;616 msg->src = cpu_to_rpmsg32(rpdev, src);617 msg->dst = cpu_to_rpmsg32(rpdev, dst);618 msg->reserved = 0;619 memcpy(msg->data, data, len);620 621 dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Reserved %d\n",622 src, dst, len, msg->flags, msg->reserved);623#if defined(CONFIG_DYNAMIC_DEBUG)624 dynamic_hex_dump("rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1,625 msg, sizeof(*msg) + len, true);626#endif627 628 rpmsg_sg_init(&sg, msg, sizeof(*msg) + len);629 630 mutex_lock(&vrp->tx_lock);631 632 /* add message to the remote processor's virtqueue */633 err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);634 if (err) {635 /*636 * need to reclaim the buffer here, otherwise it's lost637 * (memory won't leak, but rpmsg won't use it again for TX).638 * this will wait for a buffer management overhaul.639 */640 dev_err(dev, "virtqueue_add_outbuf failed: %d\n", err);641 goto out;642 }643 644 /* tell the remote processor it has a pending message to read */645 virtqueue_kick(vrp->svq);646out:647 mutex_unlock(&vrp->tx_lock);648 return err;649}650 651static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len)652{653 struct rpmsg_device *rpdev = ept->rpdev;654 u32 src = ept->addr, dst = rpdev->dst;655 656 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true);657}658 659static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len,660 u32 dst)661{662 struct rpmsg_device *rpdev = ept->rpdev;663 u32 src = ept->addr;664 665 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true);666}667 668static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src,669 u32 dst, void *data, int len)670{671 struct rpmsg_device *rpdev = ept->rpdev;672 673 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true);674}675 676static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len)677{678 struct rpmsg_device *rpdev = ept->rpdev;679 u32 src = ept->addr, dst = rpdev->dst;680 681 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false);682}683 684static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data,685 int len, u32 dst)686{687 struct rpmsg_device *rpdev = ept->rpdev;688 u32 src = ept->addr;689 690 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false);691}692 693static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src,694 u32 dst, void *data, int len)695{696 struct rpmsg_device *rpdev = ept->rpdev;697 698 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false);699}700 701static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept)702{703 struct rpmsg_device *rpdev = ept->rpdev;704 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);705 706 return vch->vrp->buf_size - sizeof(struct rpmsg_hdr);707}708 709static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,710 struct rpmsg_hdr *msg, unsigned int len)711{712 struct rpmsg_endpoint *ept;713 struct scatterlist sg;714 bool little_endian = virtio_is_little_endian(vrp->vdev);715 unsigned int msg_len = __rpmsg16_to_cpu(little_endian, msg->len);716 int err;717 718 dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n",719 __rpmsg32_to_cpu(little_endian, msg->src),720 __rpmsg32_to_cpu(little_endian, msg->dst), msg_len,721 __rpmsg16_to_cpu(little_endian, msg->flags),722 __rpmsg32_to_cpu(little_endian, msg->reserved));723#if defined(CONFIG_DYNAMIC_DEBUG)724 dynamic_hex_dump("rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1,725 msg, sizeof(*msg) + msg_len, true);726#endif727 728 /*729 * We currently use fixed-sized buffers, so trivially sanitize730 * the reported payload length.731 */732 if (len > vrp->buf_size ||733 msg_len > (len - sizeof(struct rpmsg_hdr))) {734 dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg_len);735 return -EINVAL;736 }737 738 /* use the dst addr to fetch the callback of the appropriate user */739 mutex_lock(&vrp->endpoints_lock);740 741 ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst));742 743 /* let's make sure no one deallocates ept while we use it */744 if (ept)745 kref_get(&ept->refcount);746 747 mutex_unlock(&vrp->endpoints_lock);748 749 if (ept) {750 /* make sure ept->cb doesn't go away while we use it */751 mutex_lock(&ept->cb_lock);752 753 if (ept->cb)754 ept->cb(ept->rpdev, msg->data, msg_len, ept->priv,755 __rpmsg32_to_cpu(little_endian, msg->src));756 757 mutex_unlock(&ept->cb_lock);758 759 /* farewell, ept, we don't need you anymore */760 kref_put(&ept->refcount, __ept_release);761 } else762 dev_warn_ratelimited(dev, "msg received with no recipient\n");763 764 /* publish the real size of the buffer */765 rpmsg_sg_init(&sg, msg, vrp->buf_size);766 767 /* add the buffer back to the remote processor's virtqueue */768 err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);769 if (err < 0) {770 dev_err(dev, "failed to add a virtqueue buffer: %d\n", err);771 return err;772 }773 774 return 0;775}776 777/* called when an rx buffer is used, and it's time to digest a message */778static void rpmsg_recv_done(struct virtqueue *rvq)779{780 struct virtproc_info *vrp = rvq->vdev->priv;781 struct device *dev = &rvq->vdev->dev;782 struct rpmsg_hdr *msg;783 unsigned int len, msgs_received = 0;784 int err;785 786 msg = virtqueue_get_buf(rvq, &len);787 if (!msg) {788 dev_err(dev, "uhm, incoming signal, but no used buffer ?\n");789 return;790 }791 792 while (msg) {793 err = rpmsg_recv_single(vrp, dev, msg, len);794 if (err)795 break;796 797 msgs_received++;798 799 msg = virtqueue_get_buf(rvq, &len);800 }801 802 dev_dbg(dev, "Received %u messages\n", msgs_received);803 804 /* tell the remote processor we added another available rx buffer */805 if (msgs_received)806 virtqueue_kick(vrp->rvq);807}808 809/*810 * This is invoked whenever the remote processor completed processing811 * a TX msg we just sent it, and the buffer is put back to the used ring.812 *813 * Normally, though, we suppress this "tx complete" interrupt in order to814 * avoid the incurred overhead.815 */816static void rpmsg_xmit_done(struct virtqueue *svq)817{818 struct virtproc_info *vrp = svq->vdev->priv;819 820 dev_dbg(&svq->vdev->dev, "%s\n", __func__);821 822 /* wake up potential senders that are waiting for a tx buffer */823 wake_up_interruptible(&vrp->sendq);824}825 826/*827 * Called to expose to user a /dev/rpmsg_ctrlX interface allowing to828 * create endpoint-to-endpoint communication without associated RPMsg channel.829 * The endpoints are rattached to the ctrldev RPMsg device.830 */831static struct rpmsg_device *rpmsg_virtio_add_ctrl_dev(struct virtio_device *vdev)832{833 struct virtproc_info *vrp = vdev->priv;834 struct virtio_rpmsg_channel *vch;835 struct rpmsg_device *rpdev_ctrl;836 int err = 0;837 838 vch = kzalloc(sizeof(*vch), GFP_KERNEL);839 if (!vch)840 return ERR_PTR(-ENOMEM);841 842 /* Link the channel to the vrp */843 vch->vrp = vrp;844 845 /* Assign public information to the rpmsg_device */846 rpdev_ctrl = &vch->rpdev;847 rpdev_ctrl->ops = &virtio_rpmsg_ops;848 849 rpdev_ctrl->dev.parent = &vrp->vdev->dev;850 rpdev_ctrl->dev.release = virtio_rpmsg_release_device;851 rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev);852 853 err = rpmsg_ctrldev_register_device(rpdev_ctrl);854 if (err) {855 /* vch will be free in virtio_rpmsg_release_device() */856 return ERR_PTR(err);857 }858 859 return rpdev_ctrl;860}861 862static void rpmsg_virtio_del_ctrl_dev(struct rpmsg_device *rpdev_ctrl)863{864 if (!rpdev_ctrl)865 return;866 device_unregister(&rpdev_ctrl->dev);867}868 869static int rpmsg_probe(struct virtio_device *vdev)870{871 struct virtqueue_info vqs_info[] = {872 { "input", rpmsg_recv_done },873 { "output", rpmsg_xmit_done },874 };875 struct virtqueue *vqs[2];876 struct virtproc_info *vrp;877 struct virtio_rpmsg_channel *vch = NULL;878 struct rpmsg_device *rpdev_ns, *rpdev_ctrl;879 void *bufs_va;880 int err = 0, i;881 size_t total_buf_space;882 bool notify;883 884 vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);885 if (!vrp)886 return -ENOMEM;887 888 vrp->vdev = vdev;889 890 idr_init(&vrp->endpoints);891 mutex_init(&vrp->endpoints_lock);892 mutex_init(&vrp->tx_lock);893 init_waitqueue_head(&vrp->sendq);894 895 /* We expect two virtqueues, rx and tx (and in this order) */896 err = virtio_find_vqs(vdev, 2, vqs, vqs_info, NULL);897 if (err)898 goto free_vrp;899 900 vrp->rvq = vqs[0];901 vrp->svq = vqs[1];902 903 /* we expect symmetric tx/rx vrings */904 WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=905 virtqueue_get_vring_size(vrp->svq));906 907 /* we need less buffers if vrings are small */908 if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)909 vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;910 else911 vrp->num_bufs = MAX_RPMSG_NUM_BUFS;912 913 vrp->buf_size = MAX_RPMSG_BUF_SIZE;914 915 total_buf_space = vrp->num_bufs * vrp->buf_size;916 917 /* allocate coherent memory for the buffers */918 bufs_va = dma_alloc_coherent(vdev->dev.parent,919 total_buf_space, &vrp->bufs_dma,920 GFP_KERNEL);921 if (!bufs_va) {922 err = -ENOMEM;923 goto vqs_del;924 }925 926 dev_dbg(&vdev->dev, "buffers: va %pK, dma %pad\n",927 bufs_va, &vrp->bufs_dma);928 929 /* half of the buffers is dedicated for RX */930 vrp->rbufs = bufs_va;931 932 /* and half is dedicated for TX */933 vrp->sbufs = bufs_va + total_buf_space / 2;934 935 /* set up the receive buffers */936 for (i = 0; i < vrp->num_bufs / 2; i++) {937 struct scatterlist sg;938 void *cpu_addr = vrp->rbufs + i * vrp->buf_size;939 940 rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);941 942 err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,943 GFP_KERNEL);944 WARN_ON(err); /* sanity check; this can't really happen */945 }946 947 /* suppress "tx-complete" interrupts */948 virtqueue_disable_cb(vrp->svq);949 950 vdev->priv = vrp;951 952 rpdev_ctrl = rpmsg_virtio_add_ctrl_dev(vdev);953 if (IS_ERR(rpdev_ctrl)) {954 err = PTR_ERR(rpdev_ctrl);955 goto free_coherent;956 }957 958 /* if supported by the remote processor, enable the name service */959 if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {960 vch = kzalloc(sizeof(*vch), GFP_KERNEL);961 if (!vch) {962 err = -ENOMEM;963 goto free_ctrldev;964 }965 966 /* Link the channel to our vrp */967 vch->vrp = vrp;968 969 /* Assign public information to the rpmsg_device */970 rpdev_ns = &vch->rpdev;971 rpdev_ns->ops = &virtio_rpmsg_ops;972 rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev);973 974 rpdev_ns->dev.parent = &vrp->vdev->dev;975 rpdev_ns->dev.release = virtio_rpmsg_release_device;976 977 err = rpmsg_ns_register_device(rpdev_ns);978 if (err)979 /* vch will be free in virtio_rpmsg_release_device() */980 goto free_ctrldev;981 }982 983 /*984 * Prepare to kick but don't notify yet - we can't do this before985 * device is ready.986 */987 notify = virtqueue_kick_prepare(vrp->rvq);988 989 /* From this point on, we can notify and get callbacks. */990 virtio_device_ready(vdev);991 992 /* tell the remote processor it can start sending messages */993 /*994 * this might be concurrent with callbacks, but we are only995 * doing notify, not a full kick here, so that's ok.996 */997 if (notify)998 virtqueue_notify(vrp->rvq);999 1000 dev_info(&vdev->dev, "rpmsg host is online\n");1001 1002 return 0;1003 1004free_ctrldev:1005 rpmsg_virtio_del_ctrl_dev(rpdev_ctrl);1006free_coherent:1007 dma_free_coherent(vdev->dev.parent, total_buf_space,1008 bufs_va, vrp->bufs_dma);1009vqs_del:1010 vdev->config->del_vqs(vrp->vdev);1011free_vrp:1012 kfree(vrp);1013 return err;1014}1015 1016static int rpmsg_remove_device(struct device *dev, void *data)1017{1018 device_unregister(dev);1019 1020 return 0;1021}1022 1023static void rpmsg_remove(struct virtio_device *vdev)1024{1025 struct virtproc_info *vrp = vdev->priv;1026 size_t total_buf_space = vrp->num_bufs * vrp->buf_size;1027 int ret;1028 1029 virtio_reset_device(vdev);1030 1031 ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device);1032 if (ret)1033 dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret);1034 1035 idr_destroy(&vrp->endpoints);1036 1037 vdev->config->del_vqs(vrp->vdev);1038 1039 dma_free_coherent(vdev->dev.parent, total_buf_space,1040 vrp->rbufs, vrp->bufs_dma);1041 1042 kfree(vrp);1043}1044 1045static struct virtio_device_id id_table[] = {1046 { VIRTIO_ID_RPMSG, VIRTIO_DEV_ANY_ID },1047 { 0 },1048};1049 1050static unsigned int features[] = {1051 VIRTIO_RPMSG_F_NS,1052};1053 1054static struct virtio_driver virtio_ipc_driver = {1055 .feature_table = features,1056 .feature_table_size = ARRAY_SIZE(features),1057 .driver.name = KBUILD_MODNAME,1058 .id_table = id_table,1059 .probe = rpmsg_probe,1060 .remove = rpmsg_remove,1061};1062 1063static int __init rpmsg_init(void)1064{1065 int ret;1066 1067 ret = register_virtio_driver(&virtio_ipc_driver);1068 if (ret)1069 pr_err("failed to register virtio driver: %d\n", ret);1070 1071 return ret;1072}1073subsys_initcall(rpmsg_init);1074 1075static void __exit rpmsg_fini(void)1076{1077 unregister_virtio_driver(&virtio_ipc_driver);1078}1079module_exit(rpmsg_fini);1080 1081MODULE_DEVICE_TABLE(virtio, id_table);1082MODULE_DESCRIPTION("Virtio-based remote processor messaging bus");1083MODULE_LICENSE("GPL v2");1084