1821 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2009, Microsoft Corporation.4 *5 * Authors:6 * Haiyang Zhang <haiyangz@microsoft.com>7 * Hank Janssen <hjanssen@microsoft.com>8 */9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt10 11#include <linux/kernel.h>12#include <linux/sched.h>13#include <linux/wait.h>14#include <linux/mm.h>15#include <linux/delay.h>16#include <linux/io.h>17#include <linux/slab.h>18#include <linux/netdevice.h>19#include <linux/if_ether.h>20#include <linux/vmalloc.h>21#include <linux/rtnetlink.h>22#include <linux/prefetch.h>23#include <linux/filter.h>24 25#include <asm/sync_bitops.h>26#include <asm/mshyperv.h>27 28#include "hyperv_net.h"29#include "netvsc_trace.h"30 31/*32 * Switch the data path from the synthetic interface to the VF33 * interface.34 */35int netvsc_switch_datapath(struct net_device *ndev, bool vf)36{37 struct net_device_context *net_device_ctx = netdev_priv(ndev);38 struct hv_device *dev = net_device_ctx->device_ctx;39 struct netvsc_device *nv_dev = rtnl_dereference(net_device_ctx->nvdev);40 struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt;41 int ret, retry = 0;42 43 /* Block sending traffic to VF if it's about to be gone */44 if (!vf)45 net_device_ctx->data_path_is_vf = vf;46 47 memset(init_pkt, 0, sizeof(struct nvsp_message));48 init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH;49 if (vf)50 init_pkt->msg.v4_msg.active_dp.active_datapath =51 NVSP_DATAPATH_VF;52 else53 init_pkt->msg.v4_msg.active_dp.active_datapath =54 NVSP_DATAPATH_SYNTHETIC;55 56again:57 trace_nvsp_send(ndev, init_pkt);58 59 ret = vmbus_sendpacket(dev->channel, init_pkt,60 sizeof(struct nvsp_message),61 (unsigned long)init_pkt, VM_PKT_DATA_INBAND,62 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);63 64 /* If failed to switch to/from VF, let data_path_is_vf stay false,65 * so we use synthetic path to send data.66 */67 if (ret) {68 if (ret != -EAGAIN) {69 netdev_err(ndev,70 "Unable to send sw datapath msg, err: %d\n",71 ret);72 return ret;73 }74 75 if (retry++ < RETRY_MAX) {76 usleep_range(RETRY_US_LO, RETRY_US_HI);77 goto again;78 } else {79 netdev_err(80 ndev,81 "Retry failed to send sw datapath msg, err: %d\n",82 ret);83 return ret;84 }85 }86 87 wait_for_completion(&nv_dev->channel_init_wait);88 net_device_ctx->data_path_is_vf = vf;89 90 return 0;91}92 93/* Worker to setup sub channels on initial setup94 * Initial hotplug event occurs in softirq context95 * and can't wait for channels.96 */97static void netvsc_subchan_work(struct work_struct *w)98{99 struct netvsc_device *nvdev =100 container_of(w, struct netvsc_device, subchan_work);101 struct rndis_device *rdev;102 int i, ret;103 104 /* Avoid deadlock with device removal already under RTNL */105 if (!rtnl_trylock()) {106 schedule_work(w);107 return;108 }109 110 rdev = nvdev->extension;111 if (rdev) {112 ret = rndis_set_subchannel(rdev->ndev, nvdev, NULL);113 if (ret == 0) {114 netif_device_attach(rdev->ndev);115 } else {116 /* fallback to only primary channel */117 for (i = 1; i < nvdev->num_chn; i++)118 netif_napi_del(&nvdev->chan_table[i].napi);119 120 nvdev->max_chn = 1;121 nvdev->num_chn = 1;122 }123 }124 125 rtnl_unlock();126}127 128static struct netvsc_device *alloc_net_device(void)129{130 struct netvsc_device *net_device;131 132 net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL);133 if (!net_device)134 return NULL;135 136 init_waitqueue_head(&net_device->wait_drain);137 net_device->destroy = false;138 net_device->tx_disable = true;139 140 net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;141 net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;142 143 init_completion(&net_device->channel_init_wait);144 init_waitqueue_head(&net_device->subchan_open);145 INIT_WORK(&net_device->subchan_work, netvsc_subchan_work);146 147 return net_device;148}149 150static void free_netvsc_device(struct rcu_head *head)151{152 struct netvsc_device *nvdev153 = container_of(head, struct netvsc_device, rcu);154 int i;155 156 kfree(nvdev->extension);157 158 if (!nvdev->recv_buf_gpadl_handle.decrypted)159 vfree(nvdev->recv_buf);160 if (!nvdev->send_buf_gpadl_handle.decrypted)161 vfree(nvdev->send_buf);162 bitmap_free(nvdev->send_section_map);163 164 for (i = 0; i < VRSS_CHANNEL_MAX; i++) {165 xdp_rxq_info_unreg(&nvdev->chan_table[i].xdp_rxq);166 kfree(nvdev->chan_table[i].recv_buf);167 vfree(nvdev->chan_table[i].mrc.slots);168 }169 170 kfree(nvdev);171}172 173static void free_netvsc_device_rcu(struct netvsc_device *nvdev)174{175 call_rcu(&nvdev->rcu, free_netvsc_device);176}177 178static void netvsc_revoke_recv_buf(struct hv_device *device,179 struct netvsc_device *net_device,180 struct net_device *ndev)181{182 struct nvsp_message *revoke_packet;183 int ret;184 185 /*186 * If we got a section count, it means we received a187 * SendReceiveBufferComplete msg (ie sent188 * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need189 * to send a revoke msg here190 */191 if (net_device->recv_section_cnt) {192 /* Send the revoke receive buffer */193 revoke_packet = &net_device->revoke_packet;194 memset(revoke_packet, 0, sizeof(struct nvsp_message));195 196 revoke_packet->hdr.msg_type =197 NVSP_MSG1_TYPE_REVOKE_RECV_BUF;198 revoke_packet->msg.v1_msg.199 revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID;200 201 trace_nvsp_send(ndev, revoke_packet);202 203 ret = vmbus_sendpacket(device->channel,204 revoke_packet,205 sizeof(struct nvsp_message),206 VMBUS_RQST_ID_NO_RESPONSE,207 VM_PKT_DATA_INBAND, 0);208 /* If the failure is because the channel is rescinded;209 * ignore the failure since we cannot send on a rescinded210 * channel. This would allow us to properly cleanup211 * even when the channel is rescinded.212 */213 if (device->channel->rescind)214 ret = 0;215 /*216 * If we failed here, we might as well return and217 * have a leak rather than continue and a bugchk218 */219 if (ret != 0) {220 netdev_err(ndev, "unable to send "221 "revoke receive buffer to netvsp\n");222 return;223 }224 net_device->recv_section_cnt = 0;225 }226}227 228static void netvsc_revoke_send_buf(struct hv_device *device,229 struct netvsc_device *net_device,230 struct net_device *ndev)231{232 struct nvsp_message *revoke_packet;233 int ret;234 235 /* Deal with the send buffer we may have setup.236 * If we got a send section size, it means we received a237 * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent238 * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need239 * to send a revoke msg here240 */241 if (net_device->send_section_cnt) {242 /* Send the revoke receive buffer */243 revoke_packet = &net_device->revoke_packet;244 memset(revoke_packet, 0, sizeof(struct nvsp_message));245 246 revoke_packet->hdr.msg_type =247 NVSP_MSG1_TYPE_REVOKE_SEND_BUF;248 revoke_packet->msg.v1_msg.revoke_send_buf.id =249 NETVSC_SEND_BUFFER_ID;250 251 trace_nvsp_send(ndev, revoke_packet);252 253 ret = vmbus_sendpacket(device->channel,254 revoke_packet,255 sizeof(struct nvsp_message),256 VMBUS_RQST_ID_NO_RESPONSE,257 VM_PKT_DATA_INBAND, 0);258 259 /* If the failure is because the channel is rescinded;260 * ignore the failure since we cannot send on a rescinded261 * channel. This would allow us to properly cleanup262 * even when the channel is rescinded.263 */264 if (device->channel->rescind)265 ret = 0;266 267 /* If we failed here, we might as well return and268 * have a leak rather than continue and a bugchk269 */270 if (ret != 0) {271 netdev_err(ndev, "unable to send "272 "revoke send buffer to netvsp\n");273 return;274 }275 net_device->send_section_cnt = 0;276 }277}278 279static void netvsc_teardown_recv_gpadl(struct hv_device *device,280 struct netvsc_device *net_device,281 struct net_device *ndev)282{283 int ret;284 285 if (net_device->recv_buf_gpadl_handle.gpadl_handle) {286 ret = vmbus_teardown_gpadl(device->channel,287 &net_device->recv_buf_gpadl_handle);288 289 /* If we failed here, we might as well return and have a leak290 * rather than continue and a bugchk291 */292 if (ret != 0) {293 netdev_err(ndev,294 "unable to teardown receive buffer's gpadl\n");295 return;296 }297 }298}299 300static void netvsc_teardown_send_gpadl(struct hv_device *device,301 struct netvsc_device *net_device,302 struct net_device *ndev)303{304 int ret;305 306 if (net_device->send_buf_gpadl_handle.gpadl_handle) {307 ret = vmbus_teardown_gpadl(device->channel,308 &net_device->send_buf_gpadl_handle);309 310 /* If we failed here, we might as well return and have a leak311 * rather than continue and a bugchk312 */313 if (ret != 0) {314 netdev_err(ndev,315 "unable to teardown send buffer's gpadl\n");316 return;317 }318 }319}320 321int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx)322{323 struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];324 int node = cpu_to_node(nvchan->channel->target_cpu);325 size_t size;326 327 size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data);328 nvchan->mrc.slots = vzalloc_node(size, node);329 if (!nvchan->mrc.slots)330 nvchan->mrc.slots = vzalloc(size);331 332 return nvchan->mrc.slots ? 0 : -ENOMEM;333}334 335static int netvsc_init_buf(struct hv_device *device,336 struct netvsc_device *net_device,337 const struct netvsc_device_info *device_info)338{339 struct nvsp_1_message_send_receive_buffer_complete *resp;340 struct net_device *ndev = hv_get_drvdata(device);341 struct nvsp_message *init_packet;342 unsigned int buf_size;343 int i, ret = 0;344 345 /* Get receive buffer area. */346 buf_size = device_info->recv_sections * device_info->recv_section_size;347 buf_size = roundup(buf_size, PAGE_SIZE);348 349 /* Legacy hosts only allow smaller receive buffer */350 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2)351 buf_size = min_t(unsigned int, buf_size,352 NETVSC_RECEIVE_BUFFER_SIZE_LEGACY);353 354 net_device->recv_buf = vzalloc(buf_size);355 if (!net_device->recv_buf) {356 netdev_err(ndev,357 "unable to allocate receive buffer of size %u\n",358 buf_size);359 ret = -ENOMEM;360 goto cleanup;361 }362 363 net_device->recv_buf_size = buf_size;364 365 /*366 * Establish the gpadl handle for this buffer on this367 * channel. Note: This call uses the vmbus connection rather368 * than the channel to establish the gpadl handle.369 */370 ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf,371 buf_size,372 &net_device->recv_buf_gpadl_handle);373 if (ret != 0) {374 netdev_err(ndev,375 "unable to establish receive buffer's gpadl\n");376 goto cleanup;377 }378 379 /* Notify the NetVsp of the gpadl handle */380 init_packet = &net_device->channel_init_pkt;381 memset(init_packet, 0, sizeof(struct nvsp_message));382 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF;383 init_packet->msg.v1_msg.send_recv_buf.384 gpadl_handle = net_device->recv_buf_gpadl_handle.gpadl_handle;385 init_packet->msg.v1_msg.386 send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID;387 388 trace_nvsp_send(ndev, init_packet);389 390 /* Send the gpadl notification request */391 ret = vmbus_sendpacket(device->channel, init_packet,392 sizeof(struct nvsp_message),393 (unsigned long)init_packet,394 VM_PKT_DATA_INBAND,395 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);396 if (ret != 0) {397 netdev_err(ndev,398 "unable to send receive buffer's gpadl to netvsp\n");399 goto cleanup;400 }401 402 wait_for_completion(&net_device->channel_init_wait);403 404 /* Check the response */405 resp = &init_packet->msg.v1_msg.send_recv_buf_complete;406 if (resp->status != NVSP_STAT_SUCCESS) {407 netdev_err(ndev,408 "Unable to complete receive buffer initialization with NetVsp - status %d\n",409 resp->status);410 ret = -EINVAL;411 goto cleanup;412 }413 414 /* Parse the response */415 netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n",416 resp->num_sections, resp->sections[0].sub_alloc_size,417 resp->sections[0].num_sub_allocs);418 419 /* There should only be one section for the entire receive buffer */420 if (resp->num_sections != 1 || resp->sections[0].offset != 0) {421 ret = -EINVAL;422 goto cleanup;423 }424 425 net_device->recv_section_size = resp->sections[0].sub_alloc_size;426 net_device->recv_section_cnt = resp->sections[0].num_sub_allocs;427 428 /* Ensure buffer will not overflow */429 if (net_device->recv_section_size < NETVSC_MTU_MIN || (u64)net_device->recv_section_size *430 (u64)net_device->recv_section_cnt > (u64)buf_size) {431 netdev_err(ndev, "invalid recv_section_size %u\n",432 net_device->recv_section_size);433 ret = -EINVAL;434 goto cleanup;435 }436 437 for (i = 0; i < VRSS_CHANNEL_MAX; i++) {438 struct netvsc_channel *nvchan = &net_device->chan_table[i];439 440 nvchan->recv_buf = kzalloc(net_device->recv_section_size, GFP_KERNEL);441 if (nvchan->recv_buf == NULL) {442 ret = -ENOMEM;443 goto cleanup;444 }445 }446 447 /* Setup receive completion ring.448 * Add 1 to the recv_section_cnt because at least one entry in a449 * ring buffer has to be empty.450 */451 net_device->recv_completion_cnt = net_device->recv_section_cnt + 1;452 ret = netvsc_alloc_recv_comp_ring(net_device, 0);453 if (ret)454 goto cleanup;455 456 /* Now setup the send buffer. */457 buf_size = device_info->send_sections * device_info->send_section_size;458 buf_size = round_up(buf_size, PAGE_SIZE);459 460 net_device->send_buf = vzalloc(buf_size);461 if (!net_device->send_buf) {462 netdev_err(ndev, "unable to allocate send buffer of size %u\n",463 buf_size);464 ret = -ENOMEM;465 goto cleanup;466 }467 net_device->send_buf_size = buf_size;468 469 /* Establish the gpadl handle for this buffer on this470 * channel. Note: This call uses the vmbus connection rather471 * than the channel to establish the gpadl handle.472 */473 ret = vmbus_establish_gpadl(device->channel, net_device->send_buf,474 buf_size,475 &net_device->send_buf_gpadl_handle);476 if (ret != 0) {477 netdev_err(ndev,478 "unable to establish send buffer's gpadl\n");479 goto cleanup;480 }481 482 /* Notify the NetVsp of the gpadl handle */483 init_packet = &net_device->channel_init_pkt;484 memset(init_packet, 0, sizeof(struct nvsp_message));485 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF;486 init_packet->msg.v1_msg.send_send_buf.gpadl_handle =487 net_device->send_buf_gpadl_handle.gpadl_handle;488 init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID;489 490 trace_nvsp_send(ndev, init_packet);491 492 /* Send the gpadl notification request */493 ret = vmbus_sendpacket(device->channel, init_packet,494 sizeof(struct nvsp_message),495 (unsigned long)init_packet,496 VM_PKT_DATA_INBAND,497 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);498 if (ret != 0) {499 netdev_err(ndev,500 "unable to send send buffer's gpadl to netvsp\n");501 goto cleanup;502 }503 504 wait_for_completion(&net_device->channel_init_wait);505 506 /* Check the response */507 if (init_packet->msg.v1_msg.508 send_send_buf_complete.status != NVSP_STAT_SUCCESS) {509 netdev_err(ndev, "Unable to complete send buffer "510 "initialization with NetVsp - status %d\n",511 init_packet->msg.v1_msg.512 send_send_buf_complete.status);513 ret = -EINVAL;514 goto cleanup;515 }516 517 /* Parse the response */518 net_device->send_section_size = init_packet->msg.519 v1_msg.send_send_buf_complete.section_size;520 if (net_device->send_section_size < NETVSC_MTU_MIN) {521 netdev_err(ndev, "invalid send_section_size %u\n",522 net_device->send_section_size);523 ret = -EINVAL;524 goto cleanup;525 }526 527 /* Section count is simply the size divided by the section size. */528 net_device->send_section_cnt = buf_size / net_device->send_section_size;529 530 netdev_dbg(ndev, "Send section size: %d, Section count:%d\n",531 net_device->send_section_size, net_device->send_section_cnt);532 533 /* Setup state for managing the send buffer. */534 net_device->send_section_map = bitmap_zalloc(net_device->send_section_cnt,535 GFP_KERNEL);536 if (!net_device->send_section_map) {537 ret = -ENOMEM;538 goto cleanup;539 }540 541 goto exit;542 543cleanup:544 netvsc_revoke_recv_buf(device, net_device, ndev);545 netvsc_revoke_send_buf(device, net_device, ndev);546 netvsc_teardown_recv_gpadl(device, net_device, ndev);547 netvsc_teardown_send_gpadl(device, net_device, ndev);548 549exit:550 return ret;551}552 553/* Negotiate NVSP protocol version */554static int negotiate_nvsp_ver(struct hv_device *device,555 struct netvsc_device *net_device,556 struct nvsp_message *init_packet,557 u32 nvsp_ver)558{559 struct net_device *ndev = hv_get_drvdata(device);560 int ret;561 562 memset(init_packet, 0, sizeof(struct nvsp_message));563 init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT;564 init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver;565 init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver;566 trace_nvsp_send(ndev, init_packet);567 568 /* Send the init request */569 ret = vmbus_sendpacket(device->channel, init_packet,570 sizeof(struct nvsp_message),571 (unsigned long)init_packet,572 VM_PKT_DATA_INBAND,573 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);574 575 if (ret != 0)576 return ret;577 578 wait_for_completion(&net_device->channel_init_wait);579 580 if (init_packet->msg.init_msg.init_complete.status !=581 NVSP_STAT_SUCCESS)582 return -EINVAL;583 584 if (nvsp_ver == NVSP_PROTOCOL_VERSION_1)585 return 0;586 587 /* NVSPv2 or later: Send NDIS config */588 memset(init_packet, 0, sizeof(struct nvsp_message));589 init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG;590 init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN;591 init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1;592 593 if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) {594 if (hv_is_isolation_supported())595 netdev_info(ndev, "SR-IOV not advertised by guests on the host supporting isolation\n");596 else597 init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1;598 599 /* Teaming bit is needed to receive link speed updates */600 init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1;601 }602 603 if (nvsp_ver >= NVSP_PROTOCOL_VERSION_61)604 init_packet->msg.v2_msg.send_ndis_config.capability.rsc = 1;605 606 trace_nvsp_send(ndev, init_packet);607 608 ret = vmbus_sendpacket(device->channel, init_packet,609 sizeof(struct nvsp_message),610 VMBUS_RQST_ID_NO_RESPONSE,611 VM_PKT_DATA_INBAND, 0);612 613 return ret;614}615 616static int netvsc_connect_vsp(struct hv_device *device,617 struct netvsc_device *net_device,618 const struct netvsc_device_info *device_info)619{620 struct net_device *ndev = hv_get_drvdata(device);621 static const u32 ver_list[] = {622 NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2,623 NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5,624 NVSP_PROTOCOL_VERSION_6, NVSP_PROTOCOL_VERSION_61625 };626 struct nvsp_message *init_packet;627 int ndis_version, i, ret;628 629 init_packet = &net_device->channel_init_pkt;630 631 /* Negotiate the latest NVSP protocol supported */632 for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--)633 if (negotiate_nvsp_ver(device, net_device, init_packet,634 ver_list[i]) == 0) {635 net_device->nvsp_version = ver_list[i];636 break;637 }638 639 if (i < 0) {640 ret = -EPROTO;641 goto cleanup;642 }643 644 if (hv_is_isolation_supported() && net_device->nvsp_version < NVSP_PROTOCOL_VERSION_61) {645 netdev_err(ndev, "Invalid NVSP version 0x%x (expected >= 0x%x) from the host supporting isolation\n",646 net_device->nvsp_version, NVSP_PROTOCOL_VERSION_61);647 ret = -EPROTO;648 goto cleanup;649 }650 651 pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version);652 653 /* Send the ndis version */654 memset(init_packet, 0, sizeof(struct nvsp_message));655 656 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4)657 ndis_version = 0x00060001;658 else659 ndis_version = 0x0006001e;660 661 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER;662 init_packet->msg.v1_msg.663 send_ndis_ver.ndis_major_ver =664 (ndis_version & 0xFFFF0000) >> 16;665 init_packet->msg.v1_msg.666 send_ndis_ver.ndis_minor_ver =667 ndis_version & 0xFFFF;668 669 trace_nvsp_send(ndev, init_packet);670 671 /* Send the init request */672 ret = vmbus_sendpacket(device->channel, init_packet,673 sizeof(struct nvsp_message),674 VMBUS_RQST_ID_NO_RESPONSE,675 VM_PKT_DATA_INBAND, 0);676 if (ret != 0)677 goto cleanup;678 679 680 ret = netvsc_init_buf(device, net_device, device_info);681 682cleanup:683 return ret;684}685 686/*687 * netvsc_device_remove - Callback when the root bus device is removed688 */689void netvsc_device_remove(struct hv_device *device)690{691 struct net_device *ndev = hv_get_drvdata(device);692 struct net_device_context *net_device_ctx = netdev_priv(ndev);693 struct netvsc_device *net_device694 = rtnl_dereference(net_device_ctx->nvdev);695 int i;696 697 /*698 * Revoke receive buffer. If host is pre-Win2016 then tear down699 * receive buffer GPADL. Do the same for send buffer.700 */701 netvsc_revoke_recv_buf(device, net_device, ndev);702 if (vmbus_proto_version < VERSION_WIN10)703 netvsc_teardown_recv_gpadl(device, net_device, ndev);704 705 netvsc_revoke_send_buf(device, net_device, ndev);706 if (vmbus_proto_version < VERSION_WIN10)707 netvsc_teardown_send_gpadl(device, net_device, ndev);708 709 RCU_INIT_POINTER(net_device_ctx->nvdev, NULL);710 711 /* Disable NAPI and disassociate its context from the device. */712 for (i = 0; i < net_device->num_chn; i++) {713 /* See also vmbus_reset_channel_cb(). */714 /* only disable enabled NAPI channel */715 if (i < ndev->real_num_rx_queues)716 napi_disable(&net_device->chan_table[i].napi);717 718 netif_napi_del(&net_device->chan_table[i].napi);719 }720 721 /*722 * At this point, no one should be accessing net_device723 * except in here724 */725 netdev_dbg(ndev, "net device safe to remove\n");726 727 /* Now, we can close the channel safely */728 vmbus_close(device->channel);729 730 /*731 * If host is Win2016 or higher then we do the GPADL tear down732 * here after VMBus is closed.733 */734 if (vmbus_proto_version >= VERSION_WIN10) {735 netvsc_teardown_recv_gpadl(device, net_device, ndev);736 netvsc_teardown_send_gpadl(device, net_device, ndev);737 }738 739 /* Release all resources */740 free_netvsc_device_rcu(net_device);741}742 743#define RING_AVAIL_PERCENT_HIWATER 20744#define RING_AVAIL_PERCENT_LOWATER 10745 746static inline void netvsc_free_send_slot(struct netvsc_device *net_device,747 u32 index)748{749 sync_change_bit(index, net_device->send_section_map);750}751 752static void netvsc_send_tx_complete(struct net_device *ndev,753 struct netvsc_device *net_device,754 struct vmbus_channel *channel,755 const struct vmpacket_descriptor *desc,756 int budget)757{758 struct net_device_context *ndev_ctx = netdev_priv(ndev);759 struct sk_buff *skb;760 u16 q_idx = 0;761 int queue_sends;762 u64 cmd_rqst;763 764 cmd_rqst = channel->request_addr_callback(channel, desc->trans_id);765 if (cmd_rqst == VMBUS_RQST_ERROR) {766 netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id);767 return;768 }769 770 skb = (struct sk_buff *)(unsigned long)cmd_rqst;771 772 /* Notify the layer above us */773 if (likely(skb)) {774 struct hv_netvsc_packet *packet775 = (struct hv_netvsc_packet *)skb->cb;776 u32 send_index = packet->send_buf_index;777 struct netvsc_stats_tx *tx_stats;778 779 if (send_index != NETVSC_INVALID_INDEX)780 netvsc_free_send_slot(net_device, send_index);781 q_idx = packet->q_idx;782 783 tx_stats = &net_device->chan_table[q_idx].tx_stats;784 785 u64_stats_update_begin(&tx_stats->syncp);786 tx_stats->packets += packet->total_packets;787 tx_stats->bytes += packet->total_bytes;788 u64_stats_update_end(&tx_stats->syncp);789 790 netvsc_dma_unmap(ndev_ctx->device_ctx, packet);791 napi_consume_skb(skb, budget);792 }793 794 queue_sends =795 atomic_dec_return(&net_device->chan_table[q_idx].queue_sends);796 797 if (unlikely(net_device->destroy)) {798 if (queue_sends == 0)799 wake_up(&net_device->wait_drain);800 } else {801 struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx);802 803 if (netif_tx_queue_stopped(txq) && !net_device->tx_disable &&804 (hv_get_avail_to_write_percent(&channel->outbound) >805 RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) {806 netif_tx_wake_queue(txq);807 ndev_ctx->eth_stats.wake_queue++;808 }809 }810}811 812static void netvsc_send_completion(struct net_device *ndev,813 struct netvsc_device *net_device,814 struct vmbus_channel *incoming_channel,815 const struct vmpacket_descriptor *desc,816 int budget)817{818 const struct nvsp_message *nvsp_packet;819 u32 msglen = hv_pkt_datalen(desc);820 struct nvsp_message *pkt_rqst;821 u64 cmd_rqst;822 u32 status;823 824 /* First check if this is a VMBUS completion without data payload */825 if (!msglen) {826 cmd_rqst = incoming_channel->request_addr_callback(incoming_channel,827 desc->trans_id);828 if (cmd_rqst == VMBUS_RQST_ERROR) {829 netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id);830 return;831 }832 833 pkt_rqst = (struct nvsp_message *)(uintptr_t)cmd_rqst;834 switch (pkt_rqst->hdr.msg_type) {835 case NVSP_MSG4_TYPE_SWITCH_DATA_PATH:836 complete(&net_device->channel_init_wait);837 break;838 839 default:840 netdev_err(ndev, "Unexpected VMBUS completion!!\n");841 }842 return;843 }844 845 /* Ensure packet is big enough to read header fields */846 if (msglen < sizeof(struct nvsp_message_header)) {847 netdev_err(ndev, "nvsp_message length too small: %u\n", msglen);848 return;849 }850 851 nvsp_packet = hv_pkt_data(desc);852 switch (nvsp_packet->hdr.msg_type) {853 case NVSP_MSG_TYPE_INIT_COMPLETE:854 if (msglen < sizeof(struct nvsp_message_header) +855 sizeof(struct nvsp_message_init_complete)) {856 netdev_err(ndev, "nvsp_msg length too small: %u\n",857 msglen);858 return;859 }860 break;861 862 case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE:863 if (msglen < sizeof(struct nvsp_message_header) +864 sizeof(struct nvsp_1_message_send_receive_buffer_complete)) {865 netdev_err(ndev, "nvsp_msg1 length too small: %u\n",866 msglen);867 return;868 }869 break;870 871 case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE:872 if (msglen < sizeof(struct nvsp_message_header) +873 sizeof(struct nvsp_1_message_send_send_buffer_complete)) {874 netdev_err(ndev, "nvsp_msg1 length too small: %u\n",875 msglen);876 return;877 }878 break;879 880 case NVSP_MSG5_TYPE_SUBCHANNEL:881 if (msglen < sizeof(struct nvsp_message_header) +882 sizeof(struct nvsp_5_subchannel_complete)) {883 netdev_err(ndev, "nvsp_msg5 length too small: %u\n",884 msglen);885 return;886 }887 break;888 889 case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE:890 if (msglen < sizeof(struct nvsp_message_header) +891 sizeof(struct nvsp_1_message_send_rndis_packet_complete)) {892 if (net_ratelimit())893 netdev_err(ndev, "nvsp_rndis_pkt_complete length too small: %u\n",894 msglen);895 return;896 }897 898 /* If status indicates an error, output a message so we know899 * there's a problem. But process the completion anyway so the900 * resources are released.901 */902 status = nvsp_packet->msg.v1_msg.send_rndis_pkt_complete.status;903 if (status != NVSP_STAT_SUCCESS && net_ratelimit())904 netdev_err(ndev, "nvsp_rndis_pkt_complete error status: %x\n",905 status);906 907 netvsc_send_tx_complete(ndev, net_device, incoming_channel,908 desc, budget);909 return;910 911 default:912 netdev_err(ndev,913 "Unknown send completion type %d received!!\n",914 nvsp_packet->hdr.msg_type);915 return;916 }917 918 /* Copy the response back */919 memcpy(&net_device->channel_init_pkt, nvsp_packet,920 sizeof(struct nvsp_message));921 complete(&net_device->channel_init_wait);922}923 924static u32 netvsc_get_next_send_section(struct netvsc_device *net_device)925{926 unsigned long *map_addr = net_device->send_section_map;927 unsigned int i;928 929 for_each_clear_bit(i, map_addr, net_device->send_section_cnt) {930 if (sync_test_and_set_bit(i, map_addr) == 0)931 return i;932 }933 934 return NETVSC_INVALID_INDEX;935}936 937static void netvsc_copy_to_send_buf(struct netvsc_device *net_device,938 unsigned int section_index,939 u32 pend_size,940 struct hv_netvsc_packet *packet,941 struct rndis_message *rndis_msg,942 struct hv_page_buffer *pb,943 bool xmit_more)944{945 char *start = net_device->send_buf;946 char *dest = start + (section_index * net_device->send_section_size)947 + pend_size;948 int i;949 u32 padding = 0;950 u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt :951 packet->page_buf_cnt;952 u32 remain;953 954 /* Add padding */955 remain = packet->total_data_buflen & (net_device->pkt_align - 1);956 if (xmit_more && remain) {957 padding = net_device->pkt_align - remain;958 rndis_msg->msg_len += padding;959 packet->total_data_buflen += padding;960 }961 962 for (i = 0; i < page_count; i++) {963 char *src = phys_to_virt(pb[i].pfn << HV_HYP_PAGE_SHIFT);964 u32 offset = pb[i].offset;965 u32 len = pb[i].len;966 967 memcpy(dest, (src + offset), len);968 dest += len;969 }970 971 if (padding)972 memset(dest, 0, padding);973}974 975void netvsc_dma_unmap(struct hv_device *hv_dev,976 struct hv_netvsc_packet *packet)977{978 int i;979 980 if (!hv_is_isolation_supported())981 return;982 983 if (!packet->dma_range)984 return;985 986 for (i = 0; i < packet->page_buf_cnt; i++)987 dma_unmap_single(&hv_dev->device, packet->dma_range[i].dma,988 packet->dma_range[i].mapping_size,989 DMA_TO_DEVICE);990 991 kfree(packet->dma_range);992}993 994/* netvsc_dma_map - Map swiotlb bounce buffer with data page of995 * packet sent by vmbus_sendpacket_pagebuffer() in the Isolation996 * VM.997 *998 * In isolation VM, netvsc send buffer has been marked visible to999 * host and so the data copied to send buffer doesn't need to use1000 * bounce buffer. The data pages handled by vmbus_sendpacket_pagebuffer()1001 * may not be copied to send buffer and so these pages need to be1002 * mapped with swiotlb bounce buffer. netvsc_dma_map() is to do1003 * that. The pfns in the struct hv_page_buffer need to be converted1004 * to bounce buffer's pfn. The loop here is necessary because the1005 * entries in the page buffer array are not necessarily full1006 * pages of data. Each entry in the array has a separate offset and1007 * len that may be non-zero, even for entries in the middle of the1008 * array. And the entries are not physically contiguous. So each1009 * entry must be individually mapped rather than as a contiguous unit.1010 * So not use dma_map_sg() here.1011 */1012static int netvsc_dma_map(struct hv_device *hv_dev,1013 struct hv_netvsc_packet *packet,1014 struct hv_page_buffer *pb)1015{1016 u32 page_count = packet->page_buf_cnt;1017 dma_addr_t dma;1018 int i;1019 1020 if (!hv_is_isolation_supported())1021 return 0;1022 1023 packet->dma_range = kcalloc(page_count,1024 sizeof(*packet->dma_range),1025 GFP_ATOMIC);1026 if (!packet->dma_range)1027 return -ENOMEM;1028 1029 for (i = 0; i < page_count; i++) {1030 char *src = phys_to_virt((pb[i].pfn << HV_HYP_PAGE_SHIFT)1031 + pb[i].offset);1032 u32 len = pb[i].len;1033 1034 dma = dma_map_single(&hv_dev->device, src, len,1035 DMA_TO_DEVICE);1036 if (dma_mapping_error(&hv_dev->device, dma)) {1037 kfree(packet->dma_range);1038 return -ENOMEM;1039 }1040 1041 /* pb[].offset and pb[].len are not changed during dma mapping1042 * and so not reassign.1043 */1044 packet->dma_range[i].dma = dma;1045 packet->dma_range[i].mapping_size = len;1046 pb[i].pfn = dma >> HV_HYP_PAGE_SHIFT;1047 }1048 1049 return 0;1050}1051 1052static inline int netvsc_send_pkt(1053 struct hv_device *device,1054 struct hv_netvsc_packet *packet,1055 struct netvsc_device *net_device,1056 struct hv_page_buffer *pb,1057 struct sk_buff *skb)1058{1059 struct nvsp_message nvmsg;1060 struct nvsp_1_message_send_rndis_packet *rpkt =1061 &nvmsg.msg.v1_msg.send_rndis_pkt;1062 struct netvsc_channel * const nvchan =1063 &net_device->chan_table[packet->q_idx];1064 struct vmbus_channel *out_channel = nvchan->channel;1065 struct net_device *ndev = hv_get_drvdata(device);1066 struct net_device_context *ndev_ctx = netdev_priv(ndev);1067 struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx);1068 u64 req_id;1069 int ret;1070 u32 ring_avail = hv_get_avail_to_write_percent(&out_channel->outbound);1071 1072 memset(&nvmsg, 0, sizeof(struct nvsp_message));1073 nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;1074 if (skb)1075 rpkt->channel_type = 0; /* 0 is RMC_DATA */1076 else1077 rpkt->channel_type = 1; /* 1 is RMC_CONTROL */1078 1079 rpkt->send_buf_section_index = packet->send_buf_index;1080 if (packet->send_buf_index == NETVSC_INVALID_INDEX)1081 rpkt->send_buf_section_size = 0;1082 else1083 rpkt->send_buf_section_size = packet->total_data_buflen;1084 1085 req_id = (ulong)skb;1086 1087 if (out_channel->rescind)1088 return -ENODEV;1089 1090 trace_nvsp_send_pkt(ndev, out_channel, rpkt);1091 1092 packet->dma_range = NULL;1093 if (packet->page_buf_cnt) {1094 if (packet->cp_partial)1095 pb += packet->rmsg_pgcnt;1096 1097 ret = netvsc_dma_map(ndev_ctx->device_ctx, packet, pb);1098 if (ret) {1099 ret = -EAGAIN;1100 goto exit;1101 }1102 1103 ret = vmbus_sendpacket_pagebuffer(out_channel,1104 pb, packet->page_buf_cnt,1105 &nvmsg, sizeof(nvmsg),1106 req_id);1107 1108 if (ret)1109 netvsc_dma_unmap(ndev_ctx->device_ctx, packet);1110 } else {1111 ret = vmbus_sendpacket(out_channel,1112 &nvmsg, sizeof(nvmsg),1113 req_id, VM_PKT_DATA_INBAND,1114 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);1115 }1116 1117exit:1118 if (ret == 0) {1119 atomic_inc_return(&nvchan->queue_sends);1120 1121 if (ring_avail < RING_AVAIL_PERCENT_LOWATER) {1122 netif_tx_stop_queue(txq);1123 ndev_ctx->eth_stats.stop_queue++;1124 }1125 } else if (ret == -EAGAIN) {1126 netif_tx_stop_queue(txq);1127 ndev_ctx->eth_stats.stop_queue++;1128 } else {1129 netdev_err(ndev,1130 "Unable to send packet pages %u len %u, ret %d\n",1131 packet->page_buf_cnt, packet->total_data_buflen,1132 ret);1133 }1134 1135 if (netif_tx_queue_stopped(txq) &&1136 atomic_read(&nvchan->queue_sends) < 1 &&1137 !net_device->tx_disable) {1138 netif_tx_wake_queue(txq);1139 ndev_ctx->eth_stats.wake_queue++;1140 if (ret == -EAGAIN)1141 ret = -ENOSPC;1142 }1143 1144 return ret;1145}1146 1147/* Move packet out of multi send data (msd), and clear msd */1148static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send,1149 struct sk_buff **msd_skb,1150 struct multi_send_data *msdp)1151{1152 *msd_skb = msdp->skb;1153 *msd_send = msdp->pkt;1154 msdp->skb = NULL;1155 msdp->pkt = NULL;1156 msdp->count = 0;1157}1158 1159/* RCU already held by caller */1160/* Batching/bouncing logic is designed to attempt to optimize1161 * performance.1162 *1163 * For small, non-LSO packets we copy the packet to a send buffer1164 * which is pre-registered with the Hyper-V side. This enables the1165 * hypervisor to avoid remapping the aperture to access the packet1166 * descriptor and data.1167 *1168 * If we already started using a buffer and the netdev is transmitting1169 * a burst of packets, keep on copying into the buffer until it is1170 * full or we are done collecting a burst. If there is an existing1171 * buffer with space for the RNDIS descriptor but not the packet, copy1172 * the RNDIS descriptor to the buffer, keeping the packet in place.1173 *1174 * If we do batching and send more than one packet using a single1175 * NetVSC message, free the SKBs of the packets copied, except for the1176 * last packet. This is done to streamline the handling of the case1177 * where the last packet only had the RNDIS descriptor copied to the1178 * send buffer, with the data pointers included in the NetVSC message.1179 */1180int netvsc_send(struct net_device *ndev,1181 struct hv_netvsc_packet *packet,1182 struct rndis_message *rndis_msg,1183 struct hv_page_buffer *pb,1184 struct sk_buff *skb,1185 bool xdp_tx)1186{1187 struct net_device_context *ndev_ctx = netdev_priv(ndev);1188 struct netvsc_device *net_device1189 = rcu_dereference_bh(ndev_ctx->nvdev);1190 struct hv_device *device = ndev_ctx->device_ctx;1191 int ret = 0;1192 struct netvsc_channel *nvchan;1193 u32 pktlen = packet->total_data_buflen, msd_len = 0;1194 unsigned int section_index = NETVSC_INVALID_INDEX;1195 struct multi_send_data *msdp;1196 struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL;1197 struct sk_buff *msd_skb = NULL;1198 bool try_batch, xmit_more;1199 1200 /* If device is rescinded, return error and packet will get dropped. */1201 if (unlikely(!net_device || net_device->destroy))1202 return -ENODEV;1203 1204 nvchan = &net_device->chan_table[packet->q_idx];1205 packet->send_buf_index = NETVSC_INVALID_INDEX;1206 packet->cp_partial = false;1207 1208 /* Send a control message or XDP packet directly without accessing1209 * msd (Multi-Send Data) field which may be changed during data packet1210 * processing.1211 */1212 if (!skb || xdp_tx)1213 return netvsc_send_pkt(device, packet, net_device, pb, skb);1214 1215 /* batch packets in send buffer if possible */1216 msdp = &nvchan->msd;1217 if (msdp->pkt)1218 msd_len = msdp->pkt->total_data_buflen;1219 1220 try_batch = msd_len > 0 && msdp->count < net_device->max_pkt;1221 if (try_batch && msd_len + pktlen + net_device->pkt_align <1222 net_device->send_section_size) {1223 section_index = msdp->pkt->send_buf_index;1224 1225 } else if (try_batch && msd_len + packet->rmsg_size <1226 net_device->send_section_size) {1227 section_index = msdp->pkt->send_buf_index;1228 packet->cp_partial = true;1229 1230 } else if (pktlen + net_device->pkt_align <1231 net_device->send_section_size) {1232 section_index = netvsc_get_next_send_section(net_device);1233 if (unlikely(section_index == NETVSC_INVALID_INDEX)) {1234 ++ndev_ctx->eth_stats.tx_send_full;1235 } else {1236 move_pkt_msd(&msd_send, &msd_skb, msdp);1237 msd_len = 0;1238 }1239 }1240 1241 /* Keep aggregating only if stack says more data is coming1242 * and not doing mixed modes send and not flow blocked1243 */1244 xmit_more = netdev_xmit_more() &&1245 !packet->cp_partial &&1246 !netif_xmit_stopped(netdev_get_tx_queue(ndev, packet->q_idx));1247 1248 if (section_index != NETVSC_INVALID_INDEX) {1249 netvsc_copy_to_send_buf(net_device,1250 section_index, msd_len,1251 packet, rndis_msg, pb, xmit_more);1252 1253 packet->send_buf_index = section_index;1254 1255 if (packet->cp_partial) {1256 packet->page_buf_cnt -= packet->rmsg_pgcnt;1257 packet->total_data_buflen = msd_len + packet->rmsg_size;1258 } else {1259 packet->page_buf_cnt = 0;1260 packet->total_data_buflen += msd_len;1261 }1262 1263 if (msdp->pkt) {1264 packet->total_packets += msdp->pkt->total_packets;1265 packet->total_bytes += msdp->pkt->total_bytes;1266 }1267 1268 if (msdp->skb)1269 dev_consume_skb_any(msdp->skb);1270 1271 if (xmit_more) {1272 msdp->skb = skb;1273 msdp->pkt = packet;1274 msdp->count++;1275 } else {1276 cur_send = packet;1277 msdp->skb = NULL;1278 msdp->pkt = NULL;1279 msdp->count = 0;1280 }1281 } else {1282 move_pkt_msd(&msd_send, &msd_skb, msdp);1283 cur_send = packet;1284 }1285 1286 if (msd_send) {1287 int m_ret = netvsc_send_pkt(device, msd_send, net_device,1288 NULL, msd_skb);1289 1290 if (m_ret != 0) {1291 netvsc_free_send_slot(net_device,1292 msd_send->send_buf_index);1293 dev_kfree_skb_any(msd_skb);1294 }1295 }1296 1297 if (cur_send)1298 ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb);1299 1300 if (ret != 0 && section_index != NETVSC_INVALID_INDEX)1301 netvsc_free_send_slot(net_device, section_index);1302 1303 return ret;1304}1305 1306/* Send pending recv completions */1307static int send_recv_completions(struct net_device *ndev,1308 struct netvsc_device *nvdev,1309 struct netvsc_channel *nvchan)1310{1311 struct multi_recv_comp *mrc = &nvchan->mrc;1312 struct recv_comp_msg {1313 struct nvsp_message_header hdr;1314 u32 status;1315 } __packed;1316 struct recv_comp_msg msg = {1317 .hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE,1318 };1319 int ret;1320 1321 while (mrc->first != mrc->next) {1322 const struct recv_comp_data *rcd1323 = mrc->slots + mrc->first;1324 1325 msg.status = rcd->status;1326 ret = vmbus_sendpacket(nvchan->channel, &msg, sizeof(msg),1327 rcd->tid, VM_PKT_COMP, 0);1328 if (unlikely(ret)) {1329 struct net_device_context *ndev_ctx = netdev_priv(ndev);1330 1331 ++ndev_ctx->eth_stats.rx_comp_busy;1332 return ret;1333 }1334 1335 if (++mrc->first == nvdev->recv_completion_cnt)1336 mrc->first = 0;1337 }1338 1339 /* receive completion ring has been emptied */1340 if (unlikely(nvdev->destroy))1341 wake_up(&nvdev->wait_drain);1342 1343 return 0;1344}1345 1346/* Count how many receive completions are outstanding */1347static void recv_comp_slot_avail(const struct netvsc_device *nvdev,1348 const struct multi_recv_comp *mrc,1349 u32 *filled, u32 *avail)1350{1351 u32 count = nvdev->recv_completion_cnt;1352 1353 if (mrc->next >= mrc->first)1354 *filled = mrc->next - mrc->first;1355 else1356 *filled = (count - mrc->first) + mrc->next;1357 1358 *avail = count - *filled - 1;1359}1360 1361/* Add receive complete to ring to send to host. */1362static void enq_receive_complete(struct net_device *ndev,1363 struct netvsc_device *nvdev, u16 q_idx,1364 u64 tid, u32 status)1365{1366 struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx];1367 struct multi_recv_comp *mrc = &nvchan->mrc;1368 struct recv_comp_data *rcd;1369 u32 filled, avail;1370 1371 recv_comp_slot_avail(nvdev, mrc, &filled, &avail);1372 1373 if (unlikely(filled > NAPI_POLL_WEIGHT)) {1374 send_recv_completions(ndev, nvdev, nvchan);1375 recv_comp_slot_avail(nvdev, mrc, &filled, &avail);1376 }1377 1378 if (unlikely(!avail)) {1379 netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n",1380 q_idx, tid);1381 return;1382 }1383 1384 rcd = mrc->slots + mrc->next;1385 rcd->tid = tid;1386 rcd->status = status;1387 1388 if (++mrc->next == nvdev->recv_completion_cnt)1389 mrc->next = 0;1390}1391 1392static int netvsc_receive(struct net_device *ndev,1393 struct netvsc_device *net_device,1394 struct netvsc_channel *nvchan,1395 const struct vmpacket_descriptor *desc)1396{1397 struct net_device_context *net_device_ctx = netdev_priv(ndev);1398 struct vmbus_channel *channel = nvchan->channel;1399 const struct vmtransfer_page_packet_header *vmxferpage_packet1400 = container_of(desc, const struct vmtransfer_page_packet_header, d);1401 const struct nvsp_message *nvsp = hv_pkt_data(desc);1402 u32 msglen = hv_pkt_datalen(desc);1403 u16 q_idx = channel->offermsg.offer.sub_channel_index;1404 char *recv_buf = net_device->recv_buf;1405 u32 status = NVSP_STAT_SUCCESS;1406 int i;1407 int count = 0;1408 1409 /* Ensure packet is big enough to read header fields */1410 if (msglen < sizeof(struct nvsp_message_header)) {1411 netif_err(net_device_ctx, rx_err, ndev,1412 "invalid nvsp header, length too small: %u\n",1413 msglen);1414 return 0;1415 }1416 1417 /* Make sure this is a valid nvsp packet */1418 if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) {1419 netif_err(net_device_ctx, rx_err, ndev,1420 "Unknown nvsp packet type received %u\n",1421 nvsp->hdr.msg_type);1422 return 0;1423 }1424 1425 /* Validate xfer page pkt header */1426 if ((desc->offset8 << 3) < sizeof(struct vmtransfer_page_packet_header)) {1427 netif_err(net_device_ctx, rx_err, ndev,1428 "Invalid xfer page pkt, offset too small: %u\n",1429 desc->offset8 << 3);1430 return 0;1431 }1432 1433 if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) {1434 netif_err(net_device_ctx, rx_err, ndev,1435 "Invalid xfer page set id - expecting %x got %x\n",1436 NETVSC_RECEIVE_BUFFER_ID,1437 vmxferpage_packet->xfer_pageset_id);1438 return 0;1439 }1440 1441 count = vmxferpage_packet->range_cnt;1442 1443 /* Check count for a valid value */1444 if (NETVSC_XFER_HEADER_SIZE(count) > desc->offset8 << 3) {1445 netif_err(net_device_ctx, rx_err, ndev,1446 "Range count is not valid: %d\n",1447 count);1448 return 0;1449 }1450 1451 /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */1452 for (i = 0; i < count; i++) {1453 u32 offset = vmxferpage_packet->ranges[i].byte_offset;1454 u32 buflen = vmxferpage_packet->ranges[i].byte_count;1455 void *data;1456 int ret;1457 1458 if (unlikely(offset > net_device->recv_buf_size ||1459 buflen > net_device->recv_buf_size - offset)) {1460 nvchan->rsc.cnt = 0;1461 status = NVSP_STAT_FAIL;1462 netif_err(net_device_ctx, rx_err, ndev,1463 "Packet offset:%u + len:%u too big\n",1464 offset, buflen);1465 1466 continue;1467 }1468 1469 /* We're going to copy (sections of) the packet into nvchan->recv_buf;1470 * make sure that nvchan->recv_buf is large enough to hold the packet.1471 */1472 if (unlikely(buflen > net_device->recv_section_size)) {1473 nvchan->rsc.cnt = 0;1474 status = NVSP_STAT_FAIL;1475 netif_err(net_device_ctx, rx_err, ndev,1476 "Packet too big: buflen=%u recv_section_size=%u\n",1477 buflen, net_device->recv_section_size);1478 1479 continue;1480 }1481 1482 data = recv_buf + offset;1483 1484 nvchan->rsc.is_last = (i == count - 1);1485 1486 trace_rndis_recv(ndev, q_idx, data);1487 1488 /* Pass it to the upper layer */1489 ret = rndis_filter_receive(ndev, net_device,1490 nvchan, data, buflen);1491 1492 if (unlikely(ret != NVSP_STAT_SUCCESS)) {1493 /* Drop incomplete packet */1494 nvchan->rsc.cnt = 0;1495 status = NVSP_STAT_FAIL;1496 }1497 }1498 1499 enq_receive_complete(ndev, net_device, q_idx,1500 vmxferpage_packet->d.trans_id, status);1501 1502 return count;1503}1504 1505static void netvsc_send_table(struct net_device *ndev,1506 struct netvsc_device *nvscdev,1507 const struct nvsp_message *nvmsg,1508 u32 msglen)1509{1510 struct net_device_context *net_device_ctx = netdev_priv(ndev);1511 u32 count, offset, *tab;1512 int i;1513 1514 /* Ensure packet is big enough to read send_table fields */1515 if (msglen < sizeof(struct nvsp_message_header) +1516 sizeof(struct nvsp_5_send_indirect_table)) {1517 netdev_err(ndev, "nvsp_v5_msg length too small: %u\n", msglen);1518 return;1519 }1520 1521 count = nvmsg->msg.v5_msg.send_table.count;1522 offset = nvmsg->msg.v5_msg.send_table.offset;1523 1524 if (count != VRSS_SEND_TAB_SIZE) {1525 netdev_err(ndev, "Received wrong send-table size:%u\n", count);1526 return;1527 }1528 1529 /* If negotiated version <= NVSP_PROTOCOL_VERSION_6, the offset may be1530 * wrong due to a host bug. So fix the offset here.1531 */1532 if (nvscdev->nvsp_version <= NVSP_PROTOCOL_VERSION_6 &&1533 msglen >= sizeof(struct nvsp_message_header) +1534 sizeof(union nvsp_6_message_uber) + count * sizeof(u32))1535 offset = sizeof(struct nvsp_message_header) +1536 sizeof(union nvsp_6_message_uber);1537 1538 /* Boundary check for all versions */1539 if (msglen < count * sizeof(u32) || offset > msglen - count * sizeof(u32)) {1540 netdev_err(ndev, "Received send-table offset too big:%u\n",1541 offset);1542 return;1543 }1544 1545 tab = (void *)nvmsg + offset;1546 1547 for (i = 0; i < count; i++)1548 net_device_ctx->tx_table[i] = tab[i];1549}1550 1551static void netvsc_send_vf(struct net_device *ndev,1552 const struct nvsp_message *nvmsg,1553 u32 msglen)1554{1555 struct net_device_context *net_device_ctx = netdev_priv(ndev);1556 1557 /* Ensure packet is big enough to read its fields */1558 if (msglen < sizeof(struct nvsp_message_header) +1559 sizeof(struct nvsp_4_send_vf_association)) {1560 netdev_err(ndev, "nvsp_v4_msg length too small: %u\n", msglen);1561 return;1562 }1563 1564 net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated;1565 net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial;1566 1567 if (net_device_ctx->vf_alloc)1568 complete(&net_device_ctx->vf_add);1569 1570 netdev_info(ndev, "VF slot %u %s\n",1571 net_device_ctx->vf_serial,1572 net_device_ctx->vf_alloc ? "added" : "removed");1573}1574 1575static void netvsc_receive_inband(struct net_device *ndev,1576 struct netvsc_device *nvscdev,1577 const struct vmpacket_descriptor *desc)1578{1579 const struct nvsp_message *nvmsg = hv_pkt_data(desc);1580 u32 msglen = hv_pkt_datalen(desc);1581 1582 /* Ensure packet is big enough to read header fields */1583 if (msglen < sizeof(struct nvsp_message_header)) {1584 netdev_err(ndev, "inband nvsp_message length too small: %u\n", msglen);1585 return;1586 }1587 1588 switch (nvmsg->hdr.msg_type) {1589 case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE:1590 netvsc_send_table(ndev, nvscdev, nvmsg, msglen);1591 break;1592 1593 case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION:1594 if (hv_is_isolation_supported())1595 netdev_err(ndev, "Ignore VF_ASSOCIATION msg from the host supporting isolation\n");1596 else1597 netvsc_send_vf(ndev, nvmsg, msglen);1598 break;1599 }1600}1601 1602static int netvsc_process_raw_pkt(struct hv_device *device,1603 struct netvsc_channel *nvchan,1604 struct netvsc_device *net_device,1605 struct net_device *ndev,1606 const struct vmpacket_descriptor *desc,1607 int budget)1608{1609 struct vmbus_channel *channel = nvchan->channel;1610 const struct nvsp_message *nvmsg = hv_pkt_data(desc);1611 1612 trace_nvsp_recv(ndev, channel, nvmsg);1613 1614 switch (desc->type) {1615 case VM_PKT_COMP:1616 netvsc_send_completion(ndev, net_device, channel, desc, budget);1617 break;1618 1619 case VM_PKT_DATA_USING_XFER_PAGES:1620 return netvsc_receive(ndev, net_device, nvchan, desc);1621 1622 case VM_PKT_DATA_INBAND:1623 netvsc_receive_inband(ndev, net_device, desc);1624 break;1625 1626 default:1627 netdev_err(ndev, "unhandled packet type %d, tid %llx\n",1628 desc->type, desc->trans_id);1629 break;1630 }1631 1632 return 0;1633}1634 1635static struct hv_device *netvsc_channel_to_device(struct vmbus_channel *channel)1636{1637 struct vmbus_channel *primary = channel->primary_channel;1638 1639 return primary ? primary->device_obj : channel->device_obj;1640}1641 1642/* Network processing softirq1643 * Process data in incoming ring buffer from host1644 * Stops when ring is empty or budget is met or exceeded.1645 */1646int netvsc_poll(struct napi_struct *napi, int budget)1647{1648 struct netvsc_channel *nvchan1649 = container_of(napi, struct netvsc_channel, napi);1650 struct netvsc_device *net_device = nvchan->net_device;1651 struct vmbus_channel *channel = nvchan->channel;1652 struct hv_device *device = netvsc_channel_to_device(channel);1653 struct net_device *ndev = hv_get_drvdata(device);1654 int work_done = 0;1655 int ret;1656 1657 /* If starting a new interval */1658 if (!nvchan->desc)1659 nvchan->desc = hv_pkt_iter_first(channel);1660 1661 nvchan->xdp_flush = false;1662 1663 while (nvchan->desc && work_done < budget) {1664 work_done += netvsc_process_raw_pkt(device, nvchan, net_device,1665 ndev, nvchan->desc, budget);1666 nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc);1667 }1668 1669 if (nvchan->xdp_flush)1670 xdp_do_flush();1671 1672 /* Send any pending receive completions */1673 ret = send_recv_completions(ndev, net_device, nvchan);1674 1675 /* If it did not exhaust NAPI budget this time1676 * and not doing busy poll1677 * then re-enable host interrupts1678 * and reschedule if ring is not empty1679 * or sending receive completion failed.1680 */1681 if (work_done < budget &&1682 napi_complete_done(napi, work_done) &&1683 (ret || hv_end_read(&channel->inbound)) &&1684 napi_schedule_prep(napi)) {1685 hv_begin_read(&channel->inbound);1686 __napi_schedule(napi);1687 }1688 1689 /* Driver may overshoot since multiple packets per descriptor */1690 return min(work_done, budget);1691}1692 1693/* Call back when data is available in host ring buffer.1694 * Processing is deferred until network softirq (NAPI)1695 */1696void netvsc_channel_cb(void *context)1697{1698 struct netvsc_channel *nvchan = context;1699 struct vmbus_channel *channel = nvchan->channel;1700 struct hv_ring_buffer_info *rbi = &channel->inbound;1701 1702 /* preload first vmpacket descriptor */1703 prefetch(hv_get_ring_buffer(rbi) + rbi->priv_read_index);1704 1705 if (napi_schedule_prep(&nvchan->napi)) {1706 /* disable interrupts from host */1707 hv_begin_read(rbi);1708 1709 __napi_schedule_irqoff(&nvchan->napi);1710 }1711}1712 1713/*1714 * netvsc_device_add - Callback when the device belonging to this1715 * driver is added1716 */1717struct netvsc_device *netvsc_device_add(struct hv_device *device,1718 const struct netvsc_device_info *device_info)1719{1720 int i, ret = 0;1721 struct netvsc_device *net_device;1722 struct net_device *ndev = hv_get_drvdata(device);1723 struct net_device_context *net_device_ctx = netdev_priv(ndev);1724 1725 net_device = alloc_net_device();1726 if (!net_device)1727 return ERR_PTR(-ENOMEM);1728 1729 for (i = 0; i < VRSS_SEND_TAB_SIZE; i++)1730 net_device_ctx->tx_table[i] = 0;1731 1732 /* Because the device uses NAPI, all the interrupt batching and1733 * control is done via Net softirq, not the channel handling1734 */1735 set_channel_read_mode(device->channel, HV_CALL_ISR);1736 1737 /* If we're reopening the device we may have multiple queues, fill the1738 * chn_table with the default channel to use it before subchannels are1739 * opened.1740 * Initialize the channel state before we open;1741 * we can be interrupted as soon as we open the channel.1742 */1743 1744 for (i = 0; i < VRSS_CHANNEL_MAX; i++) {1745 struct netvsc_channel *nvchan = &net_device->chan_table[i];1746 1747 nvchan->channel = device->channel;1748 nvchan->net_device = net_device;1749 u64_stats_init(&nvchan->tx_stats.syncp);1750 u64_stats_init(&nvchan->rx_stats.syncp);1751 1752 ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i, 0);1753 1754 if (ret) {1755 netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret);1756 goto cleanup2;1757 }1758 1759 ret = xdp_rxq_info_reg_mem_model(&nvchan->xdp_rxq,1760 MEM_TYPE_PAGE_SHARED, NULL);1761 1762 if (ret) {1763 netdev_err(ndev, "xdp reg_mem_model fail: %d\n", ret);1764 goto cleanup2;1765 }1766 }1767 1768 /* Enable NAPI handler before init callbacks */1769 netif_napi_add(ndev, &net_device->chan_table[0].napi, netvsc_poll);1770 1771 /* Open the channel */1772 device->channel->next_request_id_callback = vmbus_next_request_id;1773 device->channel->request_addr_callback = vmbus_request_addr;1774 device->channel->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes);1775 device->channel->max_pkt_size = NETVSC_MAX_PKT_SIZE;1776 1777 ret = vmbus_open(device->channel, netvsc_ring_bytes,1778 netvsc_ring_bytes, NULL, 0,1779 netvsc_channel_cb, net_device->chan_table);1780 1781 if (ret != 0) {1782 netdev_err(ndev, "unable to open channel: %d\n", ret);1783 goto cleanup;1784 }1785 1786 /* Channel is opened */1787 netdev_dbg(ndev, "hv_netvsc channel opened successfully\n");1788 1789 napi_enable(&net_device->chan_table[0].napi);1790 1791 /* Connect with the NetVsp */1792 ret = netvsc_connect_vsp(device, net_device, device_info);1793 if (ret != 0) {1794 netdev_err(ndev,1795 "unable to connect to NetVSP - %d\n", ret);1796 goto close;1797 }1798 1799 /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is1800 * populated.1801 */1802 rcu_assign_pointer(net_device_ctx->nvdev, net_device);1803 1804 return net_device;1805 1806close:1807 RCU_INIT_POINTER(net_device_ctx->nvdev, NULL);1808 napi_disable(&net_device->chan_table[0].napi);1809 1810 /* Now, we can close the channel safely */1811 vmbus_close(device->channel);1812 1813cleanup:1814 netif_napi_del(&net_device->chan_table[0].napi);1815 1816cleanup2:1817 free_netvsc_device(&net_device->rcu);1818 1819 return ERR_PTR(ret);1820}1821