1140 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2016 VMware, Inc., Palo Alto, CA., USA4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the7 * "Software"), to deal in the Software without restriction, including8 * without limitation the rights to use, copy, modify, merge, publish,9 * distribute, sub license, and/or sell copies of the Software, and to10 * permit persons to whom the Software is furnished to do so, subject to11 * the following conditions:12 *13 * The above copyright notice and this permission notice (including the14 * next paragraph) shall be included in all copies or substantial portions15 * of the Software.16 *17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL20 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE23 * USE OR OTHER DEALINGS IN THE SOFTWARE.24 *25 */26 27#include <linux/objtool.h>28#include <linux/kernel.h>29#include <linux/module.h>30#include <linux/slab.h>31#include <linux/cc_platform.h>32 33#include <asm/hypervisor.h>34#include <drm/drm_ioctl.h>35 36#include "vmwgfx_drv.h"37#include "vmwgfx_msg_x86.h"38#include "vmwgfx_msg_arm64.h"39#include "vmwgfx_mksstat.h"40 41#define MESSAGE_STATUS_SUCCESS 0x000142#define MESSAGE_STATUS_DORECV 0x000243#define MESSAGE_STATUS_CPT 0x001044#define MESSAGE_STATUS_HB 0x008045 46#define RPCI_PROTOCOL_NUM 0x4943505247#define GUESTMSG_FLAG_COOKIE 0x8000000048 49#define RETRIES 350 51#define VMW_PORT_CMD_MSG 3052#define VMW_PORT_CMD_HB_MSG 053#define VMW_PORT_CMD_OPEN_CHANNEL (MSG_TYPE_OPEN << 16 | VMW_PORT_CMD_MSG)54#define VMW_PORT_CMD_CLOSE_CHANNEL (MSG_TYPE_CLOSE << 16 | VMW_PORT_CMD_MSG)55#define VMW_PORT_CMD_SENDSIZE (MSG_TYPE_SENDSIZE << 16 | VMW_PORT_CMD_MSG)56#define VMW_PORT_CMD_RECVSIZE (MSG_TYPE_RECVSIZE << 16 | VMW_PORT_CMD_MSG)57#define VMW_PORT_CMD_RECVSTATUS (MSG_TYPE_RECVSTATUS << 16 | VMW_PORT_CMD_MSG)58 59#define VMW_PORT_CMD_MKS_GUEST_STATS 8560#define VMW_PORT_CMD_MKSGS_RESET (0 << 16 | VMW_PORT_CMD_MKS_GUEST_STATS)61#define VMW_PORT_CMD_MKSGS_ADD_PPN (1 << 16 | VMW_PORT_CMD_MKS_GUEST_STATS)62#define VMW_PORT_CMD_MKSGS_REMOVE_PPN (2 << 16 | VMW_PORT_CMD_MKS_GUEST_STATS)63 64#define HIGH_WORD(X) ((X & 0xFFFF0000) >> 16)65 66#define MAX_USER_MSG_LENGTH PAGE_SIZE67 68static u32 vmw_msg_enabled = 1;69 70enum rpc_msg_type {71 MSG_TYPE_OPEN,72 MSG_TYPE_SENDSIZE,73 MSG_TYPE_SENDPAYLOAD,74 MSG_TYPE_RECVSIZE,75 MSG_TYPE_RECVPAYLOAD,76 MSG_TYPE_RECVSTATUS,77 MSG_TYPE_CLOSE,78};79 80struct rpc_channel {81 u16 channel_id;82 u32 cookie_high;83 u32 cookie_low;84};85 86#if IS_ENABLED(CONFIG_DRM_VMWGFX_MKSSTATS)87/* Kernel mksGuestStats counter names and desciptions; same order as enum mksstat_kern_stats_t */88static const char* const mksstat_kern_name_desc[MKSSTAT_KERN_COUNT][2] =89{90 { "vmw_execbuf_ioctl", "vmw_execbuf_ioctl" },91 { "vmw_cotable_resize", "vmw_cotable_resize" },92};93#endif94 95/**96 * vmw_open_channel97 *98 * @channel: RPC channel99 * @protocol:100 *101 * Returns: 0 on success102 */103static int vmw_open_channel(struct rpc_channel *channel, unsigned int protocol)104{105 u32 ecx, edx, esi, edi;106 107 vmware_hypercall6(VMW_PORT_CMD_OPEN_CHANNEL,108 (protocol | GUESTMSG_FLAG_COOKIE), 0,109 &ecx, &edx, &esi, &edi);110 111 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0)112 return -EINVAL;113 114 channel->channel_id = HIGH_WORD(edx);115 channel->cookie_high = esi;116 channel->cookie_low = edi;117 118 return 0;119}120 121 122 123/**124 * vmw_close_channel125 *126 * @channel: RPC channel127 *128 * Returns: 0 on success129 */130static int vmw_close_channel(struct rpc_channel *channel)131{132 u32 ecx;133 134 vmware_hypercall5(VMW_PORT_CMD_CLOSE_CHANNEL,135 0, channel->channel_id << 16,136 channel->cookie_high,137 channel->cookie_low,138 &ecx);139 140 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0)141 return -EINVAL;142 143 return 0;144}145 146/**147 * vmw_port_hb_out - Send the message payload either through the148 * high-bandwidth port if available, or through the backdoor otherwise.149 * @channel: The rpc channel.150 * @msg: NULL-terminated message.151 * @hb: Whether the high-bandwidth port is available.152 *153 * Return: The port status.154 */155static unsigned long vmw_port_hb_out(struct rpc_channel *channel,156 const char *msg, bool hb)157{158 u32 ebx, ecx;159 unsigned long msg_len = strlen(msg);160 161 /* HB port can't access encrypted memory. */162 if (hb && !cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {163 vmware_hypercall_hb_out(164 (MESSAGE_STATUS_SUCCESS << 16) | VMW_PORT_CMD_HB_MSG,165 msg_len,166 channel->channel_id << 16,167 (uintptr_t) msg, channel->cookie_low,168 channel->cookie_high,169 &ebx);170 171 return ebx;172 }173 174 /* HB port not available. Send the message 4 bytes at a time. */175 ecx = MESSAGE_STATUS_SUCCESS << 16;176 while (msg_len && (HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS)) {177 unsigned int bytes = min_t(size_t, msg_len, 4);178 unsigned long word = 0;179 180 memcpy(&word, msg, bytes);181 msg_len -= bytes;182 msg += bytes;183 184 vmware_hypercall5(VMW_PORT_CMD_MSG |185 (MSG_TYPE_SENDPAYLOAD << 16),186 word, channel->channel_id << 16,187 channel->cookie_high,188 channel->cookie_low,189 &ecx);190 }191 192 return ecx;193}194 195/**196 * vmw_port_hb_in - Receive the message payload either through the197 * high-bandwidth port if available, or through the backdoor otherwise.198 * @channel: The rpc channel.199 * @reply: Pointer to buffer holding reply.200 * @reply_len: Length of the reply.201 * @hb: Whether the high-bandwidth port is available.202 *203 * Return: The port status.204 */205static unsigned long vmw_port_hb_in(struct rpc_channel *channel, char *reply,206 unsigned long reply_len, bool hb)207{208 u32 ebx, ecx, edx;209 210 /* HB port can't access encrypted memory */211 if (hb && !cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {212 vmware_hypercall_hb_in(213 (MESSAGE_STATUS_SUCCESS << 16) | VMW_PORT_CMD_HB_MSG,214 reply_len,215 channel->channel_id << 16,216 channel->cookie_high,217 (uintptr_t) reply, channel->cookie_low,218 &ebx);219 220 return ebx;221 }222 223 /* HB port not available. Retrieve the message 4 bytes at a time. */224 ecx = MESSAGE_STATUS_SUCCESS << 16;225 while (reply_len) {226 unsigned int bytes = min_t(unsigned long, reply_len, 4);227 228 vmware_hypercall7(VMW_PORT_CMD_MSG |229 (MSG_TYPE_RECVPAYLOAD << 16),230 MESSAGE_STATUS_SUCCESS,231 channel->channel_id << 16,232 channel->cookie_high,233 channel->cookie_low,234 &ebx, &ecx, &edx);235 236 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0)237 break;238 239 memcpy(reply, &ebx, bytes);240 reply_len -= bytes;241 reply += bytes;242 }243 244 return ecx;245}246 247 248/**249 * vmw_send_msg: Sends a message to the host250 *251 * @channel: RPC channel252 * @msg: NULL terminated string253 *254 * Returns: 0 on success255 */256static int vmw_send_msg(struct rpc_channel *channel, const char *msg)257{258 u32 ebx, ecx;259 size_t msg_len = strlen(msg);260 int retries = 0;261 262 while (retries < RETRIES) {263 retries++;264 265 vmware_hypercall5(VMW_PORT_CMD_SENDSIZE,266 msg_len, channel->channel_id << 16,267 channel->cookie_high,268 channel->cookie_low,269 &ecx);270 271 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) {272 /* Expected success. Give up. */273 return -EINVAL;274 }275 276 /* Send msg */277 ebx = vmw_port_hb_out(channel, msg,278 !!(HIGH_WORD(ecx) & MESSAGE_STATUS_HB));279 280 if ((HIGH_WORD(ebx) & MESSAGE_STATUS_SUCCESS) != 0) {281 return 0;282 } else if ((HIGH_WORD(ebx) & MESSAGE_STATUS_CPT) != 0) {283 /* A checkpoint occurred. Retry. */284 continue;285 } else {286 break;287 }288 }289 290 return -EINVAL;291}292STACK_FRAME_NON_STANDARD(vmw_send_msg);293 294 295/**296 * vmw_recv_msg: Receives a message from the host297 *298 * Note: It is the caller's responsibility to call kfree() on msg.299 *300 * @channel: channel opened by vmw_open_channel301 * @msg: [OUT] message received from the host302 * @msg_len: message length303 */304static int vmw_recv_msg(struct rpc_channel *channel, void **msg,305 size_t *msg_len)306{307 u32 ebx, ecx, edx;308 char *reply;309 size_t reply_len;310 int retries = 0;311 312 313 *msg_len = 0;314 *msg = NULL;315 316 while (retries < RETRIES) {317 retries++;318 319 vmware_hypercall7(VMW_PORT_CMD_RECVSIZE,320 0, channel->channel_id << 16,321 channel->cookie_high,322 channel->cookie_low,323 &ebx, &ecx, &edx);324 325 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) {326 DRM_ERROR("Failed to get reply size for host message.\n");327 return -EINVAL;328 }329 330 /* No reply available. This is okay. */331 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_DORECV) == 0)332 return 0;333 334 reply_len = ebx;335 reply = kzalloc(reply_len + 1, GFP_KERNEL);336 if (!reply) {337 DRM_ERROR("Cannot allocate memory for host message reply.\n");338 return -ENOMEM;339 }340 341 342 /* Receive buffer */343 ebx = vmw_port_hb_in(channel, reply, reply_len,344 !!(HIGH_WORD(ecx) & MESSAGE_STATUS_HB));345 if ((HIGH_WORD(ebx) & MESSAGE_STATUS_SUCCESS) == 0) {346 kfree(reply);347 reply = NULL;348 if ((HIGH_WORD(ebx) & MESSAGE_STATUS_CPT) != 0) {349 /* A checkpoint occurred. Retry. */350 continue;351 }352 353 return -EINVAL;354 }355 356 reply[reply_len] = '\0';357 358 vmware_hypercall5(VMW_PORT_CMD_RECVSTATUS,359 MESSAGE_STATUS_SUCCESS,360 channel->channel_id << 16,361 channel->cookie_high,362 channel->cookie_low,363 &ecx);364 365 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) {366 kfree(reply);367 reply = NULL;368 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_CPT) != 0) {369 /* A checkpoint occurred. Retry. */370 continue;371 }372 373 return -EINVAL;374 }375 376 break;377 }378 379 if (!reply)380 return -EINVAL;381 382 *msg_len = reply_len;383 *msg = reply;384 385 return 0;386}387STACK_FRAME_NON_STANDARD(vmw_recv_msg);388 389 390/**391 * vmw_host_get_guestinfo: Gets a GuestInfo parameter392 *393 * Gets the value of a GuestInfo.* parameter. The value returned will be in394 * a string, and it is up to the caller to post-process.395 *396 * @guest_info_param: Parameter to get, e.g. GuestInfo.svga.gl3397 * @buffer: if NULL, *reply_len will contain reply size.398 * @length: size of the reply_buf. Set to size of reply upon return399 *400 * Returns: 0 on success401 */402int vmw_host_get_guestinfo(const char *guest_info_param,403 char *buffer, size_t *length)404{405 struct rpc_channel channel;406 char *msg, *reply = NULL;407 size_t reply_len = 0;408 409 if (!vmw_msg_enabled)410 return -ENODEV;411 412 if (!guest_info_param || !length)413 return -EINVAL;414 415 msg = kasprintf(GFP_KERNEL, "info-get %s", guest_info_param);416 if (!msg) {417 DRM_ERROR("Cannot allocate memory to get guest info \"%s\".",418 guest_info_param);419 return -ENOMEM;420 }421 422 if (vmw_open_channel(&channel, RPCI_PROTOCOL_NUM))423 goto out_open;424 425 if (vmw_send_msg(&channel, msg) ||426 vmw_recv_msg(&channel, (void *) &reply, &reply_len))427 goto out_msg;428 429 vmw_close_channel(&channel);430 if (buffer && reply && reply_len > 0) {431 /* Remove reply code, which are the first 2 characters of432 * the reply433 */434 reply_len = max(reply_len - 2, (size_t) 0);435 reply_len = min(reply_len, *length);436 437 if (reply_len > 0)438 memcpy(buffer, reply + 2, reply_len);439 }440 441 *length = reply_len;442 443 kfree(reply);444 kfree(msg);445 446 return 0;447 448out_msg:449 vmw_close_channel(&channel);450 kfree(reply);451out_open:452 *length = 0;453 kfree(msg);454 DRM_ERROR("Failed to get guest info \"%s\".", guest_info_param);455 456 return -EINVAL;457}458 459 460/**461 * vmw_host_printf: Sends a log message to the host462 *463 * @fmt: Regular printf format string and arguments464 *465 * Returns: 0 on success466 */467__printf(1, 2)468int vmw_host_printf(const char *fmt, ...)469{470 va_list ap;471 struct rpc_channel channel;472 char *msg;473 char *log;474 int ret = 0;475 476 if (!vmw_msg_enabled)477 return -ENODEV;478 479 if (!fmt)480 return ret;481 482 va_start(ap, fmt);483 log = kvasprintf(GFP_KERNEL, fmt, ap);484 va_end(ap);485 if (!log) {486 DRM_ERROR("Cannot allocate memory for the log message.\n");487 return -ENOMEM;488 }489 490 msg = kasprintf(GFP_KERNEL, "log %s", log);491 if (!msg) {492 DRM_ERROR("Cannot allocate memory for host log message.\n");493 kfree(log);494 return -ENOMEM;495 }496 497 if (vmw_open_channel(&channel, RPCI_PROTOCOL_NUM))498 goto out_open;499 500 if (vmw_send_msg(&channel, msg))501 goto out_msg;502 503 vmw_close_channel(&channel);504 kfree(msg);505 kfree(log);506 507 return 0;508 509out_msg:510 vmw_close_channel(&channel);511out_open:512 kfree(msg);513 kfree(log);514 DRM_ERROR("Failed to send host log message.\n");515 516 return -EINVAL;517}518 519 520/**521 * vmw_msg_ioctl: Sends and receveives a message to/from host from/to user-space522 *523 * Sends a message from user-space to host.524 * Can also receive a result from host and return that to user-space.525 *526 * @dev: Identifies the drm device.527 * @data: Pointer to the ioctl argument.528 * @file_priv: Identifies the caller.529 * Return: Zero on success, negative error code on error.530 */531 532int vmw_msg_ioctl(struct drm_device *dev, void *data,533 struct drm_file *file_priv)534{535 struct drm_vmw_msg_arg *arg =536 (struct drm_vmw_msg_arg *)data;537 struct rpc_channel channel;538 char *msg;539 int length;540 541 msg = kmalloc(MAX_USER_MSG_LENGTH, GFP_KERNEL);542 if (!msg) {543 DRM_ERROR("Cannot allocate memory for log message.\n");544 return -ENOMEM;545 }546 547 length = strncpy_from_user(msg, (void __user *)((unsigned long)arg->send),548 MAX_USER_MSG_LENGTH);549 if (length < 0 || length >= MAX_USER_MSG_LENGTH) {550 DRM_ERROR("Userspace message access failure.\n");551 kfree(msg);552 return -EINVAL;553 }554 555 556 if (vmw_open_channel(&channel, RPCI_PROTOCOL_NUM)) {557 DRM_ERROR("Failed to open channel.\n");558 goto out_open;559 }560 561 if (vmw_send_msg(&channel, msg)) {562 DRM_ERROR("Failed to send message to host.\n");563 goto out_msg;564 }565 566 if (!arg->send_only) {567 char *reply = NULL;568 size_t reply_len = 0;569 570 if (vmw_recv_msg(&channel, (void *) &reply, &reply_len)) {571 DRM_ERROR("Failed to receive message from host.\n");572 goto out_msg;573 }574 if (reply && reply_len > 0) {575 if (copy_to_user((void __user *)((unsigned long)arg->receive),576 reply, reply_len)) {577 DRM_ERROR("Failed to copy message to userspace.\n");578 kfree(reply);579 goto out_msg;580 }581 arg->receive_len = (__u32)reply_len;582 }583 kfree(reply);584 }585 586 vmw_close_channel(&channel);587 kfree(msg);588 589 return 0;590 591out_msg:592 vmw_close_channel(&channel);593out_open:594 kfree(msg);595 596 return -EINVAL;597}598 599/**600 * reset_ppn_array: Resets a PPN64 array to INVALID_PPN64 content601 *602 * @arr: Array to reset.603 * @size: Array length.604 */605static inline void reset_ppn_array(PPN64 *arr, size_t size)606{607 size_t i;608 609 BUG_ON(!arr || size == 0);610 611 for (i = 0; i < size; ++i)612 arr[i] = INVALID_PPN64;613}614 615/**616 * hypervisor_ppn_reset_all: Removes all mksGuestStat instance descriptors from617 * the hypervisor. All related pages should be subsequently unpinned or freed.618 *619 */620static inline void hypervisor_ppn_reset_all(void)621{622 vmware_hypercall1(VMW_PORT_CMD_MKSGS_RESET, 0);623}624 625/**626 * hypervisor_ppn_add: Adds a single mksGuestStat instance descriptor to the627 * hypervisor. Any related userspace pages should be pinned in advance.628 *629 * @pfn: Physical page number of the instance descriptor630 */631static inline void hypervisor_ppn_add(PPN64 pfn)632{633 vmware_hypercall1(VMW_PORT_CMD_MKSGS_ADD_PPN, (unsigned long)pfn);634}635 636/**637 * hypervisor_ppn_remove: Removes a single mksGuestStat instance descriptor from638 * the hypervisor. All related pages should be subsequently unpinned or freed.639 *640 * @pfn: Physical page number of the instance descriptor641 */642static inline void hypervisor_ppn_remove(PPN64 pfn)643{644 vmware_hypercall1(VMW_PORT_CMD_MKSGS_REMOVE_PPN, (unsigned long)pfn);645}646 647#if IS_ENABLED(CONFIG_DRM_VMWGFX_MKSSTATS)648 649/* Order of the total number of pages used for kernel-internal mksGuestStat; at least 2 */650#define MKSSTAT_KERNEL_PAGES_ORDER 2651/* Header to the text description of mksGuestStat instance descriptor */652#define MKSSTAT_KERNEL_DESCRIPTION "vmwgfx"653 654/**655 * mksstat_init_record_time: Initializes an MKSGuestStatCounterTime-based record656 * for the respective mksGuestStat index.657 *658 * @stat_idx: Index of the MKSGuestStatCounterTime-based mksGuestStat record.659 * @pstat: Pointer to array of MKSGuestStatCounterTime.660 * @pinfo: Pointer to array of MKSGuestStatInfoEntry.661 * @pstrs: Pointer to current end of the name/description sequence.662 * Return: Pointer to the new end of the names/description sequence.663 */664 665static inline char *mksstat_init_record_time(mksstat_kern_stats_t stat_idx,666 MKSGuestStatCounterTime *pstat, MKSGuestStatInfoEntry *pinfo, char *pstrs)667{668 char *const pstrd = pstrs + strlen(mksstat_kern_name_desc[stat_idx][0]) + 1;669 strcpy(pstrs, mksstat_kern_name_desc[stat_idx][0]);670 strcpy(pstrd, mksstat_kern_name_desc[stat_idx][1]);671 672 pinfo[stat_idx].name.s = pstrs;673 pinfo[stat_idx].description.s = pstrd;674 pinfo[stat_idx].flags = MKS_GUEST_STAT_FLAG_TIME;675 pinfo[stat_idx].stat.counterTime = &pstat[stat_idx];676 677 return pstrd + strlen(mksstat_kern_name_desc[stat_idx][1]) + 1;678}679 680/**681 * mksstat_init_kern_id: Creates a single mksGuestStat instance descriptor and682 * kernel-internal counters. Adds PFN mapping to the hypervisor.683 *684 * Create a single mksGuestStat instance descriptor and corresponding structures685 * for all kernel-internal counters. The corresponding PFNs are mapped with the686 * hypervisor.687 *688 * @ppage: Output pointer to page containing the instance descriptor.689 * Return: Zero on success, negative error code on error.690 */691 692static int mksstat_init_kern_id(struct page **ppage)693{694 MKSGuestStatInstanceDescriptor *pdesc;695 MKSGuestStatCounterTime *pstat;696 MKSGuestStatInfoEntry *pinfo;697 char *pstrs, *pstrs_acc;698 699 /* Allocate pages for the kernel-internal instance descriptor */700 struct page *page = alloc_pages(GFP_KERNEL | __GFP_ZERO, MKSSTAT_KERNEL_PAGES_ORDER);701 702 if (!page)703 return -ENOMEM;704 705 pdesc = page_address(page);706 pstat = vmw_mksstat_get_kern_pstat(pdesc);707 pinfo = vmw_mksstat_get_kern_pinfo(pdesc);708 pstrs = vmw_mksstat_get_kern_pstrs(pdesc);709 710 /* Set up all kernel-internal counters and corresponding structures */711 pstrs_acc = pstrs;712 pstrs_acc = mksstat_init_record_time(MKSSTAT_KERN_EXECBUF, pstat, pinfo, pstrs_acc);713 pstrs_acc = mksstat_init_record_time(MKSSTAT_KERN_COTABLE_RESIZE, pstat, pinfo, pstrs_acc);714 715 /* Add new counters above, in their order of appearance in mksstat_kern_stats_t */716 717 BUG_ON(pstrs_acc - pstrs > PAGE_SIZE);718 719 /* Set up the kernel-internal instance descriptor */720 pdesc->reservedMBZ = 0;721 pdesc->statStartVA = (uintptr_t)pstat;722 pdesc->strsStartVA = (uintptr_t)pstrs;723 pdesc->statLength = sizeof(*pstat) * MKSSTAT_KERN_COUNT;724 pdesc->infoLength = sizeof(*pinfo) * MKSSTAT_KERN_COUNT;725 pdesc->strsLength = pstrs_acc - pstrs;726 snprintf(pdesc->description, ARRAY_SIZE(pdesc->description) - 1, "%s pid=%d",727 MKSSTAT_KERNEL_DESCRIPTION, current->pid);728 729 pdesc->statPPNs[0] = page_to_pfn(virt_to_page(pstat));730 reset_ppn_array(pdesc->statPPNs + 1, ARRAY_SIZE(pdesc->statPPNs) - 1);731 732 pdesc->infoPPNs[0] = page_to_pfn(virt_to_page(pinfo));733 reset_ppn_array(pdesc->infoPPNs + 1, ARRAY_SIZE(pdesc->infoPPNs) - 1);734 735 pdesc->strsPPNs[0] = page_to_pfn(virt_to_page(pstrs));736 reset_ppn_array(pdesc->strsPPNs + 1, ARRAY_SIZE(pdesc->strsPPNs) - 1);737 738 *ppage = page;739 740 hypervisor_ppn_add((PPN64)page_to_pfn(page));741 742 return 0;743}744 745/**746 * vmw_mksstat_get_kern_slot: Acquires a slot for a single kernel-internal747 * mksGuestStat instance descriptor.748 *749 * Find a slot for a single kernel-internal mksGuestStat instance descriptor.750 * In case no such was already present, allocate a new one and set up a kernel-751 * internal mksGuestStat instance descriptor for the former.752 *753 * @pid: Process for which a slot is sought.754 * @dev_priv: Identifies the drm private device.755 * Return: Non-negative slot on success, negative error code on error.756 */757 758int vmw_mksstat_get_kern_slot(pid_t pid, struct vmw_private *dev_priv)759{760 const size_t base = (u32)hash_32(pid, MKSSTAT_CAPACITY_LOG2);761 size_t i;762 763 for (i = 0; i < ARRAY_SIZE(dev_priv->mksstat_kern_pids); ++i) {764 const size_t slot = (i + base) % ARRAY_SIZE(dev_priv->mksstat_kern_pids);765 766 /* Check if an instance descriptor for this pid is already present */767 if (pid == (pid_t)atomic_read(&dev_priv->mksstat_kern_pids[slot]))768 return (int)slot;769 770 /* Set up a new instance descriptor for this pid */771 if (!atomic_cmpxchg(&dev_priv->mksstat_kern_pids[slot], 0, MKSSTAT_PID_RESERVED)) {772 const int ret = mksstat_init_kern_id(&dev_priv->mksstat_kern_pages[slot]);773 774 if (!ret) {775 /* Reset top-timer tracking for this slot */776 dev_priv->mksstat_kern_top_timer[slot] = MKSSTAT_KERN_COUNT;777 778 atomic_set(&dev_priv->mksstat_kern_pids[slot], pid);779 return (int)slot;780 }781 782 atomic_set(&dev_priv->mksstat_kern_pids[slot], 0);783 return ret;784 }785 }786 787 return -ENOSPC;788}789 790#endif791 792/**793 * vmw_mksstat_cleanup_descriptor: Frees a single userspace-originating794 * mksGuestStat instance-descriptor page and unpins all related user pages.795 *796 * Unpin all user pages realated to this instance descriptor and free797 * the instance-descriptor page itself.798 *799 * @page: Page of the instance descriptor.800 */801 802static void vmw_mksstat_cleanup_descriptor(struct page *page)803{804 MKSGuestStatInstanceDescriptor *pdesc = page_address(page);805 size_t i;806 807 for (i = 0; i < ARRAY_SIZE(pdesc->statPPNs) && pdesc->statPPNs[i] != INVALID_PPN64; ++i)808 unpin_user_page(pfn_to_page(pdesc->statPPNs[i]));809 810 for (i = 0; i < ARRAY_SIZE(pdesc->infoPPNs) && pdesc->infoPPNs[i] != INVALID_PPN64; ++i)811 unpin_user_page(pfn_to_page(pdesc->infoPPNs[i]));812 813 for (i = 0; i < ARRAY_SIZE(pdesc->strsPPNs) && pdesc->strsPPNs[i] != INVALID_PPN64; ++i)814 unpin_user_page(pfn_to_page(pdesc->strsPPNs[i]));815 816 __free_page(page);817}818 819/**820 * vmw_mksstat_remove_all: Resets all mksGuestStat instance descriptors821 * from the hypervisor.822 *823 * Discard all hypervisor PFN mappings, containing active mksGuestState instance824 * descriptors, unpin the related userspace pages and free the related kernel pages.825 *826 * @dev_priv: Identifies the drm private device.827 * Return: Zero on success, negative error code on error.828 */829 830int vmw_mksstat_remove_all(struct vmw_private *dev_priv)831{832 int ret = 0;833 size_t i;834 835 /* Discard all PFN mappings with the hypervisor */836 hypervisor_ppn_reset_all();837 838 /* Discard all userspace-originating instance descriptors and unpin all related pages */839 for (i = 0; i < ARRAY_SIZE(dev_priv->mksstat_user_pids); ++i) {840 const pid_t pid0 = (pid_t)atomic_read(&dev_priv->mksstat_user_pids[i]);841 842 if (!pid0)843 continue;844 845 if (pid0 != MKSSTAT_PID_RESERVED) {846 const pid_t pid1 = atomic_cmpxchg(&dev_priv->mksstat_user_pids[i], pid0, MKSSTAT_PID_RESERVED);847 848 if (!pid1)849 continue;850 851 if (pid1 == pid0) {852 struct page *const page = dev_priv->mksstat_user_pages[i];853 854 BUG_ON(!page);855 856 dev_priv->mksstat_user_pages[i] = NULL;857 atomic_set(&dev_priv->mksstat_user_pids[i], 0);858 859 vmw_mksstat_cleanup_descriptor(page);860 continue;861 }862 }863 864 ret = -EAGAIN;865 }866 867#if IS_ENABLED(CONFIG_DRM_VMWGFX_MKSSTATS)868 /* Discard all kernel-internal instance descriptors and free all related pages */869 for (i = 0; i < ARRAY_SIZE(dev_priv->mksstat_kern_pids); ++i) {870 const pid_t pid0 = (pid_t)atomic_read(&dev_priv->mksstat_kern_pids[i]);871 872 if (!pid0)873 continue;874 875 if (pid0 != MKSSTAT_PID_RESERVED) {876 const pid_t pid1 = atomic_cmpxchg(&dev_priv->mksstat_kern_pids[i], pid0, MKSSTAT_PID_RESERVED);877 878 if (!pid1)879 continue;880 881 if (pid1 == pid0) {882 struct page *const page = dev_priv->mksstat_kern_pages[i];883 884 BUG_ON(!page);885 886 dev_priv->mksstat_kern_pages[i] = NULL;887 atomic_set(&dev_priv->mksstat_kern_pids[i], 0);888 889 __free_pages(page, MKSSTAT_KERNEL_PAGES_ORDER);890 continue;891 }892 }893 894 ret = -EAGAIN;895 }896 897#endif898 return ret;899}900 901/**902 * vmw_mksstat_reset_ioctl: Resets all mksGuestStat instance descriptors903 * from the hypervisor.904 *905 * Discard all hypervisor PFN mappings, containing active mksGuestStat instance906 * descriptors, unpin the related userspace pages and free the related kernel pages.907 *908 * @dev: Identifies the drm device.909 * @data: Pointer to the ioctl argument.910 * @file_priv: Identifies the caller; unused.911 * Return: Zero on success, negative error code on error.912 */913 914int vmw_mksstat_reset_ioctl(struct drm_device *dev, void *data,915 struct drm_file *file_priv)916{917 struct vmw_private *const dev_priv = vmw_priv(dev);918 return vmw_mksstat_remove_all(dev_priv);919}920 921/**922 * vmw_mksstat_add_ioctl: Creates a single userspace-originating mksGuestStat923 * instance descriptor and registers that with the hypervisor.924 *925 * Create a hypervisor PFN mapping, containing a single mksGuestStat instance926 * descriptor and pin the corresponding userspace pages.927 *928 * @dev: Identifies the drm device.929 * @data: Pointer to the ioctl argument.930 * @file_priv: Identifies the caller; unused.931 * Return: Zero on success, negative error code on error.932 */933 934int vmw_mksstat_add_ioctl(struct drm_device *dev, void *data,935 struct drm_file *file_priv)936{937 struct drm_vmw_mksstat_add_arg *arg =938 (struct drm_vmw_mksstat_add_arg *) data;939 940 struct vmw_private *const dev_priv = vmw_priv(dev);941 942 const size_t num_pages_stat = PFN_UP(arg->stat_len);943 const size_t num_pages_info = PFN_UP(arg->info_len);944 const size_t num_pages_strs = PFN_UP(arg->strs_len);945 long desc_len;946 long nr_pinned_stat;947 long nr_pinned_info;948 long nr_pinned_strs;949 MKSGuestStatInstanceDescriptor *pdesc;950 struct page *page = NULL;951 struct page **pages_stat = NULL;952 struct page **pages_info = NULL;953 struct page **pages_strs = NULL;954 size_t i, slot;955 int ret_err = -ENOMEM;956 957 arg->id = -1;958 959 if (!arg->stat || !arg->info || !arg->strs)960 return -EINVAL;961 962 if (!arg->stat_len || !arg->info_len || !arg->strs_len)963 return -EINVAL;964 965 if (!arg->description)966 return -EINVAL;967 968 if (num_pages_stat > ARRAY_SIZE(pdesc->statPPNs) ||969 num_pages_info > ARRAY_SIZE(pdesc->infoPPNs) ||970 num_pages_strs > ARRAY_SIZE(pdesc->strsPPNs))971 return -EINVAL;972 973 /* Find an available slot in the mksGuestStats user array and reserve it */974 for (slot = 0; slot < ARRAY_SIZE(dev_priv->mksstat_user_pids); ++slot)975 if (!atomic_cmpxchg(&dev_priv->mksstat_user_pids[slot], 0, MKSSTAT_PID_RESERVED))976 break;977 978 if (slot == ARRAY_SIZE(dev_priv->mksstat_user_pids))979 return -ENOSPC;980 981 BUG_ON(dev_priv->mksstat_user_pages[slot]);982 983 /* Allocate statically-sized temp arrays for pages -- too big to keep in frame */984 pages_stat = (struct page **)kmalloc_array(985 ARRAY_SIZE(pdesc->statPPNs) +986 ARRAY_SIZE(pdesc->infoPPNs) +987 ARRAY_SIZE(pdesc->strsPPNs), sizeof(*pages_stat), GFP_KERNEL);988 989 if (!pages_stat)990 goto err_nomem;991 992 pages_info = pages_stat + ARRAY_SIZE(pdesc->statPPNs);993 pages_strs = pages_info + ARRAY_SIZE(pdesc->infoPPNs);994 995 /* Allocate a page for the instance descriptor */996 page = alloc_page(GFP_KERNEL | __GFP_ZERO);997 998 if (!page)999 goto err_nomem;1000 1001 /* Set up the instance descriptor */1002 pdesc = page_address(page);1003 1004 pdesc->reservedMBZ = 0;1005 pdesc->statStartVA = arg->stat;1006 pdesc->strsStartVA = arg->strs;1007 pdesc->statLength = arg->stat_len;1008 pdesc->infoLength = arg->info_len;1009 pdesc->strsLength = arg->strs_len;1010 desc_len = strncpy_from_user(pdesc->description, u64_to_user_ptr(arg->description),1011 ARRAY_SIZE(pdesc->description) - 1);1012 1013 if (desc_len < 0) {1014 ret_err = -EFAULT;1015 goto err_nomem;1016 }1017 1018 reset_ppn_array(pdesc->statPPNs, ARRAY_SIZE(pdesc->statPPNs));1019 reset_ppn_array(pdesc->infoPPNs, ARRAY_SIZE(pdesc->infoPPNs));1020 reset_ppn_array(pdesc->strsPPNs, ARRAY_SIZE(pdesc->strsPPNs));1021 1022 /* Pin mksGuestStat user pages and store those in the instance descriptor */1023 nr_pinned_stat = pin_user_pages_fast(arg->stat, num_pages_stat, FOLL_LONGTERM, pages_stat);1024 if (num_pages_stat != nr_pinned_stat)1025 goto err_pin_stat;1026 1027 for (i = 0; i < num_pages_stat; ++i)1028 pdesc->statPPNs[i] = page_to_pfn(pages_stat[i]);1029 1030 nr_pinned_info = pin_user_pages_fast(arg->info, num_pages_info, FOLL_LONGTERM, pages_info);1031 if (num_pages_info != nr_pinned_info)1032 goto err_pin_info;1033 1034 for (i = 0; i < num_pages_info; ++i)1035 pdesc->infoPPNs[i] = page_to_pfn(pages_info[i]);1036 1037 nr_pinned_strs = pin_user_pages_fast(arg->strs, num_pages_strs, FOLL_LONGTERM, pages_strs);1038 if (num_pages_strs != nr_pinned_strs)1039 goto err_pin_strs;1040 1041 for (i = 0; i < num_pages_strs; ++i)1042 pdesc->strsPPNs[i] = page_to_pfn(pages_strs[i]);1043 1044 /* Send the descriptor to the host via a hypervisor call. The mksGuestStat1045 pages will remain in use until the user requests a matching remove stats1046 or a stats reset occurs. */1047 hypervisor_ppn_add((PPN64)page_to_pfn(page));1048 1049 dev_priv->mksstat_user_pages[slot] = page;1050 atomic_set(&dev_priv->mksstat_user_pids[slot], task_pgrp_vnr(current));1051 1052 arg->id = slot;1053 1054 DRM_DEV_INFO(dev->dev, "pid=%d arg.description='%.*s' id=%zu\n", current->pid, (int)desc_len, pdesc->description, slot);1055 1056 kfree(pages_stat);1057 return 0;1058 1059err_pin_strs:1060 if (nr_pinned_strs > 0)1061 unpin_user_pages(pages_strs, nr_pinned_strs);1062 1063err_pin_info:1064 if (nr_pinned_info > 0)1065 unpin_user_pages(pages_info, nr_pinned_info);1066 1067err_pin_stat:1068 if (nr_pinned_stat > 0)1069 unpin_user_pages(pages_stat, nr_pinned_stat);1070 1071err_nomem:1072 atomic_set(&dev_priv->mksstat_user_pids[slot], 0);1073 if (page)1074 __free_page(page);1075 kfree(pages_stat);1076 1077 return ret_err;1078}1079 1080/**1081 * vmw_mksstat_remove_ioctl: Removes a single userspace-originating mksGuestStat1082 * instance descriptor from the hypervisor.1083 *1084 * Discard a hypervisor PFN mapping, containing a single mksGuestStat instance1085 * descriptor and unpin the corresponding userspace pages.1086 *1087 * @dev: Identifies the drm device.1088 * @data: Pointer to the ioctl argument.1089 * @file_priv: Identifies the caller; unused.1090 * Return: Zero on success, negative error code on error.1091 */1092 1093int vmw_mksstat_remove_ioctl(struct drm_device *dev, void *data,1094 struct drm_file *file_priv)1095{1096 struct drm_vmw_mksstat_remove_arg *arg =1097 (struct drm_vmw_mksstat_remove_arg *) data;1098 1099 struct vmw_private *const dev_priv = vmw_priv(dev);1100 1101 const size_t slot = arg->id;1102 pid_t pgid, pid;1103 1104 if (slot >= ARRAY_SIZE(dev_priv->mksstat_user_pids))1105 return -EINVAL;1106 1107 DRM_DEV_INFO(dev->dev, "pid=%d arg.id=%zu\n", current->pid, slot);1108 1109 pgid = task_pgrp_vnr(current);1110 pid = atomic_cmpxchg(&dev_priv->mksstat_user_pids[slot], pgid, MKSSTAT_PID_RESERVED);1111 1112 if (!pid)1113 return 0;1114 1115 if (pid == pgid) {1116 struct page *const page = dev_priv->mksstat_user_pages[slot];1117 1118 BUG_ON(!page);1119 1120 dev_priv->mksstat_user_pages[slot] = NULL;1121 atomic_set(&dev_priv->mksstat_user_pids[slot], 0);1122 1123 hypervisor_ppn_remove((PPN64)page_to_pfn(page));1124 1125 vmw_mksstat_cleanup_descriptor(page);1126 return 0;1127 }1128 1129 return -EAGAIN;1130}1131 1132/**1133 * vmw_disable_backdoor: Disables all backdoor communication1134 * with the hypervisor.1135 */1136void vmw_disable_backdoor(void)1137{1138 vmw_msg_enabled = 0;1139}1140