224 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * SN Platform GRU Driver4 *5 * Dump GRU State6 *7 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.8 */9 10#include <linux/kernel.h>11#include <linux/mm.h>12#include <linux/spinlock.h>13#include <linux/uaccess.h>14#include <linux/delay.h>15#include <linux/bitops.h>16#include <asm/uv/uv_hub.h>17 18#include <linux/nospec.h>19 20#include "gru.h"21#include "grutables.h"22#include "gruhandles.h"23#include "grulib.h"24 25#define CCH_LOCK_ATTEMPTS 1026 27static int gru_user_copy_handle(void __user **dp, void *s)28{29 if (copy_to_user(*dp, s, GRU_HANDLE_BYTES))30 return -1;31 *dp += GRU_HANDLE_BYTES;32 return 0;33}34 35static int gru_dump_context_data(void *grubase,36 struct gru_context_configuration_handle *cch,37 void __user *ubuf, int ctxnum, int dsrcnt,38 int flush_cbrs)39{40 void *cb, *cbe, *tfh, *gseg;41 int i, scr;42 43 gseg = grubase + ctxnum * GRU_GSEG_STRIDE;44 cb = gseg + GRU_CB_BASE;45 cbe = grubase + GRU_CBE_BASE;46 tfh = grubase + GRU_TFH_BASE;47 48 for_each_cbr_in_allocation_map(i, &cch->cbr_allocation_map, scr) {49 if (flush_cbrs)50 gru_flush_cache(cb);51 if (gru_user_copy_handle(&ubuf, cb))52 goto fail;53 if (gru_user_copy_handle(&ubuf, tfh + i * GRU_HANDLE_STRIDE))54 goto fail;55 if (gru_user_copy_handle(&ubuf, cbe + i * GRU_HANDLE_STRIDE))56 goto fail;57 cb += GRU_HANDLE_STRIDE;58 }59 if (dsrcnt)60 memcpy(ubuf, gseg + GRU_DS_BASE, dsrcnt * GRU_HANDLE_STRIDE);61 return 0;62 63fail:64 return -EFAULT;65}66 67static int gru_dump_tfm(struct gru_state *gru,68 void __user *ubuf, void __user *ubufend)69{70 struct gru_tlb_fault_map *tfm;71 int i;72 73 if (GRU_NUM_TFM * GRU_CACHE_LINE_BYTES > ubufend - ubuf)74 return -EFBIG;75 76 for (i = 0; i < GRU_NUM_TFM; i++) {77 tfm = get_tfm(gru->gs_gru_base_vaddr, i);78 if (gru_user_copy_handle(&ubuf, tfm))79 goto fail;80 }81 return GRU_NUM_TFM * GRU_CACHE_LINE_BYTES;82 83fail:84 return -EFAULT;85}86 87static int gru_dump_tgh(struct gru_state *gru,88 void __user *ubuf, void __user *ubufend)89{90 struct gru_tlb_global_handle *tgh;91 int i;92 93 if (GRU_NUM_TGH * GRU_CACHE_LINE_BYTES > ubufend - ubuf)94 return -EFBIG;95 96 for (i = 0; i < GRU_NUM_TGH; i++) {97 tgh = get_tgh(gru->gs_gru_base_vaddr, i);98 if (gru_user_copy_handle(&ubuf, tgh))99 goto fail;100 }101 return GRU_NUM_TGH * GRU_CACHE_LINE_BYTES;102 103fail:104 return -EFAULT;105}106 107static int gru_dump_context(struct gru_state *gru, int ctxnum,108 void __user *ubuf, void __user *ubufend, char data_opt,109 char lock_cch, char flush_cbrs)110{111 struct gru_dump_context_header hdr;112 struct gru_dump_context_header __user *uhdr = ubuf;113 struct gru_context_configuration_handle *cch, *ubufcch;114 struct gru_thread_state *gts;115 int try, cch_locked, cbrcnt = 0, dsrcnt = 0, bytes = 0, ret = 0;116 void *grubase;117 118 memset(&hdr, 0, sizeof(hdr));119 grubase = gru->gs_gru_base_vaddr;120 cch = get_cch(grubase, ctxnum);121 for (try = 0; try < CCH_LOCK_ATTEMPTS; try++) {122 cch_locked = trylock_cch_handle(cch);123 if (cch_locked)124 break;125 msleep(1);126 }127 128 ubuf += sizeof(hdr);129 ubufcch = ubuf;130 if (gru_user_copy_handle(&ubuf, cch)) {131 if (cch_locked)132 unlock_cch_handle(cch);133 return -EFAULT;134 }135 if (cch_locked)136 ubufcch->delresp = 0;137 bytes = sizeof(hdr) + GRU_CACHE_LINE_BYTES;138 139 if (cch_locked || !lock_cch) {140 gts = gru->gs_gts[ctxnum];141 if (gts && gts->ts_vma) {142 hdr.pid = gts->ts_tgid_owner;143 hdr.vaddr = gts->ts_vma->vm_start;144 }145 if (cch->state != CCHSTATE_INACTIVE) {146 cbrcnt = hweight64(cch->cbr_allocation_map) *147 GRU_CBR_AU_SIZE;148 dsrcnt = data_opt ? hweight32(cch->dsr_allocation_map) *149 GRU_DSR_AU_CL : 0;150 }151 bytes += (3 * cbrcnt + dsrcnt) * GRU_CACHE_LINE_BYTES;152 if (bytes > ubufend - ubuf)153 ret = -EFBIG;154 else155 ret = gru_dump_context_data(grubase, cch, ubuf, ctxnum,156 dsrcnt, flush_cbrs);157 }158 if (cch_locked)159 unlock_cch_handle(cch);160 if (ret)161 return ret;162 163 hdr.magic = GRU_DUMP_MAGIC;164 hdr.gid = gru->gs_gid;165 hdr.ctxnum = ctxnum;166 hdr.cbrcnt = cbrcnt;167 hdr.dsrcnt = dsrcnt;168 hdr.cch_locked = cch_locked;169 if (copy_to_user(uhdr, &hdr, sizeof(hdr)))170 return -EFAULT;171 172 return bytes;173}174 175int gru_dump_chiplet_request(unsigned long arg)176{177 struct gru_state *gru;178 struct gru_dump_chiplet_state_req req;179 void __user *ubuf;180 void __user *ubufend;181 int ctxnum, ret, cnt = 0;182 183 if (copy_from_user(&req, (void __user *)arg, sizeof(req)))184 return -EFAULT;185 186 /* Currently, only dump by gid is implemented */187 if (req.gid >= gru_max_gids)188 return -EINVAL;189 req.gid = array_index_nospec(req.gid, gru_max_gids);190 191 gru = GID_TO_GRU(req.gid);192 ubuf = req.buf;193 ubufend = req.buf + req.buflen;194 195 ret = gru_dump_tfm(gru, ubuf, ubufend);196 if (ret < 0)197 goto fail;198 ubuf += ret;199 200 ret = gru_dump_tgh(gru, ubuf, ubufend);201 if (ret < 0)202 goto fail;203 ubuf += ret;204 205 for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {206 if (req.ctxnum == ctxnum || req.ctxnum < 0) {207 ret = gru_dump_context(gru, ctxnum, ubuf, ubufend,208 req.data_opt, req.lock_cch,209 req.flush_cbrs);210 if (ret < 0)211 goto fail;212 ubuf += ret;213 cnt++;214 }215 }216 217 if (copy_to_user((void __user *)arg, &req, sizeof(req)))218 return -EFAULT;219 return cnt;220 221fail:222 return ret;223}224