402 lines · c
1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)2/* Copyright(c) 2021 Intel Corporation */3#include <linux/delay.h>4#include <linux/iopoll.h>5#include <linux/mutex.h>6#include <linux/types.h>7#include "adf_accel_devices.h"8#include "adf_common_drv.h"9#include "adf_gen2_pfvf.h"10#include "adf_pfvf_msg.h"11#include "adf_pfvf_pf_proto.h"12#include "adf_pfvf_vf_proto.h"13#include "adf_pfvf_utils.h"14 15 /* VF2PF interrupts */16#define ADF_GEN2_VF_MSK 0xFFFF17#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)18#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & ADF_GEN2_VF_MSK) << 9)19 20#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))21#define ADF_GEN2_VF_PF2VF_OFFSET 0x20022 23#define ADF_GEN2_CSR_IN_USE 0x6AC224#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE25 26enum gen2_csr_pos {27 ADF_GEN2_CSR_PF2VF_OFFSET = 0,28 ADF_GEN2_CSR_VF2PF_OFFSET = 16,29};30 31#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 232#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F33#define ADF_PFVF_GEN2_MSGDATA_SHIFT 634#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF35 36static const struct pfvf_csr_format csr_gen2_fmt = {37 { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK },38 { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK },39};40 41#define ADF_PFVF_MSG_RETRY_DELAY 542#define ADF_PFVF_MSG_MAX_RETRIES 343 44static u32 adf_gen2_pf_get_pfvf_offset(u32 i)45{46 return ADF_GEN2_PF_PF2VF_OFFSET(i);47}48 49static u32 adf_gen2_vf_get_pfvf_offset(u32 i)50{51 return ADF_GEN2_VF_PF2VF_OFFSET;52}53 54static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)55{56 /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */57 if (vf_mask & ADF_GEN2_VF_MSK) {58 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)59 & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);60 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);61 }62}63 64static void adf_gen2_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr)65{66 /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */67 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)68 | ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK);69 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);70}71 72static u32 adf_gen2_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr)73{74 u32 sources, disabled, pending;75 u32 errsou3, errmsk3;76 77 /* Get the interrupt sources triggered by VFs */78 errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);79 sources = ADF_GEN2_ERR_REG_VF2PF(errsou3);80 81 if (!sources)82 return 0;83 84 /* Get the already disabled interrupts */85 errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);86 disabled = ADF_GEN2_ERR_REG_VF2PF(errmsk3);87 88 pending = sources & ~disabled;89 if (!pending)90 return 0;91 92 /* Due to HW limitations, when disabling the interrupts, we can't93 * just disable the requested sources, as this would lead to missed94 * interrupts if ERRSOU3 changes just before writing to ERRMSK3.95 * To work around it, disable all and re-enable only the sources that96 * are not in vf_mask and were not already disabled. Re-enabling will97 * trigger a new interrupt for the sources that have changed in the98 * meantime, if any.99 */100 errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK);101 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3);102 103 /* Update only section of errmsk3 related to VF2PF */104 errmsk3 &= ~ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK);105 errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(sources | disabled);106 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3);107 108 /* Return the sources of the (new) interrupt(s) */109 return pending;110}111 112static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset)113{114 return ADF_PFVF_INT << offset;115}116 117static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset)118{119 return (csr_msg & 0xFFFF) << offset;120}121 122static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset)123{124 return (csr_val >> offset) & 0xFFFF;125}126 127static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset)128{129 return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE;130}131 132static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset)133{134 *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset);135}136 137static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset)138{139 *msg |= (ADF_GEN2_CSR_IN_USE << offset);140}141 142static bool is_legacy_user_pfvf_message(u32 msg)143{144 return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM);145}146 147static bool is_pf2vf_notification(u8 msg_type)148{149 switch (msg_type) {150 case ADF_PF2VF_MSGTYPE_RESTARTING:151 return true;152 default:153 return false;154 }155}156 157static bool is_vf2pf_notification(u8 msg_type)158{159 switch (msg_type) {160 case ADF_VF2PF_MSGTYPE_INIT:161 case ADF_VF2PF_MSGTYPE_SHUTDOWN:162 return true;163 default:164 return false;165 }166}167 168struct pfvf_gen2_params {169 u32 pfvf_offset;170 struct mutex *csr_lock; /* lock preventing concurrent access of CSR */171 enum gen2_csr_pos local_offset;172 enum gen2_csr_pos remote_offset;173 bool (*is_notification_message)(u8 msg_type);174 u8 compat_ver;175};176 177static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev,178 struct pfvf_message msg,179 struct pfvf_gen2_params *params)180{181 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);182 enum gen2_csr_pos remote_offset = params->remote_offset;183 enum gen2_csr_pos local_offset = params->local_offset;184 unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES;185 struct mutex *lock = params->csr_lock;186 u32 pfvf_offset = params->pfvf_offset;187 u32 int_bit;188 u32 csr_val;189 u32 csr_msg;190 int ret;191 192 /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This193 * allows us to build and read messages as if they where all 0 based.194 * However, send and receive are in a single shared 32 bits register,195 * so we need to shift and/or mask the message half before decoding196 * it and after encoding it. Which one to shift depends on the197 * direction.198 */199 200 int_bit = gen2_csr_get_int_bit(local_offset);201 202 csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt);203 if (unlikely(!csr_msg))204 return -EINVAL;205 206 /* Prepare for CSR format, shifting the wire message in place and207 * setting the in use pattern208 */209 csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset);210 gen2_csr_set_in_use(&csr_msg, remote_offset);211 212 mutex_lock(lock);213 214start:215 /* Check if the PFVF CSR is in use by remote function */216 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);217 if (gen2_csr_is_in_use(csr_val, local_offset)) {218 dev_dbg(&GET_DEV(accel_dev),219 "PFVF CSR in use by remote function\n");220 goto retry;221 }222 223 /* Attempt to get ownership of the PFVF CSR */224 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit);225 226 /* Wait for confirmation from remote func it received the message */227 ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit),228 ADF_PFVF_MSG_ACK_DELAY_US,229 ADF_PFVF_MSG_ACK_MAX_DELAY_US,230 true, pmisc_addr, pfvf_offset);231 if (unlikely(ret < 0)) {232 dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");233 csr_val &= ~int_bit;234 }235 236 /* For fire-and-forget notifications, the receiver does not clear237 * the in-use pattern. This is used to detect collisions.238 */239 if (params->is_notification_message(msg.type) && csr_val != csr_msg) {240 /* Collision must have overwritten the message */241 dev_err(&GET_DEV(accel_dev),242 "Collision on notification - PFVF CSR overwritten by remote function\n");243 goto retry;244 }245 246 /* If the far side did not clear the in-use pattern it is either247 * 1) Notification - message left intact to detect collision248 * 2) Older protocol (compatibility version < 3) on the far side249 * where the sender is responsible for clearing the in-use250 * pattern after the received has acknowledged receipt.251 * In either case, clear the in-use pattern now.252 */253 if (gen2_csr_is_in_use(csr_val, remote_offset)) {254 gen2_csr_clear_in_use(&csr_val, remote_offset);255 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);256 }257 258out:259 mutex_unlock(lock);260 return ret;261 262retry:263 if (--retries) {264 msleep(ADF_PFVF_MSG_RETRY_DELAY);265 goto start;266 } else {267 ret = -EBUSY;268 goto out;269 }270}271 272static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev,273 struct pfvf_gen2_params *params)274{275 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);276 enum gen2_csr_pos remote_offset = params->remote_offset;277 enum gen2_csr_pos local_offset = params->local_offset;278 u32 pfvf_offset = params->pfvf_offset;279 struct pfvf_message msg = { 0 };280 u32 int_bit;281 u32 csr_val;282 u16 csr_msg;283 284 int_bit = gen2_csr_get_int_bit(local_offset);285 286 /* Read message */287 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);288 if (!(csr_val & int_bit)) {289 dev_info(&GET_DEV(accel_dev),290 "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val);291 return msg;292 }293 294 /* Extract the message from the CSR */295 csr_msg = gen2_csr_msg_from_position(csr_val, local_offset);296 297 /* Ignore legacy non-system (non-kernel) messages */298 if (unlikely(is_legacy_user_pfvf_message(csr_msg))) {299 dev_dbg(&GET_DEV(accel_dev),300 "Ignored non-system message (0x%.8x);\n", csr_val);301 /* Because this must be a legacy message, the far side302 * must clear the in-use pattern, so don't do it.303 */304 return msg;305 }306 307 /* Return the pfvf_message format */308 msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt);309 310 /* The in-use pattern is not cleared for notifications (so that311 * it can be used for collision detection) or older implementations312 */313 if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK &&314 !params->is_notification_message(msg.type))315 gen2_csr_clear_in_use(&csr_val, remote_offset);316 317 /* To ACK, clear the INT bit */318 csr_val &= ~int_bit;319 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);320 321 return msg;322}323 324static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,325 u32 pfvf_offset, struct mutex *csr_lock)326{327 struct pfvf_gen2_params params = {328 .csr_lock = csr_lock,329 .pfvf_offset = pfvf_offset,330 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,331 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,332 .is_notification_message = is_pf2vf_notification,333 };334 335 return adf_gen2_pfvf_send(accel_dev, msg, ¶ms);336}337 338static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,339 u32 pfvf_offset, struct mutex *csr_lock)340{341 struct pfvf_gen2_params params = {342 .csr_lock = csr_lock,343 .pfvf_offset = pfvf_offset,344 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,345 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,346 .is_notification_message = is_vf2pf_notification,347 };348 349 return adf_gen2_pfvf_send(accel_dev, msg, ¶ms);350}351 352static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev,353 u32 pfvf_offset, u8 compat_ver)354{355 struct pfvf_gen2_params params = {356 .pfvf_offset = pfvf_offset,357 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,358 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,359 .is_notification_message = is_pf2vf_notification,360 .compat_ver = compat_ver,361 };362 363 return adf_gen2_pfvf_recv(accel_dev, ¶ms);364}365 366static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev,367 u32 pfvf_offset, u8 compat_ver)368{369 struct pfvf_gen2_params params = {370 .pfvf_offset = pfvf_offset,371 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,372 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,373 .is_notification_message = is_vf2pf_notification,374 .compat_ver = compat_ver,375 };376 377 return adf_gen2_pfvf_recv(accel_dev, ¶ms);378}379 380void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)381{382 pfvf_ops->enable_comms = adf_enable_pf2vf_comms;383 pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset;384 pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset;385 pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;386 pfvf_ops->disable_all_vf2pf_interrupts = adf_gen2_disable_all_vf2pf_interrupts;387 pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen2_disable_pending_vf2pf_interrupts;388 pfvf_ops->send_msg = adf_gen2_pf2vf_send;389 pfvf_ops->recv_msg = adf_gen2_vf2pf_recv;390}391EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops);392 393void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)394{395 pfvf_ops->enable_comms = adf_enable_vf2pf_comms;396 pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset;397 pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset;398 pfvf_ops->send_msg = adf_gen2_vf2pf_send;399 pfvf_ops->recv_msg = adf_gen2_pf2vf_recv;400}401EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops);402