337 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * SolidRun DPU driver for control plane4 *5 * Copyright (C) 2022-2023 SolidRun6 *7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>8 *9 */10 11#include <linux/iopoll.h>12 13#include "snet_vdpa.h"14 15enum snet_ctrl_opcodes {16 SNET_CTRL_OP_DESTROY = 1,17 SNET_CTRL_OP_READ_VQ_STATE,18 SNET_CTRL_OP_SUSPEND,19 SNET_CTRL_OP_RESUME,20};21 22#define SNET_CTRL_TIMEOUT 200000023 24#define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF25#define SNET_CTRL_IN_PROCESS_MASK 0x0001000026#define SNET_CTRL_CHUNK_RDY_MASK 0x0002000027#define SNET_CTRL_ERROR_MASK 0x0FFC000028 29#define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18))30#define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \31 !((val) & SNET_CTRL_IN_PROCESS_MASK))32#define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))33 34/* Control register used to read data from the DPU */35struct snet_ctrl_reg_ctrl {36 /* Chunk size in 4B words */37 u16 data_size;38 /* We are in the middle of a command */39 u16 in_process:1;40 /* A data chunk is ready and can be consumed */41 u16 chunk_ready:1;42 /* Error code */43 u16 error:10;44 /* Saved for future usage */45 u16 rsvd:4;46};47 48/* Opcode register */49struct snet_ctrl_reg_op {50 u16 opcode;51 /* Only if VQ index is relevant for the command */52 u16 vq_idx;53};54 55struct snet_ctrl_regs {56 struct snet_ctrl_reg_op op;57 struct snet_ctrl_reg_ctrl ctrl;58 u32 rsvd;59 u32 data[];60};61 62static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)63{64 return snet->bar + snet->psnet->cfg.ctrl_off;65}66 67static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)68{69 u32 val;70 71 return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_EMPTY_CTRL(val), 10,72 SNET_CTRL_TIMEOUT);73}74 75static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)76{77 u32 val;78 79 return readx_poll_timeout(ioread32, ®s->op, val, !val, 10, SNET_CTRL_TIMEOUT);80}81 82static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)83{84 u32 val;85 86 return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_DATA_READY(val), 10,87 SNET_CTRL_TIMEOUT);88}89 90static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)91{92 return ioread32(&ctrl_regs->data[word_idx]);93}94 95static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)96{97 return ioread32(&ctrl_regs->ctrl);98}99 100static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)101{102 iowrite32(val, &ctrl_regs->ctrl);103}104 105static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)106{107 iowrite32(val, &ctrl_regs->op);108}109 110static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)111{112 /* Wait until the DPU finishes completely.113 * It will clear the opcode register.114 */115 return snet_wait_for_empty_op(ctrl_regs);116}117 118/* Reading ctrl from the DPU:119 * buf_size must be 4B aligned120 *121 * Steps:122 *123 * (1) Verify that the DPU is not in the middle of another operation by124 * reading the in_process and error bits in the control register.125 * (2) Write the request opcode and the VQ idx in the opcode register126 * and write the buffer size in the control register.127 * (3) Start readind chunks of data, chunk_ready bit indicates that a128 * data chunk is available, we signal that we read the data by clearing the bit.129 * (4) Detect that the transfer is completed when the in_process bit130 * in the control register is cleared or when the an error appears.131 */132static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,133 u32 buf_size)134{135 struct pci_dev *pdev = snet->pdev;136 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);137 u32 *bfr_ptr = (u32 *)buffer;138 u32 val;139 u16 buf_words;140 int ret;141 u16 words, i, tot_words = 0;142 143 /* Supported for config 2+ */144 if (!SNET_CFG_VER(snet, 2))145 return -EOPNOTSUPP;146 147 if (!IS_ALIGNED(buf_size, 4))148 return -EINVAL;149 150 mutex_lock(&snet->ctrl_lock);151 152 buf_words = buf_size / 4;153 154 /* Make sure control register is empty */155 ret = snet_wait_for_empty_ctrl(regs);156 if (ret) {157 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");158 goto exit;159 }160 161 /* We need to write the buffer size in the control register, and the opcode + vq index in162 * the opcode register.163 * We use a spinlock to serialize the writes.164 */165 spin_lock(&snet->ctrl_spinlock);166 167 snet_write_ctrl(regs, buf_words);168 snet_write_op(regs, opcode | (vq_idx << 16));169 170 spin_unlock(&snet->ctrl_spinlock);171 172 while (buf_words != tot_words) {173 ret = snet_wait_for_data(regs);174 if (ret) {175 SNET_WARN(pdev, "Timeout waiting for control data\n");176 goto exit;177 }178 179 val = snet_read_ctrl(regs);180 181 /* Error? */182 if (val & SNET_CTRL_ERROR_MASK) {183 ret = SNET_VAL_TO_ERR(val);184 SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);185 goto exit;186 }187 188 words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);189 190 for (i = 0; i < words; i++) {191 *bfr_ptr = snet_read32_word(regs, i);192 bfr_ptr++;193 }194 195 tot_words += words;196 197 /* Is the job completed? */198 if (!(val & SNET_CTRL_IN_PROCESS_MASK))199 break;200 201 /* Clear the chunk ready bit and continue */202 val &= ~SNET_CTRL_CHUNK_RDY_MASK;203 snet_write_ctrl(regs, val);204 }205 206 ret = snet_wait_for_dpu_completion(regs);207 if (ret)208 SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");209 210exit:211 mutex_unlock(&snet->ctrl_lock);212 return ret;213}214 215/* Send a control message to the DPU using the old mechanism216 * used with config version 1.217 */218static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)219{220 struct pci_dev *pdev = snet->pdev;221 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);222 int ret;223 224 mutex_lock(&snet->ctrl_lock);225 226 /* Old mechanism uses just 1 register, the opcode register.227 * Make sure that the opcode register is empty, and that the DPU isn't228 * processing an old message.229 */230 ret = snet_wait_for_empty_op(regs);231 if (ret) {232 SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");233 goto exit;234 }235 236 /* Write the message */237 snet_write_op(regs, opcode);238 239 /* DPU ACKs the message by clearing the opcode register */240 ret = snet_wait_for_empty_op(regs);241 if (ret)242 SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");243 244exit:245 mutex_unlock(&snet->ctrl_lock);246 return ret;247}248 249/* Send a control message to the DPU.250 * A control message is a message without payload.251 */252static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)253{254 struct pci_dev *pdev = snet->pdev;255 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);256 u32 val;257 int ret;258 259 /* If config version is not 2+, use the old mechanism */260 if (!SNET_CFG_VER(snet, 2))261 return snet_send_ctrl_msg_old(snet, opcode);262 263 mutex_lock(&snet->ctrl_lock);264 265 /* Make sure control register is empty */266 ret = snet_wait_for_empty_ctrl(regs);267 if (ret) {268 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");269 goto exit;270 }271 272 /* We need to clear the control register and write the opcode + vq index in the opcode273 * register.274 * We use a spinlock to serialize the writes.275 */276 spin_lock(&snet->ctrl_spinlock);277 278 snet_write_ctrl(regs, 0);279 snet_write_op(regs, opcode | (vq_idx << 16));280 281 spin_unlock(&snet->ctrl_spinlock);282 283 /* The DPU ACKs control messages by setting the chunk ready bit284 * without data.285 */286 ret = snet_wait_for_data(regs);287 if (ret) {288 SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");289 goto exit;290 }291 292 /* Check for errors */293 val = snet_read_ctrl(regs);294 ret = SNET_VAL_TO_ERR(val);295 296 /* Clear the chunk ready bit */297 val &= ~SNET_CTRL_CHUNK_RDY_MASK;298 snet_write_ctrl(regs, val);299 300 ret = snet_wait_for_dpu_completion(regs);301 if (ret)302 SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",303 ret);304 305exit:306 mutex_unlock(&snet->ctrl_lock);307 return ret;308}309 310void snet_ctrl_clear(struct snet *snet)311{312 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);313 314 snet_write_op(regs, 0);315}316 317int snet_destroy_dev(struct snet *snet)318{319 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);320}321 322int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)323{324 return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,325 sizeof(*state));326}327 328int snet_suspend_dev(struct snet *snet)329{330 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);331}332 333int snet_resume_dev(struct snet *snet)334{335 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0);336}337