403 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Provide TDMA helper functions used by cipher and hash algorithm4 * implementations.5 *6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>7 * Author: Arnaud Ebalard <arno@natisbad.org>8 *9 * This work is based on an initial version written by10 * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc >11 */12 13#include "cesa.h"14 15bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter,16 struct mv_cesa_sg_dma_iter *sgiter,17 unsigned int len)18{19 if (!sgiter->sg)20 return false;21 22 sgiter->op_offset += len;23 sgiter->offset += len;24 if (sgiter->offset == sg_dma_len(sgiter->sg)) {25 if (sg_is_last(sgiter->sg))26 return false;27 sgiter->offset = 0;28 sgiter->sg = sg_next(sgiter->sg);29 }30 31 if (sgiter->op_offset == iter->op_len)32 return false;33 34 return true;35}36 37void mv_cesa_dma_step(struct mv_cesa_req *dreq)38{39 struct mv_cesa_engine *engine = dreq->engine;40 41 writel_relaxed(0, engine->regs + CESA_SA_CFG);42 43 mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE);44 writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B |45 CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN,46 engine->regs + CESA_TDMA_CONTROL);47 48 writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT |49 CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS,50 engine->regs + CESA_SA_CFG);51 writel_relaxed(dreq->chain.first->cur_dma,52 engine->regs + CESA_TDMA_NEXT_ADDR);53 WARN_ON(readl(engine->regs + CESA_SA_CMD) &54 CESA_SA_CMD_EN_CESA_SA_ACCL0);55 writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD);56}57 58void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq)59{60 struct mv_cesa_tdma_desc *tdma;61 62 for (tdma = dreq->chain.first; tdma;) {63 struct mv_cesa_tdma_desc *old_tdma = tdma;64 u32 type = tdma->flags & CESA_TDMA_TYPE_MSK;65 66 if (type == CESA_TDMA_OP)67 dma_pool_free(cesa_dev->dma->op_pool, tdma->op,68 le32_to_cpu(tdma->src));69 70 tdma = tdma->next;71 dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma,72 old_tdma->cur_dma);73 }74 75 dreq->chain.first = NULL;76 dreq->chain.last = NULL;77}78 79void mv_cesa_dma_prepare(struct mv_cesa_req *dreq,80 struct mv_cesa_engine *engine)81{82 struct mv_cesa_tdma_desc *tdma;83 84 for (tdma = dreq->chain.first; tdma; tdma = tdma->next) {85 if (tdma->flags & CESA_TDMA_DST_IN_SRAM)86 tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma);87 88 if (tdma->flags & CESA_TDMA_SRC_IN_SRAM)89 tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma);90 91 if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP)92 mv_cesa_adjust_op(engine, tdma->op);93 }94}95 96void mv_cesa_tdma_chain(struct mv_cesa_engine *engine,97 struct mv_cesa_req *dreq)98{99 if (engine->chain.first == NULL && engine->chain.last == NULL) {100 engine->chain.first = dreq->chain.first;101 engine->chain.last = dreq->chain.last;102 } else {103 struct mv_cesa_tdma_desc *last;104 105 last = engine->chain.last;106 last->next = dreq->chain.first;107 engine->chain.last = dreq->chain.last;108 109 /*110 * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on111 * the last element of the current chain, or if the request112 * being queued needs the IV regs to be set before lauching113 * the request.114 */115 if (!(last->flags & CESA_TDMA_BREAK_CHAIN) &&116 !(dreq->chain.first->flags & CESA_TDMA_SET_STATE))117 last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma);118 }119}120 121int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status)122{123 struct crypto_async_request *req = NULL;124 struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL;125 dma_addr_t tdma_cur;126 int res = 0;127 128 tdma_cur = readl(engine->regs + CESA_TDMA_CUR);129 130 for (tdma = engine->chain.first; tdma; tdma = next) {131 spin_lock_bh(&engine->lock);132 next = tdma->next;133 spin_unlock_bh(&engine->lock);134 135 if (tdma->flags & CESA_TDMA_END_OF_REQ) {136 struct crypto_async_request *backlog = NULL;137 struct mv_cesa_ctx *ctx;138 u32 current_status;139 140 spin_lock_bh(&engine->lock);141 /*142 * if req is NULL, this means we're processing the143 * request in engine->req.144 */145 if (!req)146 req = engine->req;147 else148 req = mv_cesa_dequeue_req_locked(engine,149 &backlog);150 151 /* Re-chaining to the next request */152 engine->chain.first = tdma->next;153 tdma->next = NULL;154 155 /* If this is the last request, clear the chain */156 if (engine->chain.first == NULL)157 engine->chain.last = NULL;158 spin_unlock_bh(&engine->lock);159 160 ctx = crypto_tfm_ctx(req->tfm);161 current_status = (tdma->cur_dma == tdma_cur) ?162 status : CESA_SA_INT_ACC0_IDMA_DONE;163 res = ctx->ops->process(req, current_status);164 ctx->ops->complete(req);165 166 if (res == 0)167 mv_cesa_engine_enqueue_complete_request(engine,168 req);169 170 if (backlog)171 crypto_request_complete(backlog, -EINPROGRESS);172 }173 174 if (res || tdma->cur_dma == tdma_cur)175 break;176 }177 178 /*179 * Save the last request in error to engine->req, so that the core180 * knows which request was faulty181 */182 if (res) {183 spin_lock_bh(&engine->lock);184 engine->req = req;185 spin_unlock_bh(&engine->lock);186 }187 188 return res;189}190 191static struct mv_cesa_tdma_desc *192mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags)193{194 struct mv_cesa_tdma_desc *new_tdma = NULL;195 dma_addr_t dma_handle;196 197 new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags,198 &dma_handle);199 if (!new_tdma)200 return ERR_PTR(-ENOMEM);201 202 new_tdma->cur_dma = dma_handle;203 if (chain->last) {204 chain->last->next_dma = cpu_to_le32(dma_handle);205 chain->last->next = new_tdma;206 } else {207 chain->first = new_tdma;208 }209 210 chain->last = new_tdma;211 212 return new_tdma;213}214 215int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src,216 u32 size, u32 flags, gfp_t gfp_flags)217{218 struct mv_cesa_tdma_desc *tdma, *op_desc;219 220 tdma = mv_cesa_dma_add_desc(chain, gfp_flags);221 if (IS_ERR(tdma))222 return PTR_ERR(tdma);223 224 /* We re-use an existing op_desc object to retrieve the context225 * and result instead of allocating a new one.226 * There is at least one object of this type in a CESA crypto227 * req, just pick the first one in the chain.228 */229 for (op_desc = chain->first; op_desc; op_desc = op_desc->next) {230 u32 type = op_desc->flags & CESA_TDMA_TYPE_MSK;231 232 if (type == CESA_TDMA_OP)233 break;234 }235 236 if (!op_desc)237 return -EIO;238 239 tdma->byte_cnt = cpu_to_le32(size | BIT(31));240 tdma->src_dma = src;241 tdma->dst_dma = op_desc->src_dma;242 tdma->op = op_desc->op;243 244 flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);245 tdma->flags = flags | CESA_TDMA_RESULT;246 return 0;247}248 249struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain,250 const struct mv_cesa_op_ctx *op_templ,251 bool skip_ctx,252 gfp_t flags)253{254 struct mv_cesa_tdma_desc *tdma;255 struct mv_cesa_op_ctx *op;256 dma_addr_t dma_handle;257 unsigned int size;258 259 tdma = mv_cesa_dma_add_desc(chain, flags);260 if (IS_ERR(tdma))261 return ERR_CAST(tdma);262 263 op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle);264 if (!op)265 return ERR_PTR(-ENOMEM);266 267 *op = *op_templ;268 269 size = skip_ctx ? sizeof(op->desc) : sizeof(*op);270 271 tdma = chain->last;272 tdma->op = op;273 tdma->byte_cnt = cpu_to_le32(size | BIT(31));274 tdma->src = cpu_to_le32(dma_handle);275 tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET;276 tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP;277 278 return op;279}280 281int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain,282 dma_addr_t dst, dma_addr_t src, u32 size,283 u32 flags, gfp_t gfp_flags)284{285 struct mv_cesa_tdma_desc *tdma;286 287 tdma = mv_cesa_dma_add_desc(chain, gfp_flags);288 if (IS_ERR(tdma))289 return PTR_ERR(tdma);290 291 tdma->byte_cnt = cpu_to_le32(size | BIT(31));292 tdma->src_dma = src;293 tdma->dst_dma = dst;294 295 flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);296 tdma->flags = flags | CESA_TDMA_DATA;297 298 return 0;299}300 301int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags)302{303 struct mv_cesa_tdma_desc *tdma;304 305 tdma = mv_cesa_dma_add_desc(chain, flags);306 return PTR_ERR_OR_ZERO(tdma);307}308 309int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags)310{311 struct mv_cesa_tdma_desc *tdma;312 313 tdma = mv_cesa_dma_add_desc(chain, flags);314 if (IS_ERR(tdma))315 return PTR_ERR(tdma);316 317 tdma->byte_cnt = cpu_to_le32(BIT(31));318 319 return 0;320}321 322int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain,323 struct mv_cesa_dma_iter *dma_iter,324 struct mv_cesa_sg_dma_iter *sgiter,325 gfp_t gfp_flags)326{327 u32 flags = sgiter->dir == DMA_TO_DEVICE ?328 CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM;329 unsigned int len;330 331 do {332 dma_addr_t dst, src;333 int ret;334 335 len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter);336 if (sgiter->dir == DMA_TO_DEVICE) {337 dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;338 src = sg_dma_address(sgiter->sg) + sgiter->offset;339 } else {340 dst = sg_dma_address(sgiter->sg) + sgiter->offset;341 src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;342 }343 344 ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len,345 flags, gfp_flags);346 if (ret)347 return ret;348 349 } while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len));350 351 return 0;352}353 354size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine,355 struct scatterlist *sgl, unsigned int nents,356 unsigned int sram_off, size_t buflen, off_t skip,357 bool to_sram)358{359 unsigned int sg_flags = SG_MITER_ATOMIC;360 struct sg_mapping_iter miter;361 unsigned int offset = 0;362 363 if (to_sram)364 sg_flags |= SG_MITER_FROM_SG;365 else366 sg_flags |= SG_MITER_TO_SG;367 368 sg_miter_start(&miter, sgl, nents, sg_flags);369 370 if (!sg_miter_skip(&miter, skip))371 return 0;372 373 while ((offset < buflen) && sg_miter_next(&miter)) {374 unsigned int len;375 376 len = min(miter.length, buflen - offset);377 378 if (to_sram) {379 if (engine->pool)380 memcpy(engine->sram_pool + sram_off + offset,381 miter.addr, len);382 else383 memcpy_toio(engine->sram + sram_off + offset,384 miter.addr, len);385 } else {386 if (engine->pool)387 memcpy(miter.addr,388 engine->sram_pool + sram_off + offset,389 len);390 else391 memcpy_fromio(miter.addr,392 engine->sram + sram_off + offset,393 len);394 }395 396 offset += len;397 }398 399 sg_miter_stop(&miter);400 401 return offset;402}403