295 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 HiSilicon Limited. */3#include <linux/align.h>4#include <linux/dma-mapping.h>5#include <linux/hisi_acc_qm.h>6#include <linux/module.h>7#include <linux/slab.h>8 9#define HISI_ACC_SGL_SGE_NR_MIN 110#define HISI_ACC_SGL_NR_MAX 25611#define HISI_ACC_SGL_ALIGN_SIZE 6412#define HISI_ACC_MEM_BLOCK_NR 513 14struct acc_hw_sge {15 dma_addr_t buf;16 void *page_ctrl;17 __le32 len;18 __le32 pad;19 __le32 pad0;20 __le32 pad1;21};22 23/* use default sgl head size 64B */24struct hisi_acc_hw_sgl {25 dma_addr_t next_dma;26 __le16 entry_sum_in_chain;27 __le16 entry_sum_in_sgl;28 __le16 entry_length_in_sgl;29 __le16 pad0;30 __le64 pad1[5];31 struct hisi_acc_hw_sgl *next;32 struct acc_hw_sge sge_entries[];33} __aligned(1);34 35struct hisi_acc_sgl_pool {36 struct mem_block {37 struct hisi_acc_hw_sgl *sgl;38 dma_addr_t sgl_dma;39 size_t size;40 } mem_block[HISI_ACC_MEM_BLOCK_NR];41 u32 sgl_num_per_block;42 u32 block_num;43 u32 count;44 u32 sge_nr;45 size_t sgl_size;46};47 48/**49 * hisi_acc_create_sgl_pool() - Create a hw sgl pool.50 * @dev: The device which hw sgl pool belongs to.51 * @count: Count of hisi_acc_hw_sgl in pool.52 * @sge_nr: The count of sge in hw_sgl53 *54 * This function creates a hw sgl pool, after this user can get hw sgl memory55 * from it.56 */57struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,58 u32 count, u32 sge_nr)59{60 u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl;61 struct hisi_acc_sgl_pool *pool;62 struct mem_block *block;63 u32 i, j;64 65 if (!dev || !count || !sge_nr || sge_nr > HISI_ACC_SGL_SGE_NR_MAX)66 return ERR_PTR(-EINVAL);67 68 sgl_size = ALIGN(sizeof(struct acc_hw_sge) * sge_nr +69 sizeof(struct hisi_acc_hw_sgl),70 HISI_ACC_SGL_ALIGN_SIZE);71 72 /*73 * the pool may allocate a block of memory of size PAGE_SIZE * 2^MAX_PAGE_ORDER,74 * block size may exceed 2^31 on ia64, so the max of block size is 2^3175 */76 block_size = 1 << (PAGE_SHIFT + MAX_PAGE_ORDER < 32 ?77 PAGE_SHIFT + MAX_PAGE_ORDER : 31);78 sgl_num_per_block = block_size / sgl_size;79 block_num = count / sgl_num_per_block;80 remain_sgl = count % sgl_num_per_block;81 82 if ((!remain_sgl && block_num > HISI_ACC_MEM_BLOCK_NR) ||83 (remain_sgl > 0 && block_num > HISI_ACC_MEM_BLOCK_NR - 1))84 return ERR_PTR(-EINVAL);85 86 pool = kzalloc(sizeof(*pool), GFP_KERNEL);87 if (!pool)88 return ERR_PTR(-ENOMEM);89 block = pool->mem_block;90 91 for (i = 0; i < block_num; i++) {92 block[i].sgl = dma_alloc_coherent(dev, block_size,93 &block[i].sgl_dma,94 GFP_KERNEL);95 if (!block[i].sgl) {96 dev_err(dev, "Fail to allocate hw SG buffer!\n");97 goto err_free_mem;98 }99 100 block[i].size = block_size;101 }102 103 if (remain_sgl > 0) {104 block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size,105 &block[i].sgl_dma,106 GFP_KERNEL);107 if (!block[i].sgl) {108 dev_err(dev, "Fail to allocate remained hw SG buffer!\n");109 goto err_free_mem;110 }111 112 block[i].size = remain_sgl * sgl_size;113 }114 115 pool->sgl_num_per_block = sgl_num_per_block;116 pool->block_num = remain_sgl ? block_num + 1 : block_num;117 pool->count = count;118 pool->sgl_size = sgl_size;119 pool->sge_nr = sge_nr;120 121 return pool;122 123err_free_mem:124 for (j = 0; j < i; j++)125 dma_free_coherent(dev, block_size, block[j].sgl,126 block[j].sgl_dma);127 128 kfree_sensitive(pool);129 return ERR_PTR(-ENOMEM);130}131EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool);132 133/**134 * hisi_acc_free_sgl_pool() - Free a hw sgl pool.135 * @dev: The device which hw sgl pool belongs to.136 * @pool: Pointer of pool.137 *138 * This function frees memory of a hw sgl pool.139 */140void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool)141{142 struct mem_block *block;143 u32 i;144 145 if (!dev || !pool)146 return;147 148 block = pool->mem_block;149 150 for (i = 0; i < pool->block_num; i++)151 dma_free_coherent(dev, block[i].size, block[i].sgl,152 block[i].sgl_dma);153 154 kfree(pool);155}156EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool);157 158static struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool,159 u32 index, dma_addr_t *hw_sgl_dma)160{161 struct mem_block *block;162 u32 block_index, offset;163 164 block = pool->mem_block;165 block_index = index / pool->sgl_num_per_block;166 offset = index % pool->sgl_num_per_block;167 168 *hw_sgl_dma = block[block_index].sgl_dma + pool->sgl_size * offset;169 return (void *)block[block_index].sgl + pool->sgl_size * offset;170}171 172static void sg_map_to_hw_sg(struct scatterlist *sgl,173 struct acc_hw_sge *hw_sge)174{175 hw_sge->buf = sg_dma_address(sgl);176 hw_sge->len = cpu_to_le32(sg_dma_len(sgl));177 hw_sge->page_ctrl = sg_virt(sgl);178}179 180static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)181{182 u16 var = le16_to_cpu(hw_sgl->entry_sum_in_sgl);183 184 var++;185 hw_sgl->entry_sum_in_sgl = cpu_to_le16(var);186}187 188static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)189{190 hw_sgl->entry_sum_in_chain = cpu_to_le16(sum);191}192 193static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)194{195 struct acc_hw_sge *hw_sge = hw_sgl->sge_entries;196 u16 entry_sum = le16_to_cpu(hw_sgl->entry_sum_in_sgl);197 int i;198 199 for (i = 0; i < entry_sum; i++) {200 hw_sge[i].page_ctrl = NULL;201 hw_sge[i].buf = 0;202 hw_sge[i].len = 0;203 }204}205 206/**207 * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.208 * @dev: The device which hw sgl belongs to.209 * @sgl: Scatterlist which will be mapped to hw sgl.210 * @pool: Pool which hw sgl memory will be allocated in.211 * @index: Index of hisi_acc_hw_sgl in pool.212 * @hw_sgl_dma: The dma address of allocated hw sgl.213 *214 * This function builds hw sgl according input sgl, user can use hw_sgl_dma215 * as src/dst in its BD. Only support single hw sgl currently.216 */217struct hisi_acc_hw_sgl *218hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,219 struct scatterlist *sgl,220 struct hisi_acc_sgl_pool *pool,221 u32 index, dma_addr_t *hw_sgl_dma)222{223 struct hisi_acc_hw_sgl *curr_hw_sgl;224 unsigned int i, sg_n_mapped;225 dma_addr_t curr_sgl_dma = 0;226 struct acc_hw_sge *curr_hw_sge;227 struct scatterlist *sg;228 int sg_n, ret;229 230 if (!dev || !sgl || !pool || !hw_sgl_dma || index >= pool->count)231 return ERR_PTR(-EINVAL);232 233 sg_n = sg_nents(sgl);234 235 sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);236 if (!sg_n_mapped) {237 dev_err(dev, "DMA mapping for SG error!\n");238 return ERR_PTR(-EINVAL);239 }240 241 if (sg_n_mapped > pool->sge_nr) {242 dev_err(dev, "the number of entries in input scatterlist is bigger than SGL pool setting.\n");243 ret = -EINVAL;244 goto err_unmap;245 }246 247 curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);248 if (IS_ERR(curr_hw_sgl)) {249 dev_err(dev, "Get SGL error!\n");250 ret = -ENOMEM;251 goto err_unmap;252 }253 curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr);254 curr_hw_sge = curr_hw_sgl->sge_entries;255 256 for_each_sg(sgl, sg, sg_n_mapped, i) {257 sg_map_to_hw_sg(sg, curr_hw_sge);258 inc_hw_sgl_sge(curr_hw_sgl);259 curr_hw_sge++;260 }261 262 update_hw_sgl_sum_sge(curr_hw_sgl, pool->sge_nr);263 *hw_sgl_dma = curr_sgl_dma;264 265 return curr_hw_sgl;266 267err_unmap:268 dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);269 270 return ERR_PTR(ret);271}272EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl);273 274/**275 * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl.276 * @dev: The device which hw sgl belongs to.277 * @sgl: Related scatterlist.278 * @hw_sgl: Virtual address of hw sgl.279 *280 * This function unmaps allocated hw sgl.281 */282void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,283 struct hisi_acc_hw_sgl *hw_sgl)284{285 if (!dev || !sgl || !hw_sgl)286 return;287 288 dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);289 clear_hw_sgl_sge(hw_sgl);290 hw_sgl->entry_sum_in_chain = 0;291 hw_sgl->entry_sum_in_sgl = 0;292 hw_sgl->entry_length_in_sgl = 0;293}294EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap);295