276 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright 2018-2020 Broadcom.4 */5#include <linux/dma-mapping.h>6#include <linux/mm.h>7#include <linux/pagemap.h>8#include <linux/pgtable.h>9#include <linux/vmalloc.h>10 11#include <asm/page.h>12#include <linux/unaligned.h>13 14#include <uapi/linux/misc/bcm_vk.h>15 16#include "bcm_vk.h"17#include "bcm_vk_msg.h"18#include "bcm_vk_sg.h"19 20/*21 * Valkyrie has a hardware limitation of 16M transfer size.22 * So limit the SGL chunks to 16M.23 */24#define BCM_VK_MAX_SGL_CHUNK SZ_16M25 26static int bcm_vk_dma_alloc(struct device *dev,27 struct bcm_vk_dma *dma,28 int dir,29 struct _vk_data *vkdata);30static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);31 32/* Uncomment to dump SGLIST */33/* #define BCM_VK_DUMP_SGLIST */34 35static int bcm_vk_dma_alloc(struct device *dev,36 struct bcm_vk_dma *dma,37 int direction,38 struct _vk_data *vkdata)39{40 dma_addr_t addr, sg_addr;41 int err;42 int i;43 int offset;44 u32 size;45 u32 remaining_size;46 u32 transfer_size;47 u64 data;48 unsigned long first, last;49 struct _vk_data *sgdata;50 51 /* Get 64-bit user address */52 data = get_unaligned(&vkdata->address);53 54 /* offset into first page */55 offset = offset_in_page(data);56 57 /* Calculate number of pages */58 first = (data & PAGE_MASK) >> PAGE_SHIFT;59 last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;60 dma->nr_pages = last - first + 1;61 62 /* Allocate DMA pages */63 dma->pages = kmalloc_array(dma->nr_pages,64 sizeof(struct page *),65 GFP_KERNEL);66 if (!dma->pages)67 return -ENOMEM;68 69 dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",70 data, vkdata->size, dma->nr_pages);71 72 dma->direction = direction;73 74 /* Get user pages into memory */75 err = get_user_pages_fast(data & PAGE_MASK,76 dma->nr_pages,77 direction == DMA_FROM_DEVICE,78 dma->pages);79 if (err != dma->nr_pages) {80 dma->nr_pages = (err >= 0) ? err : 0;81 dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",82 err, dma->nr_pages);83 return err < 0 ? err : -EINVAL;84 }85 86 /* Max size of sg list is 1 per mapped page + fields at start */87 dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +88 (sizeof(u32) * SGLIST_VKDATA_START);89 90 /* Allocate sglist */91 dma->sglist = dma_alloc_coherent(dev,92 dma->sglen,93 &dma->handle,94 GFP_KERNEL);95 if (!dma->sglist)96 return -ENOMEM;97 98 dma->sglist[SGLIST_NUM_SG] = 0;99 dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;100 remaining_size = vkdata->size;101 sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];102 103 /* Map all pages into DMA */104 size = min_t(size_t, PAGE_SIZE - offset, remaining_size);105 remaining_size -= size;106 sg_addr = dma_map_page(dev,107 dma->pages[0],108 offset,109 size,110 dma->direction);111 transfer_size = size;112 if (unlikely(dma_mapping_error(dev, sg_addr))) {113 __free_page(dma->pages[0]);114 return -EIO;115 }116 117 for (i = 1; i < dma->nr_pages; i++) {118 size = min_t(size_t, PAGE_SIZE, remaining_size);119 remaining_size -= size;120 addr = dma_map_page(dev,121 dma->pages[i],122 0,123 size,124 dma->direction);125 if (unlikely(dma_mapping_error(dev, addr))) {126 __free_page(dma->pages[i]);127 return -EIO;128 }129 130 /*131 * Compress SG list entry when pages are contiguous132 * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK133 */134 if ((addr == (sg_addr + transfer_size)) &&135 ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {136 /* pages are contiguous, add to same sg entry */137 transfer_size += size;138 } else {139 /* pages are not contiguous, write sg entry */140 sgdata->size = transfer_size;141 put_unaligned(sg_addr, (u64 *)&sgdata->address);142 dma->sglist[SGLIST_NUM_SG]++;143 144 /* start new sg entry */145 sgdata++;146 sg_addr = addr;147 transfer_size = size;148 }149 }150 /* Write last sg list entry */151 sgdata->size = transfer_size;152 put_unaligned(sg_addr, (u64 *)&sgdata->address);153 dma->sglist[SGLIST_NUM_SG]++;154 155 /* Update pointers and size field to point to sglist */156 put_unaligned((u64)dma->handle, &vkdata->address);157 vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +158 (sizeof(u32) * SGLIST_VKDATA_START);159 160#ifdef BCM_VK_DUMP_SGLIST161 dev_dbg(dev,162 "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",163 (u64)dma->sglist,164 dma->handle,165 dma->sglen,166 vkdata->size);167 for (i = 0; i < vkdata->size / sizeof(u32); i++)168 dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);169#endif170 171 return 0;172}173 174int bcm_vk_sg_alloc(struct device *dev,175 struct bcm_vk_dma *dma,176 int dir,177 struct _vk_data *vkdata,178 int num)179{180 int i;181 int rc = -EINVAL;182 183 /* Convert user addresses to DMA SG List */184 for (i = 0; i < num; i++) {185 if (vkdata[i].size && vkdata[i].address) {186 /*187 * If both size and address are non-zero188 * then DMA alloc.189 */190 rc = bcm_vk_dma_alloc(dev,191 &dma[i],192 dir,193 &vkdata[i]);194 } else if (vkdata[i].size ||195 vkdata[i].address) {196 /*197 * If one of size and address are zero198 * there is a problem.199 */200 dev_err(dev,201 "Invalid vkdata %x 0x%x 0x%llx\n",202 i, vkdata[i].size, vkdata[i].address);203 rc = -EINVAL;204 } else {205 /*206 * If size and address are both zero207 * don't convert, but return success.208 */209 rc = 0;210 }211 212 if (rc)213 goto fail_alloc;214 }215 return rc;216 217fail_alloc:218 while (i > 0) {219 i--;220 if (dma[i].sglist)221 bcm_vk_dma_free(dev, &dma[i]);222 }223 return rc;224}225 226static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)227{228 dma_addr_t addr;229 int i;230 int num_sg;231 u32 size;232 struct _vk_data *vkdata;233 234 dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);235 236 /* Unmap all pages in the sglist */237 num_sg = dma->sglist[SGLIST_NUM_SG];238 vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];239 for (i = 0; i < num_sg; i++) {240 size = vkdata[i].size;241 addr = get_unaligned(&vkdata[i].address);242 243 dma_unmap_page(dev, addr, size, dma->direction);244 }245 246 /* Free allocated sglist */247 dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);248 249 /* Release lock on all pages */250 for (i = 0; i < dma->nr_pages; i++)251 put_page(dma->pages[i]);252 253 /* Free allocated dma pages */254 kfree(dma->pages);255 dma->sglist = NULL;256 257 return 0;258}259 260int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,261 int *proc_cnt)262{263 int i;264 265 *proc_cnt = 0;266 /* Unmap and free all pages and sglists */267 for (i = 0; i < num; i++) {268 if (dma[i].sglist) {269 bcm_vk_dma_free(dev, &dma[i]);270 *proc_cnt += 1;271 }272 }273 274 return 0;275}276