183 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2#include <linux/prefetch.h>3 4/**5 * iommu_fill_pdir - Insert coalesced scatter/gather chunks into the I/O Pdir.6 * @ioc: The I/O Controller.7 * @startsg: The scatter/gather list of coalesced chunks.8 * @nents: The number of entries in the scatter/gather list.9 * @hint: The DMA Hint.10 *11 * This function inserts the coalesced scatter/gather list chunks into the12 * I/O Controller's I/O Pdir.13 */ 14static inline unsigned int15iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents, 16 unsigned long hint,17 void (*iommu_io_pdir_entry)(__le64 *, space_t, unsigned long,18 unsigned long))19{20 struct scatterlist *dma_sg = startsg; /* pointer to current DMA */21 unsigned int n_mappings = 0;22 unsigned long dma_offset = 0, dma_len = 0;23 __le64 *pdirp = NULL;24 25 /* Horrible hack. For efficiency's sake, dma_sg starts one 26 * entry below the true start (it is immediately incremented27 * in the loop) */28 dma_sg--;29 30 while (nents-- > 0) {31 unsigned long vaddr;32 long size;33 34 DBG_RUN_SG(" %d : %08lx %p/%05x\n", nents,35 (unsigned long)sg_dma_address(startsg),36 sg_virt(startsg), startsg->length37 );38 39 40 /*41 ** Look for the start of a new DMA stream42 */43 44 if (sg_dma_address(startsg) & PIDE_FLAG) {45 u32 pide = sg_dma_address(startsg) & ~PIDE_FLAG;46 47 BUG_ON(pdirp && (dma_len != sg_dma_len(dma_sg)));48 49 dma_sg++;50 51 dma_len = sg_dma_len(startsg);52 sg_dma_len(startsg) = 0;53 dma_offset = (unsigned long) pide & ~IOVP_MASK;54 n_mappings++;55#if defined(ZX1_SUPPORT)56 /* Pluto IOMMU IO Virt Address is not zero based */57 sg_dma_address(dma_sg) = pide | ioc->ibase;58#else59 /* SBA, ccio, and dino are zero based.60 * Trying to save a few CPU cycles for most users.61 */62 sg_dma_address(dma_sg) = pide;63#endif64 pdirp = &(ioc->pdir_base[pide >> IOVP_SHIFT]);65 prefetchw(pdirp);66 }67 68 BUG_ON(pdirp == NULL);69 70 vaddr = (unsigned long)sg_virt(startsg);71 sg_dma_len(dma_sg) += startsg->length;72 size = startsg->length + dma_offset;73 dma_offset = 0;74#ifdef IOMMU_MAP_STATS75 ioc->msg_pages += startsg->length >> IOVP_SHIFT;76#endif77 do {78 iommu_io_pdir_entry(pdirp, KERNEL_SPACE, 79 vaddr, hint);80 vaddr += IOVP_SIZE;81 size -= IOVP_SIZE;82 pdirp++;83 } while(unlikely(size > 0));84 startsg++;85 }86 return(n_mappings);87}88 89 90/*91** First pass is to walk the SG list and determine where the breaks are92** in the DMA stream. Allocates PDIR entries but does not fill them.93** Returns the number of DMA chunks.94**95** Doing the fill separate from the coalescing/allocation keeps the96** code simpler. Future enhancement could make one pass through97** the sglist do both.98*/99 100static inline unsigned int101iommu_coalesce_chunks(struct ioc *ioc, struct device *dev,102 struct scatterlist *startsg, int nents,103 int (*iommu_alloc_range)(struct ioc *, struct device *, size_t))104{105 struct scatterlist *contig_sg; /* contig chunk head */106 unsigned long dma_offset, dma_len; /* start/len of DMA stream */107 unsigned int n_mappings = 0;108 unsigned int max_seg_size = min(dma_get_max_seg_size(dev),109 (unsigned)DMA_CHUNK_SIZE);110 unsigned int max_seg_boundary = dma_get_seg_boundary(dev) + 1;111 if (max_seg_boundary) /* check if the addition above didn't overflow */112 max_seg_size = min(max_seg_size, max_seg_boundary);113 114 while (nents > 0) {115 116 /*117 ** Prepare for first/next DMA stream118 */119 contig_sg = startsg;120 dma_len = startsg->length;121 dma_offset = startsg->offset;122 123 /* PARANOID: clear entries */124 sg_dma_address(startsg) = 0;125 sg_dma_len(startsg) = 0;126 127 /*128 ** This loop terminates one iteration "early" since129 ** it's always looking one "ahead".130 */131 while(--nents > 0) {132 unsigned long prev_end, sg_start;133 134 prev_end = (unsigned long)sg_virt(startsg) +135 startsg->length;136 137 startsg++;138 sg_start = (unsigned long)sg_virt(startsg);139 140 /* PARANOID: clear entries */141 sg_dma_address(startsg) = 0;142 sg_dma_len(startsg) = 0;143 144 /*145 ** First make sure current dma stream won't146 ** exceed max_seg_size if we coalesce the147 ** next entry.148 */ 149 if (unlikely(ALIGN(dma_len + dma_offset + startsg->length, IOVP_SIZE) >150 max_seg_size))151 break;152 153 /*154 * Next see if we can append the next chunk (i.e.155 * it must end on one page and begin on another, or156 * it must start on the same address as the previous157 * entry ended.158 */159 if (unlikely((prev_end != sg_start) ||160 ((prev_end | sg_start) & ~PAGE_MASK)))161 break;162 163 dma_len += startsg->length;164 }165 166 /*167 ** End of DMA Stream168 ** Terminate last VCONTIG block.169 ** Allocate space for DMA stream.170 */171 sg_dma_len(contig_sg) = dma_len;172 dma_len = ALIGN(dma_len + dma_offset, IOVP_SIZE);173 sg_dma_address(contig_sg) =174 PIDE_FLAG 175 | (iommu_alloc_range(ioc, dev, dma_len) << IOVP_SHIFT)176 | dma_offset;177 n_mappings++;178 }179 180 return n_mappings;181}182 183