181 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2#include <linux/init.h>3#include <linux/scatterlist.h>4#include <linux/mempool.h>5#include <linux/slab.h>6 7#define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools)8#define SG_MEMPOOL_SIZE 29 10struct sg_pool {11 size_t size;12 char *name;13 struct kmem_cache *slab;14 mempool_t *pool;15};16 17#define SP(x) { .size = x, "sgpool-" __stringify(x) }18#if (SG_CHUNK_SIZE < 32)19#error SG_CHUNK_SIZE is too small (must be 32 or greater)20#endif21static struct sg_pool sg_pools[] = {22 SP(8),23 SP(16),24#if (SG_CHUNK_SIZE > 32)25 SP(32),26#if (SG_CHUNK_SIZE > 64)27 SP(64),28#if (SG_CHUNK_SIZE > 128)29 SP(128),30#if (SG_CHUNK_SIZE > 256)31#error SG_CHUNK_SIZE is too large (256 MAX)32#endif33#endif34#endif35#endif36 SP(SG_CHUNK_SIZE)37};38#undef SP39 40static inline unsigned int sg_pool_index(unsigned short nents)41{42 unsigned int index;43 44 BUG_ON(nents > SG_CHUNK_SIZE);45 46 if (nents <= 8)47 index = 0;48 else49 index = get_count_order(nents) - 3;50 51 return index;52}53 54static void sg_pool_free(struct scatterlist *sgl, unsigned int nents)55{56 struct sg_pool *sgp;57 58 sgp = sg_pools + sg_pool_index(nents);59 mempool_free(sgl, sgp->pool);60}61 62static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask)63{64 struct sg_pool *sgp;65 66 sgp = sg_pools + sg_pool_index(nents);67 return mempool_alloc(sgp->pool, gfp_mask);68}69 70/**71 * sg_free_table_chained - Free a previously mapped sg table72 * @table: The sg table header to use73 * @nents_first_chunk: size of the first_chunk SGL passed to74 * sg_alloc_table_chained75 *76 * Description:77 * Free an sg table previously allocated and setup with78 * sg_alloc_table_chained().79 *80 * @nents_first_chunk has to be same with that same parameter passed81 * to sg_alloc_table_chained().82 *83 **/84void sg_free_table_chained(struct sg_table *table,85 unsigned nents_first_chunk)86{87 if (table->orig_nents <= nents_first_chunk)88 return;89 90 if (nents_first_chunk == 1)91 nents_first_chunk = 0;92 93 __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free,94 table->orig_nents);95}96EXPORT_SYMBOL_GPL(sg_free_table_chained);97 98/**99 * sg_alloc_table_chained - Allocate and chain SGLs in an sg table100 * @table: The sg table header to use101 * @nents: Number of entries in sg list102 * @first_chunk: first SGL103 * @nents_first_chunk: number of the SGL of @first_chunk104 *105 * Description:106 * Allocate and chain SGLs in an sg table. If @nents@ is larger than107 * @nents_first_chunk a chained sg table will be setup. @first_chunk is108 * ignored if nents_first_chunk <= 1 because user expects the SGL points109 * non-chain SGL.110 *111 **/112int sg_alloc_table_chained(struct sg_table *table, int nents,113 struct scatterlist *first_chunk, unsigned nents_first_chunk)114{115 int ret;116 117 BUG_ON(!nents);118 119 if (first_chunk && nents_first_chunk) {120 if (nents <= nents_first_chunk) {121 table->nents = table->orig_nents = nents;122 sg_init_table(table->sgl, nents);123 return 0;124 }125 }126 127 /* User supposes that the 1st SGL includes real entry */128 if (nents_first_chunk <= 1) {129 first_chunk = NULL;130 nents_first_chunk = 0;131 }132 133 ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE,134 first_chunk, nents_first_chunk,135 GFP_ATOMIC, sg_pool_alloc);136 if (unlikely(ret))137 sg_free_table_chained(table, nents_first_chunk);138 return ret;139}140EXPORT_SYMBOL_GPL(sg_alloc_table_chained);141 142static __init int sg_pool_init(void)143{144 int i;145 146 for (i = 0; i < SG_MEMPOOL_NR; i++) {147 struct sg_pool *sgp = sg_pools + i;148 int size = sgp->size * sizeof(struct scatterlist);149 150 sgp->slab = kmem_cache_create(sgp->name, size, 0,151 SLAB_HWCACHE_ALIGN, NULL);152 if (!sgp->slab) {153 printk(KERN_ERR "SG_POOL: can't init sg slab %s\n",154 sgp->name);155 goto cleanup_sdb;156 }157 158 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,159 sgp->slab);160 if (!sgp->pool) {161 printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n",162 sgp->name);163 goto cleanup_sdb;164 }165 }166 167 return 0;168 169cleanup_sdb:170 for (i = 0; i < SG_MEMPOOL_NR; i++) {171 struct sg_pool *sgp = sg_pools + i;172 173 mempool_destroy(sgp->pool);174 kmem_cache_destroy(sgp->slab);175 }176 177 return -ENOMEM;178}179 180subsys_initcall(sg_pool_init);181