285 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.5 */6 7#ifndef RXE_QUEUE_H8#define RXE_QUEUE_H9 10/* Implements a simple circular buffer that is shared between user11 * and the driver and can be resized. The requested element size is12 * rounded up to a power of 2 and the number of elements in the buffer13 * is also rounded up to a power of 2. Since the queue is empty when14 * the producer and consumer indices match the maximum capacity of the15 * queue is one less than the number of element slots.16 *17 * Notes:18 * - The driver indices are always masked off to q->index_mask19 * before storing so do not need to be checked on reads.20 * - The user whether user space or kernel is generally21 * not trusted so its parameters are masked to make sure22 * they do not access the queue out of bounds on reads.23 * - The driver indices for queues must not be written24 * by user so a local copy is used and a shared copy is25 * stored when the local copy is changed.26 * - By passing the type in the parameter list separate from q27 * the compiler can eliminate the switch statement when the28 * actual queue type is known when the function is called at29 * compile time.30 * - These queues are lock free. The user and driver must protect31 * changes to their end of the queues with locks if more than one32 * CPU can be accessing it at the same time.33 */34 35/**36 * enum queue_type - type of queue37 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and38 * read by client which may be a user space39 * application or a kernel ulp.40 * Used by rxe internals only.41 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and42 * read by rxe driver.43 * Used by rxe internals only.44 * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and45 * read by rxe driver.46 * Used by kernel verbs APIs only on47 * behalf of ulps.48 * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and49 * read by kernel ulp.50 * Used by kernel verbs APIs only on51 * behalf of ulps.52 */53enum queue_type {54 QUEUE_TYPE_TO_CLIENT,55 QUEUE_TYPE_FROM_CLIENT,56 QUEUE_TYPE_FROM_ULP,57 QUEUE_TYPE_TO_ULP,58};59 60struct rxe_queue_buf;61 62struct rxe_queue {63 struct rxe_dev *rxe;64 struct rxe_queue_buf *buf;65 struct rxe_mmap_info *ip;66 size_t buf_size;67 size_t elem_size;68 unsigned int log2_elem_size;69 u32 index_mask;70 enum queue_type type;71 /* private copy of index for shared queues between72 * driver and clients. Driver reads and writes73 * this copy and then replicates to rxe_queue_buf74 * for read access by clients.75 */76 u32 index;77};78 79int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,80 struct ib_udata *udata, struct rxe_queue_buf *buf,81 size_t buf_size, struct rxe_mmap_info **ip_p);82 83void rxe_queue_reset(struct rxe_queue *q);84 85struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,86 unsigned int elem_size, enum queue_type type);87 88int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,89 unsigned int elem_size, struct ib_udata *udata,90 struct mminfo __user *outbuf,91 spinlock_t *producer_lock, spinlock_t *consumer_lock);92 93void rxe_queue_cleanup(struct rxe_queue *queue);94 95static inline u32 queue_next_index(struct rxe_queue *q, int index)96{97 return (index + 1) & q->index_mask;98}99 100static inline u32 queue_get_producer(const struct rxe_queue *q,101 enum queue_type type)102{103 u32 prod;104 105 switch (type) {106 case QUEUE_TYPE_FROM_CLIENT:107 /* used by rxe, client owns the index */108 prod = smp_load_acquire(&q->buf->producer_index);109 break;110 case QUEUE_TYPE_TO_CLIENT:111 /* used by rxe which owns the index */112 prod = q->index;113 break;114 case QUEUE_TYPE_FROM_ULP:115 /* used by ulp which owns the index */116 prod = q->buf->producer_index;117 break;118 case QUEUE_TYPE_TO_ULP:119 /* used by ulp, rxe owns the index */120 prod = smp_load_acquire(&q->buf->producer_index);121 break;122 }123 124 return prod;125}126 127static inline u32 queue_get_consumer(const struct rxe_queue *q,128 enum queue_type type)129{130 u32 cons;131 132 switch (type) {133 case QUEUE_TYPE_FROM_CLIENT:134 /* used by rxe which owns the index */135 cons = q->index;136 break;137 case QUEUE_TYPE_TO_CLIENT:138 /* used by rxe, client owns the index */139 cons = smp_load_acquire(&q->buf->consumer_index);140 break;141 case QUEUE_TYPE_FROM_ULP:142 /* used by ulp, rxe owns the index */143 cons = smp_load_acquire(&q->buf->consumer_index);144 break;145 case QUEUE_TYPE_TO_ULP:146 /* used by ulp which owns the index */147 cons = q->buf->consumer_index;148 break;149 }150 151 return cons;152}153 154static inline int queue_empty(struct rxe_queue *q, enum queue_type type)155{156 u32 prod = queue_get_producer(q, type);157 u32 cons = queue_get_consumer(q, type);158 159 return ((prod - cons) & q->index_mask) == 0;160}161 162static inline int queue_full(struct rxe_queue *q, enum queue_type type)163{164 u32 prod = queue_get_producer(q, type);165 u32 cons = queue_get_consumer(q, type);166 167 return ((prod + 1 - cons) & q->index_mask) == 0;168}169 170static inline u32 queue_count(const struct rxe_queue *q,171 enum queue_type type)172{173 u32 prod = queue_get_producer(q, type);174 u32 cons = queue_get_consumer(q, type);175 176 return (prod - cons) & q->index_mask;177}178 179static inline void queue_advance_producer(struct rxe_queue *q,180 enum queue_type type)181{182 u32 prod;183 184 switch (type) {185 case QUEUE_TYPE_FROM_CLIENT:186 /* used by rxe, client owns the index */187 if (WARN_ON(1))188 pr_warn("%s: attempt to advance client index\n",189 __func__);190 break;191 case QUEUE_TYPE_TO_CLIENT:192 /* used by rxe which owns the index */193 prod = q->index;194 prod = (prod + 1) & q->index_mask;195 q->index = prod;196 /* release so client can read it safely */197 smp_store_release(&q->buf->producer_index, prod);198 break;199 case QUEUE_TYPE_FROM_ULP:200 /* used by ulp which owns the index */201 prod = q->buf->producer_index;202 prod = (prod + 1) & q->index_mask;203 /* release so rxe can read it safely */204 smp_store_release(&q->buf->producer_index, prod);205 break;206 case QUEUE_TYPE_TO_ULP:207 /* used by ulp, rxe owns the index */208 if (WARN_ON(1))209 pr_warn("%s: attempt to advance driver index\n",210 __func__);211 break;212 }213}214 215static inline void queue_advance_consumer(struct rxe_queue *q,216 enum queue_type type)217{218 u32 cons;219 220 switch (type) {221 case QUEUE_TYPE_FROM_CLIENT:222 /* used by rxe which owns the index */223 cons = (q->index + 1) & q->index_mask;224 q->index = cons;225 /* release so client can read it safely */226 smp_store_release(&q->buf->consumer_index, cons);227 break;228 case QUEUE_TYPE_TO_CLIENT:229 /* used by rxe, client owns the index */230 if (WARN_ON(1))231 pr_warn("%s: attempt to advance client index\n",232 __func__);233 break;234 case QUEUE_TYPE_FROM_ULP:235 /* used by ulp, rxe owns the index */236 if (WARN_ON(1))237 pr_warn("%s: attempt to advance driver index\n",238 __func__);239 break;240 case QUEUE_TYPE_TO_ULP:241 /* used by ulp which owns the index */242 cons = q->buf->consumer_index;243 cons = (cons + 1) & q->index_mask;244 /* release so rxe can read it safely */245 smp_store_release(&q->buf->consumer_index, cons);246 break;247 }248}249 250static inline void *queue_producer_addr(struct rxe_queue *q,251 enum queue_type type)252{253 u32 prod = queue_get_producer(q, type);254 255 return q->buf->data + (prod << q->log2_elem_size);256}257 258static inline void *queue_consumer_addr(struct rxe_queue *q,259 enum queue_type type)260{261 u32 cons = queue_get_consumer(q, type);262 263 return q->buf->data + (cons << q->log2_elem_size);264}265 266static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)267{268 return q->buf->data + ((index & q->index_mask)269 << q->log2_elem_size);270}271 272static inline u32 queue_index_from_addr(const struct rxe_queue *q,273 const void *addr)274{275 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)276 & q->index_mask;277}278 279static inline void *queue_head(struct rxe_queue *q, enum queue_type type)280{281 return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);282}283 284#endif /* RXE_QUEUE_H */285