314 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.5 */6 7#include "rxe.h"8 9static struct workqueue_struct *rxe_wq;10 11int rxe_alloc_wq(void)12{13 rxe_wq = alloc_workqueue("rxe_wq", WQ_UNBOUND, WQ_MAX_ACTIVE);14 if (!rxe_wq)15 return -ENOMEM;16 17 return 0;18}19 20void rxe_destroy_wq(void)21{22 destroy_workqueue(rxe_wq);23}24 25/* Check if task is idle i.e. not running, not scheduled in26 * work queue and not draining. If so move to busy to27 * reserve a slot in do_task() by setting to busy and taking28 * a qp reference to cover the gap from now until the task finishes.29 * state will move out of busy if task returns a non zero value30 * in do_task(). If state is already busy it is raised to armed31 * to indicate to do_task that additional pass should be made32 * over the task.33 * Context: caller should hold task->lock.34 * Returns: true if state transitioned from idle to busy else false.35 */36static bool __reserve_if_idle(struct rxe_task *task)37{38 WARN_ON(rxe_read(task->qp) <= 0);39 40 if (task->state == TASK_STATE_IDLE) {41 rxe_get(task->qp);42 task->state = TASK_STATE_BUSY;43 task->num_sched++;44 return true;45 }46 47 if (task->state == TASK_STATE_BUSY)48 task->state = TASK_STATE_ARMED;49 50 return false;51}52 53/* check if task is idle or drained and not currently54 * scheduled in the work queue. This routine is55 * called by rxe_cleanup_task or rxe_disable_task to56 * see if the queue is empty.57 * Context: caller should hold task->lock.58 * Returns true if done else false.59 */60static bool __is_done(struct rxe_task *task)61{62 if (work_pending(&task->work))63 return false;64 65 if (task->state == TASK_STATE_IDLE ||66 task->state == TASK_STATE_DRAINED) {67 return true;68 }69 70 return false;71}72 73/* a locked version of __is_done */74static bool is_done(struct rxe_task *task)75{76 unsigned long flags;77 int done;78 79 spin_lock_irqsave(&task->lock, flags);80 done = __is_done(task);81 spin_unlock_irqrestore(&task->lock, flags);82 83 return done;84}85 86/* do_task is a wrapper for the three tasks (requester,87 * completer, responder) and calls them in a loop until88 * they return a non-zero value. It is called either89 * directly by rxe_run_task or indirectly if rxe_sched_task90 * schedules the task. They must call __reserve_if_idle to91 * move the task to busy before calling or scheduling.92 * The task can also be moved to drained or invalid93 * by calls to rxe_cleanup_task or rxe_disable_task.94 * In that case tasks which get here are not executed but95 * just flushed. The tasks are designed to look to see if96 * there is work to do and then do part of it before returning97 * here with a return value of zero until all the work98 * has been consumed then it returns a non-zero value.99 * The number of times the task can be run is limited by100 * max iterations so one task cannot hold the cpu forever.101 * If the limit is hit and work remains the task is rescheduled.102 */103static void do_task(struct rxe_task *task)104{105 unsigned int iterations;106 unsigned long flags;107 int resched = 0;108 int cont;109 int ret;110 111 WARN_ON(rxe_read(task->qp) <= 0);112 113 spin_lock_irqsave(&task->lock, flags);114 if (task->state >= TASK_STATE_DRAINED) {115 rxe_put(task->qp);116 task->num_done++;117 spin_unlock_irqrestore(&task->lock, flags);118 return;119 }120 spin_unlock_irqrestore(&task->lock, flags);121 122 do {123 iterations = RXE_MAX_ITERATIONS;124 cont = 0;125 126 do {127 ret = task->func(task->qp);128 } while (ret == 0 && iterations-- > 0);129 130 spin_lock_irqsave(&task->lock, flags);131 /* we're not done yet but we ran out of iterations.132 * yield the cpu and reschedule the task133 */134 if (!ret) {135 task->state = TASK_STATE_IDLE;136 resched = 1;137 goto exit;138 }139 140 switch (task->state) {141 case TASK_STATE_BUSY:142 task->state = TASK_STATE_IDLE;143 break;144 145 /* someone tried to schedule the task while we146 * were running, keep going147 */148 case TASK_STATE_ARMED:149 task->state = TASK_STATE_BUSY;150 cont = 1;151 break;152 153 case TASK_STATE_DRAINING:154 task->state = TASK_STATE_DRAINED;155 break;156 157 default:158 WARN_ON(1);159 rxe_dbg_qp(task->qp, "unexpected task state = %d\n",160 task->state);161 task->state = TASK_STATE_IDLE;162 }163 164exit:165 if (!cont) {166 task->num_done++;167 if (WARN_ON(task->num_done != task->num_sched))168 rxe_dbg_qp(169 task->qp,170 "%ld tasks scheduled, %ld tasks done\n",171 task->num_sched, task->num_done);172 }173 spin_unlock_irqrestore(&task->lock, flags);174 } while (cont);175 176 task->ret = ret;177 178 if (resched)179 rxe_sched_task(task);180 181 rxe_put(task->qp);182}183 184/* wrapper around do_task to fix argument for work queue */185static void do_work(struct work_struct *work)186{187 do_task(container_of(work, struct rxe_task, work));188}189 190int rxe_init_task(struct rxe_task *task, struct rxe_qp *qp,191 int (*func)(struct rxe_qp *))192{193 WARN_ON(rxe_read(qp) <= 0);194 195 task->qp = qp;196 task->func = func;197 task->state = TASK_STATE_IDLE;198 spin_lock_init(&task->lock);199 INIT_WORK(&task->work, do_work);200 201 return 0;202}203 204/* rxe_cleanup_task is only called from rxe_do_qp_cleanup in205 * process context. The qp is already completed with no206 * remaining references. Once the queue is drained the207 * task is moved to invalid and returns. The qp cleanup208 * code then calls the task functions directly without209 * using the task struct to drain any late arriving packets210 * or work requests.211 */212void rxe_cleanup_task(struct rxe_task *task)213{214 unsigned long flags;215 216 spin_lock_irqsave(&task->lock, flags);217 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {218 task->state = TASK_STATE_DRAINING;219 } else {220 task->state = TASK_STATE_INVALID;221 spin_unlock_irqrestore(&task->lock, flags);222 return;223 }224 spin_unlock_irqrestore(&task->lock, flags);225 226 /* now the task cannot be scheduled or run just wait227 * for the previously scheduled tasks to finish.228 */229 while (!is_done(task))230 cond_resched();231 232 spin_lock_irqsave(&task->lock, flags);233 task->state = TASK_STATE_INVALID;234 spin_unlock_irqrestore(&task->lock, flags);235}236 237/* run the task inline if it is currently idle238 * cannot call do_task holding the lock239 */240void rxe_run_task(struct rxe_task *task)241{242 unsigned long flags;243 bool run;244 245 WARN_ON(rxe_read(task->qp) <= 0);246 247 spin_lock_irqsave(&task->lock, flags);248 run = __reserve_if_idle(task);249 spin_unlock_irqrestore(&task->lock, flags);250 251 if (run)252 do_task(task);253}254 255/* schedule the task to run later as a work queue entry.256 * the queue_work call can be called holding257 * the lock.258 */259void rxe_sched_task(struct rxe_task *task)260{261 unsigned long flags;262 263 WARN_ON(rxe_read(task->qp) <= 0);264 265 spin_lock_irqsave(&task->lock, flags);266 if (__reserve_if_idle(task))267 queue_work(rxe_wq, &task->work);268 spin_unlock_irqrestore(&task->lock, flags);269}270 271/* rxe_disable/enable_task are only called from272 * rxe_modify_qp in process context. Task is moved273 * to the drained state by do_task.274 */275void rxe_disable_task(struct rxe_task *task)276{277 unsigned long flags;278 279 WARN_ON(rxe_read(task->qp) <= 0);280 281 spin_lock_irqsave(&task->lock, flags);282 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {283 task->state = TASK_STATE_DRAINING;284 } else {285 task->state = TASK_STATE_DRAINED;286 spin_unlock_irqrestore(&task->lock, flags);287 return;288 }289 spin_unlock_irqrestore(&task->lock, flags);290 291 while (!is_done(task))292 cond_resched();293 294 spin_lock_irqsave(&task->lock, flags);295 task->state = TASK_STATE_DRAINED;296 spin_unlock_irqrestore(&task->lock, flags);297}298 299void rxe_enable_task(struct rxe_task *task)300{301 unsigned long flags;302 303 WARN_ON(rxe_read(task->qp) <= 0);304 305 spin_lock_irqsave(&task->lock, flags);306 if (task->state == TASK_STATE_INVALID) {307 spin_unlock_irqrestore(&task->lock, flags);308 return;309 }310 311 task->state = TASK_STATE_IDLE;312 spin_unlock_irqrestore(&task->lock, flags);313}314