388 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 *4 * MMC software queue support based on command queue interfaces5 *6 * Copyright (C) 2019 Linaro, Inc.7 * Author: Baolin Wang <baolin.wang@linaro.org>8 */9 10#include <linux/mmc/card.h>11#include <linux/mmc/host.h>12#include <linux/module.h>13 14#include "mmc_hsq.h"15 16static void mmc_hsq_retry_handler(struct work_struct *work)17{18 struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work);19 struct mmc_host *mmc = hsq->mmc;20 21 mmc->ops->request(mmc, hsq->mrq);22}23 24static void mmc_hsq_modify_threshold(struct mmc_hsq *hsq)25{26 struct mmc_host *mmc = hsq->mmc;27 struct mmc_request *mrq;28 unsigned int tag, need_change = 0;29 30 mmc->hsq_depth = HSQ_NORMAL_DEPTH;31 for (tag = 0; tag < HSQ_NUM_SLOTS; tag++) {32 mrq = hsq->slot[tag].mrq;33 if (mrq && mrq->data &&34 (mrq->data->blksz * mrq->data->blocks == 4096) &&35 (mrq->data->flags & MMC_DATA_WRITE) &&36 (++need_change == 2)) {37 mmc->hsq_depth = HSQ_PERFORMANCE_DEPTH;38 break;39 }40 }41}42 43static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)44{45 struct mmc_host *mmc = hsq->mmc;46 struct hsq_slot *slot;47 unsigned long flags;48 int ret = 0;49 50 spin_lock_irqsave(&hsq->lock, flags);51 52 /* Make sure we are not already running a request now */53 if (hsq->mrq || hsq->recovery_halt) {54 spin_unlock_irqrestore(&hsq->lock, flags);55 return;56 }57 58 /* Make sure there are remain requests need to pump */59 if (!hsq->qcnt || !hsq->enabled) {60 spin_unlock_irqrestore(&hsq->lock, flags);61 return;62 }63 64 mmc_hsq_modify_threshold(hsq);65 66 slot = &hsq->slot[hsq->next_tag];67 hsq->mrq = slot->mrq;68 hsq->qcnt--;69 70 spin_unlock_irqrestore(&hsq->lock, flags);71 72 if (mmc->ops->request_atomic)73 ret = mmc->ops->request_atomic(mmc, hsq->mrq);74 else75 mmc->ops->request(mmc, hsq->mrq);76 77 /*78 * If returning BUSY from request_atomic(), which means the card79 * may be busy now, and we should change to non-atomic context to80 * try again for this unusual case, to avoid time-consuming operations81 * in the atomic context.82 *83 * Note: we just give a warning for other error cases, since the host84 * driver will handle them.85 */86 if (ret == -EBUSY)87 schedule_work(&hsq->retry_work);88 else89 WARN_ON_ONCE(ret);90}91 92static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains)93{94 int tag;95 96 /*97 * If there are no remain requests in software queue, then set a invalid98 * tag.99 */100 if (!remains) {101 hsq->next_tag = HSQ_INVALID_TAG;102 hsq->tail_tag = HSQ_INVALID_TAG;103 return;104 }105 106 tag = hsq->tag_slot[hsq->next_tag];107 hsq->tag_slot[hsq->next_tag] = HSQ_INVALID_TAG;108 hsq->next_tag = tag;109}110 111static void mmc_hsq_post_request(struct mmc_hsq *hsq)112{113 unsigned long flags;114 int remains;115 116 spin_lock_irqsave(&hsq->lock, flags);117 118 remains = hsq->qcnt;119 hsq->mrq = NULL;120 121 /* Update the next available tag to be queued. */122 mmc_hsq_update_next_tag(hsq, remains);123 124 if (hsq->waiting_for_idle && !remains) {125 hsq->waiting_for_idle = false;126 wake_up(&hsq->wait_queue);127 }128 129 /* Do not pump new request in recovery mode. */130 if (hsq->recovery_halt) {131 spin_unlock_irqrestore(&hsq->lock, flags);132 return;133 }134 135 spin_unlock_irqrestore(&hsq->lock, flags);136 137 /*138 * Try to pump new request to host controller as fast as possible,139 * after completing previous request.140 */141 if (remains > 0)142 mmc_hsq_pump_requests(hsq);143}144 145/**146 * mmc_hsq_finalize_request - finalize one request if the request is done147 * @mmc: the host controller148 * @mrq: the request need to be finalized149 *150 * Return true if we finalized the corresponding request in software queue,151 * otherwise return false.152 */153bool mmc_hsq_finalize_request(struct mmc_host *mmc, struct mmc_request *mrq)154{155 struct mmc_hsq *hsq = mmc->cqe_private;156 unsigned long flags;157 158 spin_lock_irqsave(&hsq->lock, flags);159 160 if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) {161 spin_unlock_irqrestore(&hsq->lock, flags);162 return false;163 }164 165 /*166 * Clear current completed slot request to make a room for new request.167 */168 hsq->slot[hsq->next_tag].mrq = NULL;169 170 spin_unlock_irqrestore(&hsq->lock, flags);171 172 mmc_cqe_request_done(mmc, hsq->mrq);173 174 mmc_hsq_post_request(hsq);175 176 return true;177}178EXPORT_SYMBOL_GPL(mmc_hsq_finalize_request);179 180static void mmc_hsq_recovery_start(struct mmc_host *mmc)181{182 struct mmc_hsq *hsq = mmc->cqe_private;183 unsigned long flags;184 185 spin_lock_irqsave(&hsq->lock, flags);186 187 hsq->recovery_halt = true;188 189 spin_unlock_irqrestore(&hsq->lock, flags);190}191 192static void mmc_hsq_recovery_finish(struct mmc_host *mmc)193{194 struct mmc_hsq *hsq = mmc->cqe_private;195 int remains;196 197 spin_lock_irq(&hsq->lock);198 199 hsq->recovery_halt = false;200 remains = hsq->qcnt;201 202 spin_unlock_irq(&hsq->lock);203 204 /*205 * Try to pump new request if there are request pending in software206 * queue after finishing recovery.207 */208 if (remains > 0)209 mmc_hsq_pump_requests(hsq);210}211 212static int mmc_hsq_request(struct mmc_host *mmc, struct mmc_request *mrq)213{214 struct mmc_hsq *hsq = mmc->cqe_private;215 int tag = mrq->tag;216 217 spin_lock_irq(&hsq->lock);218 219 if (!hsq->enabled) {220 spin_unlock_irq(&hsq->lock);221 return -ESHUTDOWN;222 }223 224 /* Do not queue any new requests in recovery mode. */225 if (hsq->recovery_halt) {226 spin_unlock_irq(&hsq->lock);227 return -EBUSY;228 }229 230 hsq->slot[tag].mrq = mrq;231 232 /*233 * Set the next tag as current request tag if no available234 * next tag.235 */236 if (hsq->next_tag == HSQ_INVALID_TAG) {237 hsq->next_tag = tag;238 hsq->tail_tag = tag;239 hsq->tag_slot[hsq->tail_tag] = HSQ_INVALID_TAG;240 } else {241 hsq->tag_slot[hsq->tail_tag] = tag;242 hsq->tail_tag = tag;243 }244 245 hsq->qcnt++;246 247 spin_unlock_irq(&hsq->lock);248 249 mmc_hsq_pump_requests(hsq);250 251 return 0;252}253 254static void mmc_hsq_post_req(struct mmc_host *mmc, struct mmc_request *mrq)255{256 if (mmc->ops->post_req)257 mmc->ops->post_req(mmc, mrq, 0);258}259 260static bool mmc_hsq_queue_is_idle(struct mmc_hsq *hsq, int *ret)261{262 bool is_idle;263 264 spin_lock_irq(&hsq->lock);265 266 is_idle = (!hsq->mrq && !hsq->qcnt) ||267 hsq->recovery_halt;268 269 *ret = hsq->recovery_halt ? -EBUSY : 0;270 hsq->waiting_for_idle = !is_idle;271 272 spin_unlock_irq(&hsq->lock);273 274 return is_idle;275}276 277static int mmc_hsq_wait_for_idle(struct mmc_host *mmc)278{279 struct mmc_hsq *hsq = mmc->cqe_private;280 int ret;281 282 wait_event(hsq->wait_queue,283 mmc_hsq_queue_is_idle(hsq, &ret));284 285 return ret;286}287 288static void mmc_hsq_disable(struct mmc_host *mmc)289{290 struct mmc_hsq *hsq = mmc->cqe_private;291 u32 timeout = 500;292 int ret;293 294 spin_lock_irq(&hsq->lock);295 296 if (!hsq->enabled) {297 spin_unlock_irq(&hsq->lock);298 return;299 }300 301 spin_unlock_irq(&hsq->lock);302 303 ret = wait_event_timeout(hsq->wait_queue,304 mmc_hsq_queue_is_idle(hsq, &ret),305 msecs_to_jiffies(timeout));306 if (ret == 0) {307 pr_warn("could not stop mmc software queue\n");308 return;309 }310 311 spin_lock_irq(&hsq->lock);312 313 hsq->enabled = false;314 315 spin_unlock_irq(&hsq->lock);316}317 318static int mmc_hsq_enable(struct mmc_host *mmc, struct mmc_card *card)319{320 struct mmc_hsq *hsq = mmc->cqe_private;321 322 spin_lock_irq(&hsq->lock);323 324 if (hsq->enabled) {325 spin_unlock_irq(&hsq->lock);326 return -EBUSY;327 }328 329 hsq->enabled = true;330 331 spin_unlock_irq(&hsq->lock);332 333 return 0;334}335 336static const struct mmc_cqe_ops mmc_hsq_ops = {337 .cqe_enable = mmc_hsq_enable,338 .cqe_disable = mmc_hsq_disable,339 .cqe_request = mmc_hsq_request,340 .cqe_post_req = mmc_hsq_post_req,341 .cqe_wait_for_idle = mmc_hsq_wait_for_idle,342 .cqe_recovery_start = mmc_hsq_recovery_start,343 .cqe_recovery_finish = mmc_hsq_recovery_finish,344};345 346int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc)347{348 int i;349 hsq->num_slots = HSQ_NUM_SLOTS;350 hsq->next_tag = HSQ_INVALID_TAG;351 hsq->tail_tag = HSQ_INVALID_TAG;352 353 hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,354 sizeof(struct hsq_slot), GFP_KERNEL);355 if (!hsq->slot)356 return -ENOMEM;357 358 hsq->mmc = mmc;359 hsq->mmc->cqe_private = hsq;360 mmc->cqe_ops = &mmc_hsq_ops;361 mmc->hsq_depth = HSQ_NORMAL_DEPTH;362 363 for (i = 0; i < HSQ_NUM_SLOTS; i++)364 hsq->tag_slot[i] = HSQ_INVALID_TAG;365 366 INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);367 spin_lock_init(&hsq->lock);368 init_waitqueue_head(&hsq->wait_queue);369 370 return 0;371}372EXPORT_SYMBOL_GPL(mmc_hsq_init);373 374void mmc_hsq_suspend(struct mmc_host *mmc)375{376 mmc_hsq_disable(mmc);377}378EXPORT_SYMBOL_GPL(mmc_hsq_suspend);379 380int mmc_hsq_resume(struct mmc_host *mmc)381{382 return mmc_hsq_enable(mmc, NULL);383}384EXPORT_SYMBOL_GPL(mmc_hsq_resume);385 386MODULE_DESCRIPTION("MMC Host Software Queue support");387MODULE_LICENSE("GPL v2");388