879 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Support for Macronix external hardware ECC engine for NAND devices, also4 * called DPE for Data Processing Engine.5 *6 * Copyright © 2019 Macronix7 * Author: Miquel Raynal <miquel.raynal@bootlin.com>8 */9 10#include <linux/dma-mapping.h>11#include <linux/init.h>12#include <linux/interrupt.h>13#include <linux/io.h>14#include <linux/iopoll.h>15#include <linux/kernel.h>16#include <linux/module.h>17#include <linux/mtd/mtd.h>18#include <linux/mtd/nand.h>19#include <linux/mtd/nand-ecc-mxic.h>20#include <linux/mutex.h>21#include <linux/of.h>22#include <linux/of_platform.h>23#include <linux/platform_device.h>24#include <linux/slab.h>25 26/* DPE Configuration */27#define DP_CONFIG 0x0028#define ECC_EN BIT(0)29#define ECC_TYP(idx) (((idx) << 3) & GENMASK(6, 3))30/* DPE Interrupt Status */31#define INTRPT_STS 0x0432#define TRANS_CMPLT BIT(0)33#define SDMA_MAIN BIT(1)34#define SDMA_SPARE BIT(2)35#define ECC_ERR BIT(3)36#define TO_SPARE BIT(4)37#define TO_MAIN BIT(5)38/* DPE Interrupt Status Enable */39#define INTRPT_STS_EN 0x0840/* DPE Interrupt Signal Enable */41#define INTRPT_SIG_EN 0x0C42/* Host Controller Configuration */43#define HC_CONFIG 0x1044#define DEV2MEM 0 /* TRANS_TYP_DMA in the spec */45#define MEM2MEM BIT(4) /* TRANS_TYP_IO in the spec */46#define MAPPING BIT(5) /* TRANS_TYP_MAPPING in the spec */47#define ECC_PACKED 0 /* LAYOUT_TYP_INTEGRATED in the spec */48#define ECC_INTERLEAVED BIT(2) /* LAYOUT_TYP_DISTRIBUTED in the spec */49#define BURST_TYP_FIXED 050#define BURST_TYP_INCREASING BIT(0)51/* Host Controller Slave Address */52#define HC_SLV_ADDR 0x1453/* ECC Chunk Size */54#define CHUNK_SIZE 0x2055/* Main Data Size */56#define MAIN_SIZE 0x2457/* Spare Data Size */58#define SPARE_SIZE 0x2859#define META_SZ(reg) ((reg) & GENMASK(7, 0))60#define PARITY_SZ(reg) (((reg) & GENMASK(15, 8)) >> 8)61#define RSV_SZ(reg) (((reg) & GENMASK(23, 16)) >> 16)62#define SPARE_SZ(reg) ((reg) >> 24)63/* ECC Chunk Count */64#define CHUNK_CNT 0x3065/* SDMA Control */66#define SDMA_CTRL 0x4067#define WRITE_NAND 068#define READ_NAND BIT(1)69#define CONT_NAND BIT(29)70#define CONT_SYSM BIT(30) /* Continue System Memory? */71#define SDMA_STRT BIT(31)72/* SDMA Address of Main Data */73#define SDMA_MAIN_ADDR 0x4474/* SDMA Address of Spare Data */75#define SDMA_SPARE_ADDR 0x4876/* DPE Version Number */77#define DP_VER 0xD078#define DP_VER_OFFSET 1679 80/* Status bytes between each chunk of spare data */81#define STAT_BYTES 482#define NO_ERR 0x0083#define MAX_CORR_ERR 0x2884#define UNCORR_ERR 0xFE85#define ERASED_CHUNK 0xFF86 87struct mxic_ecc_engine {88 struct device *dev;89 void __iomem *regs;90 int irq;91 struct completion complete;92 struct nand_ecc_engine external_engine;93 struct nand_ecc_engine pipelined_engine;94 struct mutex lock;95};96 97struct mxic_ecc_ctx {98 /* ECC machinery */99 unsigned int data_step_sz;100 unsigned int oob_step_sz;101 unsigned int parity_sz;102 unsigned int meta_sz;103 u8 *status;104 int steps;105 106 /* DMA boilerplate */107 struct nand_ecc_req_tweak_ctx req_ctx;108 u8 *oobwithstat;109 struct scatterlist sg[2];110 struct nand_page_io_req *req;111 unsigned int pageoffs;112};113 114static struct mxic_ecc_engine *ext_ecc_eng_to_mxic(struct nand_ecc_engine *eng)115{116 return container_of(eng, struct mxic_ecc_engine, external_engine);117}118 119static struct mxic_ecc_engine *pip_ecc_eng_to_mxic(struct nand_ecc_engine *eng)120{121 return container_of(eng, struct mxic_ecc_engine, pipelined_engine);122}123 124static struct mxic_ecc_engine *nand_to_mxic(struct nand_device *nand)125{126 struct nand_ecc_engine *eng = nand->ecc.engine;127 128 if (eng->integration == NAND_ECC_ENGINE_INTEGRATION_EXTERNAL)129 return ext_ecc_eng_to_mxic(eng);130 else131 return pip_ecc_eng_to_mxic(eng);132}133 134static int mxic_ecc_ooblayout_ecc(struct mtd_info *mtd, int section,135 struct mtd_oob_region *oobregion)136{137 struct nand_device *nand = mtd_to_nanddev(mtd);138 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);139 140 if (section < 0 || section >= ctx->steps)141 return -ERANGE;142 143 oobregion->offset = (section * ctx->oob_step_sz) + ctx->meta_sz;144 oobregion->length = ctx->parity_sz;145 146 return 0;147}148 149static int mxic_ecc_ooblayout_free(struct mtd_info *mtd, int section,150 struct mtd_oob_region *oobregion)151{152 struct nand_device *nand = mtd_to_nanddev(mtd);153 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);154 155 if (section < 0 || section >= ctx->steps)156 return -ERANGE;157 158 if (!section) {159 oobregion->offset = 2;160 oobregion->length = ctx->meta_sz - 2;161 } else {162 oobregion->offset = section * ctx->oob_step_sz;163 oobregion->length = ctx->meta_sz;164 }165 166 return 0;167}168 169static const struct mtd_ooblayout_ops mxic_ecc_ooblayout_ops = {170 .ecc = mxic_ecc_ooblayout_ecc,171 .free = mxic_ecc_ooblayout_free,172};173 174static void mxic_ecc_disable_engine(struct mxic_ecc_engine *mxic)175{176 u32 reg;177 178 reg = readl(mxic->regs + DP_CONFIG);179 reg &= ~ECC_EN;180 writel(reg, mxic->regs + DP_CONFIG);181}182 183static void mxic_ecc_enable_engine(struct mxic_ecc_engine *mxic)184{185 u32 reg;186 187 reg = readl(mxic->regs + DP_CONFIG);188 reg |= ECC_EN;189 writel(reg, mxic->regs + DP_CONFIG);190}191 192static void mxic_ecc_disable_int(struct mxic_ecc_engine *mxic)193{194 writel(0, mxic->regs + INTRPT_SIG_EN);195}196 197static void mxic_ecc_enable_int(struct mxic_ecc_engine *mxic)198{199 writel(TRANS_CMPLT, mxic->regs + INTRPT_SIG_EN);200}201 202static irqreturn_t mxic_ecc_isr(int irq, void *dev_id)203{204 struct mxic_ecc_engine *mxic = dev_id;205 u32 sts;206 207 sts = readl(mxic->regs + INTRPT_STS);208 if (!sts)209 return IRQ_NONE;210 211 if (sts & TRANS_CMPLT)212 complete(&mxic->complete);213 214 writel(sts, mxic->regs + INTRPT_STS);215 216 return IRQ_HANDLED;217}218 219static int mxic_ecc_init_ctx(struct nand_device *nand, struct device *dev)220{221 struct mxic_ecc_engine *mxic = nand_to_mxic(nand);222 struct nand_ecc_props *conf = &nand->ecc.ctx.conf;223 struct nand_ecc_props *reqs = &nand->ecc.requirements;224 struct nand_ecc_props *user = &nand->ecc.user_conf;225 struct mtd_info *mtd = nanddev_to_mtd(nand);226 int step_size = 0, strength = 0, desired_correction = 0, steps, idx;227 static const int possible_strength[] = {4, 8, 40, 48};228 static const int spare_size[] = {32, 32, 96, 96};229 struct mxic_ecc_ctx *ctx;230 u32 spare_reg;231 int ret;232 233 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);234 if (!ctx)235 return -ENOMEM;236 237 nand->ecc.ctx.priv = ctx;238 239 /* Only large page NAND chips may use BCH */240 if (mtd->oobsize < 64) {241 pr_err("BCH cannot be used with small page NAND chips\n");242 return -EINVAL;243 }244 245 mtd_set_ooblayout(mtd, &mxic_ecc_ooblayout_ops);246 247 /* Enable all status bits */248 writel(TRANS_CMPLT | SDMA_MAIN | SDMA_SPARE | ECC_ERR |249 TO_SPARE | TO_MAIN, mxic->regs + INTRPT_STS_EN);250 251 /* Configure the correction depending on the NAND device topology */252 if (user->step_size && user->strength) {253 step_size = user->step_size;254 strength = user->strength;255 } else if (reqs->step_size && reqs->strength) {256 step_size = reqs->step_size;257 strength = reqs->strength;258 }259 260 if (step_size && strength) {261 steps = mtd->writesize / step_size;262 desired_correction = steps * strength;263 }264 265 /* Step size is fixed to 1kiB, strength may vary (4 possible values) */266 conf->step_size = SZ_1K;267 steps = mtd->writesize / conf->step_size;268 269 ctx->status = devm_kzalloc(dev, steps * sizeof(u8), GFP_KERNEL);270 if (!ctx->status)271 return -ENOMEM;272 273 if (desired_correction) {274 strength = desired_correction / steps;275 276 for (idx = 0; idx < ARRAY_SIZE(possible_strength); idx++)277 if (possible_strength[idx] >= strength)278 break;279 280 idx = min_t(unsigned int, idx,281 ARRAY_SIZE(possible_strength) - 1);282 } else {283 /* Missing data, maximize the correction */284 idx = ARRAY_SIZE(possible_strength) - 1;285 }286 287 /* Tune the selected strength until it fits in the OOB area */288 for (; idx >= 0; idx--) {289 if (spare_size[idx] * steps <= mtd->oobsize)290 break;291 }292 293 /* This engine cannot be used with this NAND device */294 if (idx < 0)295 return -EINVAL;296 297 /* Configure the engine for the desired strength */298 writel(ECC_TYP(idx), mxic->regs + DP_CONFIG);299 conf->strength = possible_strength[idx];300 spare_reg = readl(mxic->regs + SPARE_SIZE);301 302 ctx->steps = steps;303 ctx->data_step_sz = mtd->writesize / steps;304 ctx->oob_step_sz = mtd->oobsize / steps;305 ctx->parity_sz = PARITY_SZ(spare_reg);306 ctx->meta_sz = META_SZ(spare_reg);307 308 /* Ensure buffers will contain enough bytes to store the STAT_BYTES */309 ctx->req_ctx.oob_buffer_size = nanddev_per_page_oobsize(nand) +310 (ctx->steps * STAT_BYTES);311 ret = nand_ecc_init_req_tweaking(&ctx->req_ctx, nand);312 if (ret)313 return ret;314 315 ctx->oobwithstat = kmalloc(mtd->oobsize + (ctx->steps * STAT_BYTES),316 GFP_KERNEL);317 if (!ctx->oobwithstat) {318 ret = -ENOMEM;319 goto cleanup_req_tweak;320 }321 322 sg_init_table(ctx->sg, 2);323 324 /* Configuration dump and sanity checks */325 dev_err(dev, "DPE version number: %d\n",326 readl(mxic->regs + DP_VER) >> DP_VER_OFFSET);327 dev_err(dev, "Chunk size: %d\n", readl(mxic->regs + CHUNK_SIZE));328 dev_err(dev, "Main size: %d\n", readl(mxic->regs + MAIN_SIZE));329 dev_err(dev, "Spare size: %d\n", SPARE_SZ(spare_reg));330 dev_err(dev, "Rsv size: %ld\n", RSV_SZ(spare_reg));331 dev_err(dev, "Parity size: %d\n", ctx->parity_sz);332 dev_err(dev, "Meta size: %d\n", ctx->meta_sz);333 334 if ((ctx->meta_sz + ctx->parity_sz + RSV_SZ(spare_reg)) !=335 SPARE_SZ(spare_reg)) {336 dev_err(dev, "Wrong OOB configuration: %d + %d + %ld != %d\n",337 ctx->meta_sz, ctx->parity_sz, RSV_SZ(spare_reg),338 SPARE_SZ(spare_reg));339 ret = -EINVAL;340 goto free_oobwithstat;341 }342 343 if (ctx->oob_step_sz != SPARE_SZ(spare_reg)) {344 dev_err(dev, "Wrong OOB configuration: %d != %d\n",345 ctx->oob_step_sz, SPARE_SZ(spare_reg));346 ret = -EINVAL;347 goto free_oobwithstat;348 }349 350 return 0;351 352free_oobwithstat:353 kfree(ctx->oobwithstat);354cleanup_req_tweak:355 nand_ecc_cleanup_req_tweaking(&ctx->req_ctx);356 357 return ret;358}359 360static int mxic_ecc_init_ctx_external(struct nand_device *nand)361{362 struct mxic_ecc_engine *mxic = nand_to_mxic(nand);363 struct device *dev = nand->ecc.engine->dev;364 int ret;365 366 dev_info(dev, "Macronix ECC engine in external mode\n");367 368 ret = mxic_ecc_init_ctx(nand, dev);369 if (ret)370 return ret;371 372 /* Trigger each step manually */373 writel(1, mxic->regs + CHUNK_CNT);374 writel(BURST_TYP_INCREASING | ECC_PACKED | MEM2MEM,375 mxic->regs + HC_CONFIG);376 377 return 0;378}379 380static int mxic_ecc_init_ctx_pipelined(struct nand_device *nand)381{382 struct mxic_ecc_engine *mxic = nand_to_mxic(nand);383 struct mxic_ecc_ctx *ctx;384 struct device *dev;385 int ret;386 387 dev = nand_ecc_get_engine_dev(nand->ecc.engine->dev);388 if (!dev)389 return -EINVAL;390 391 dev_info(dev, "Macronix ECC engine in pipelined/mapping mode\n");392 393 ret = mxic_ecc_init_ctx(nand, dev);394 if (ret)395 return ret;396 397 ctx = nand_to_ecc_ctx(nand);398 399 /* All steps should be handled in one go directly by the internal DMA */400 writel(ctx->steps, mxic->regs + CHUNK_CNT);401 402 /*403 * Interleaved ECC scheme cannot be used otherwise factory bad block404 * markers would be lost. A packed layout is mandatory.405 */406 writel(BURST_TYP_INCREASING | ECC_PACKED | MAPPING,407 mxic->regs + HC_CONFIG);408 409 return 0;410}411 412static void mxic_ecc_cleanup_ctx(struct nand_device *nand)413{414 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);415 416 if (ctx) {417 nand_ecc_cleanup_req_tweaking(&ctx->req_ctx);418 kfree(ctx->oobwithstat);419 }420}421 422static int mxic_ecc_data_xfer_wait_for_completion(struct mxic_ecc_engine *mxic)423{424 u32 val;425 int ret;426 427 if (mxic->irq) {428 reinit_completion(&mxic->complete);429 mxic_ecc_enable_int(mxic);430 ret = wait_for_completion_timeout(&mxic->complete,431 msecs_to_jiffies(1000));432 ret = ret ? 0 : -ETIMEDOUT;433 mxic_ecc_disable_int(mxic);434 } else {435 ret = readl_poll_timeout(mxic->regs + INTRPT_STS, val,436 val & TRANS_CMPLT, 10, USEC_PER_SEC);437 writel(val, mxic->regs + INTRPT_STS);438 }439 440 if (ret) {441 dev_err(mxic->dev, "Timeout on data xfer completion\n");442 return -ETIMEDOUT;443 }444 445 return 0;446}447 448static int mxic_ecc_process_data(struct mxic_ecc_engine *mxic,449 unsigned int direction)450{451 unsigned int dir = (direction == NAND_PAGE_READ) ?452 READ_NAND : WRITE_NAND;453 int ret;454 455 mxic_ecc_enable_engine(mxic);456 457 /* Trigger processing */458 writel(SDMA_STRT | dir, mxic->regs + SDMA_CTRL);459 460 /* Wait for completion */461 ret = mxic_ecc_data_xfer_wait_for_completion(mxic);462 463 mxic_ecc_disable_engine(mxic);464 465 return ret;466}467 468int mxic_ecc_process_data_pipelined(struct nand_ecc_engine *eng,469 unsigned int direction, dma_addr_t dirmap)470{471 struct mxic_ecc_engine *mxic = pip_ecc_eng_to_mxic(eng);472 473 if (dirmap)474 writel(dirmap, mxic->regs + HC_SLV_ADDR);475 476 return mxic_ecc_process_data(mxic, direction);477}478EXPORT_SYMBOL_GPL(mxic_ecc_process_data_pipelined);479 480static void mxic_ecc_extract_status_bytes(struct mxic_ecc_ctx *ctx)481{482 u8 *buf = ctx->oobwithstat;483 int next_stat_pos;484 int step;485 486 /* Extract the ECC status */487 for (step = 0; step < ctx->steps; step++) {488 next_stat_pos = ctx->oob_step_sz +489 ((STAT_BYTES + ctx->oob_step_sz) * step);490 491 ctx->status[step] = buf[next_stat_pos];492 }493}494 495static void mxic_ecc_reconstruct_oobbuf(struct mxic_ecc_ctx *ctx,496 u8 *dst, const u8 *src)497{498 int step;499 500 /* Reconstruct the OOB buffer linearly (without the ECC status bytes) */501 for (step = 0; step < ctx->steps; step++)502 memcpy(dst + (step * ctx->oob_step_sz),503 src + (step * (ctx->oob_step_sz + STAT_BYTES)),504 ctx->oob_step_sz);505}506 507static void mxic_ecc_add_room_in_oobbuf(struct mxic_ecc_ctx *ctx,508 u8 *dst, const u8 *src)509{510 int step;511 512 /* Add some space in the OOB buffer for the status bytes */513 for (step = 0; step < ctx->steps; step++)514 memcpy(dst + (step * (ctx->oob_step_sz + STAT_BYTES)),515 src + (step * ctx->oob_step_sz),516 ctx->oob_step_sz);517}518 519static int mxic_ecc_count_biterrs(struct mxic_ecc_engine *mxic,520 struct nand_device *nand)521{522 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);523 struct mtd_info *mtd = nanddev_to_mtd(nand);524 struct device *dev = mxic->dev;525 unsigned int max_bf = 0;526 bool failure = false;527 int step;528 529 for (step = 0; step < ctx->steps; step++) {530 u8 stat = ctx->status[step];531 532 if (stat == NO_ERR) {533 dev_dbg(dev, "ECC step %d: no error\n", step);534 } else if (stat == ERASED_CHUNK) {535 dev_dbg(dev, "ECC step %d: erased\n", step);536 } else if (stat == UNCORR_ERR || stat > MAX_CORR_ERR) {537 dev_dbg(dev, "ECC step %d: uncorrectable\n", step);538 mtd->ecc_stats.failed++;539 failure = true;540 } else {541 dev_dbg(dev, "ECC step %d: %d bits corrected\n",542 step, stat);543 max_bf = max_t(unsigned int, max_bf, stat);544 mtd->ecc_stats.corrected += stat;545 }546 }547 548 return failure ? -EBADMSG : max_bf;549}550 551/* External ECC engine helpers */552static int mxic_ecc_prepare_io_req_external(struct nand_device *nand,553 struct nand_page_io_req *req)554{555 struct mxic_ecc_engine *mxic = nand_to_mxic(nand);556 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);557 struct mtd_info *mtd = nanddev_to_mtd(nand);558 int offset, nents, step, ret;559 560 if (req->mode == MTD_OPS_RAW)561 return 0;562 563 nand_ecc_tweak_req(&ctx->req_ctx, req);564 ctx->req = req;565 566 if (req->type == NAND_PAGE_READ)567 return 0;568 569 mxic_ecc_add_room_in_oobbuf(ctx, ctx->oobwithstat,570 ctx->req->oobbuf.out);571 572 sg_set_buf(&ctx->sg[0], req->databuf.out, req->datalen);573 sg_set_buf(&ctx->sg[1], ctx->oobwithstat,574 req->ooblen + (ctx->steps * STAT_BYTES));575 576 nents = dma_map_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL);577 if (!nents)578 return -EINVAL;579 580 mutex_lock(&mxic->lock);581 582 for (step = 0; step < ctx->steps; step++) {583 writel(sg_dma_address(&ctx->sg[0]) + (step * ctx->data_step_sz),584 mxic->regs + SDMA_MAIN_ADDR);585 writel(sg_dma_address(&ctx->sg[1]) + (step * (ctx->oob_step_sz + STAT_BYTES)),586 mxic->regs + SDMA_SPARE_ADDR);587 ret = mxic_ecc_process_data(mxic, ctx->req->type);588 if (ret)589 break;590 }591 592 mutex_unlock(&mxic->lock);593 594 dma_unmap_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL);595 596 if (ret)597 return ret;598 599 /* Retrieve the calculated ECC bytes */600 for (step = 0; step < ctx->steps; step++) {601 offset = ctx->meta_sz + (step * ctx->oob_step_sz);602 mtd_ooblayout_get_eccbytes(mtd,603 (u8 *)ctx->req->oobbuf.out + offset,604 ctx->oobwithstat + (step * STAT_BYTES),605 step * ctx->parity_sz,606 ctx->parity_sz);607 }608 609 return 0;610}611 612static int mxic_ecc_finish_io_req_external(struct nand_device *nand,613 struct nand_page_io_req *req)614{615 struct mxic_ecc_engine *mxic = nand_to_mxic(nand);616 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);617 int nents, step, ret;618 619 if (req->mode == MTD_OPS_RAW)620 return 0;621 622 if (req->type == NAND_PAGE_WRITE) {623 nand_ecc_restore_req(&ctx->req_ctx, req);624 return 0;625 }626 627 /* Copy the OOB buffer and add room for the ECC engine status bytes */628 mxic_ecc_add_room_in_oobbuf(ctx, ctx->oobwithstat, ctx->req->oobbuf.in);629 630 sg_set_buf(&ctx->sg[0], req->databuf.in, req->datalen);631 sg_set_buf(&ctx->sg[1], ctx->oobwithstat,632 req->ooblen + (ctx->steps * STAT_BYTES));633 nents = dma_map_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL);634 if (!nents)635 return -EINVAL;636 637 mutex_lock(&mxic->lock);638 639 for (step = 0; step < ctx->steps; step++) {640 writel(sg_dma_address(&ctx->sg[0]) + (step * ctx->data_step_sz),641 mxic->regs + SDMA_MAIN_ADDR);642 writel(sg_dma_address(&ctx->sg[1]) + (step * (ctx->oob_step_sz + STAT_BYTES)),643 mxic->regs + SDMA_SPARE_ADDR);644 ret = mxic_ecc_process_data(mxic, ctx->req->type);645 if (ret)646 break;647 }648 649 mutex_unlock(&mxic->lock);650 651 dma_unmap_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL);652 653 if (ret) {654 nand_ecc_restore_req(&ctx->req_ctx, req);655 return ret;656 }657 658 /* Extract the status bytes and reconstruct the buffer */659 mxic_ecc_extract_status_bytes(ctx);660 mxic_ecc_reconstruct_oobbuf(ctx, ctx->req->oobbuf.in, ctx->oobwithstat);661 662 nand_ecc_restore_req(&ctx->req_ctx, req);663 664 return mxic_ecc_count_biterrs(mxic, nand);665}666 667/* Pipelined ECC engine helpers */668static int mxic_ecc_prepare_io_req_pipelined(struct nand_device *nand,669 struct nand_page_io_req *req)670{671 struct mxic_ecc_engine *mxic = nand_to_mxic(nand);672 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);673 int nents;674 675 if (req->mode == MTD_OPS_RAW)676 return 0;677 678 nand_ecc_tweak_req(&ctx->req_ctx, req);679 ctx->req = req;680 681 /* Copy the OOB buffer and add room for the ECC engine status bytes */682 mxic_ecc_add_room_in_oobbuf(ctx, ctx->oobwithstat, ctx->req->oobbuf.in);683 684 sg_set_buf(&ctx->sg[0], req->databuf.in, req->datalen);685 sg_set_buf(&ctx->sg[1], ctx->oobwithstat,686 req->ooblen + (ctx->steps * STAT_BYTES));687 688 nents = dma_map_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL);689 if (!nents)690 return -EINVAL;691 692 mutex_lock(&mxic->lock);693 694 writel(sg_dma_address(&ctx->sg[0]), mxic->regs + SDMA_MAIN_ADDR);695 writel(sg_dma_address(&ctx->sg[1]), mxic->regs + SDMA_SPARE_ADDR);696 697 return 0;698}699 700static int mxic_ecc_finish_io_req_pipelined(struct nand_device *nand,701 struct nand_page_io_req *req)702{703 struct mxic_ecc_engine *mxic = nand_to_mxic(nand);704 struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand);705 int ret = 0;706 707 if (req->mode == MTD_OPS_RAW)708 return 0;709 710 mutex_unlock(&mxic->lock);711 712 dma_unmap_sg(mxic->dev, ctx->sg, 2, DMA_BIDIRECTIONAL);713 714 if (req->type == NAND_PAGE_READ) {715 mxic_ecc_extract_status_bytes(ctx);716 mxic_ecc_reconstruct_oobbuf(ctx, ctx->req->oobbuf.in,717 ctx->oobwithstat);718 ret = mxic_ecc_count_biterrs(mxic, nand);719 }720 721 nand_ecc_restore_req(&ctx->req_ctx, req);722 723 return ret;724}725 726static struct nand_ecc_engine_ops mxic_ecc_engine_external_ops = {727 .init_ctx = mxic_ecc_init_ctx_external,728 .cleanup_ctx = mxic_ecc_cleanup_ctx,729 .prepare_io_req = mxic_ecc_prepare_io_req_external,730 .finish_io_req = mxic_ecc_finish_io_req_external,731};732 733static struct nand_ecc_engine_ops mxic_ecc_engine_pipelined_ops = {734 .init_ctx = mxic_ecc_init_ctx_pipelined,735 .cleanup_ctx = mxic_ecc_cleanup_ctx,736 .prepare_io_req = mxic_ecc_prepare_io_req_pipelined,737 .finish_io_req = mxic_ecc_finish_io_req_pipelined,738};739 740struct nand_ecc_engine_ops *mxic_ecc_get_pipelined_ops(void)741{742 return &mxic_ecc_engine_pipelined_ops;743}744EXPORT_SYMBOL_GPL(mxic_ecc_get_pipelined_ops);745 746static struct platform_device *747mxic_ecc_get_pdev(struct platform_device *spi_pdev)748{749 struct platform_device *eng_pdev;750 struct device_node *np;751 752 /* Retrieve the nand-ecc-engine phandle */753 np = of_parse_phandle(spi_pdev->dev.of_node, "nand-ecc-engine", 0);754 if (!np)755 return NULL;756 757 /* Jump to the engine's device node */758 eng_pdev = of_find_device_by_node(np);759 of_node_put(np);760 761 return eng_pdev;762}763 764void mxic_ecc_put_pipelined_engine(struct nand_ecc_engine *eng)765{766 struct mxic_ecc_engine *mxic = pip_ecc_eng_to_mxic(eng);767 768 platform_device_put(to_platform_device(mxic->dev));769}770EXPORT_SYMBOL_GPL(mxic_ecc_put_pipelined_engine);771 772struct nand_ecc_engine *773mxic_ecc_get_pipelined_engine(struct platform_device *spi_pdev)774{775 struct platform_device *eng_pdev;776 struct mxic_ecc_engine *mxic;777 778 eng_pdev = mxic_ecc_get_pdev(spi_pdev);779 if (!eng_pdev)780 return ERR_PTR(-ENODEV);781 782 mxic = platform_get_drvdata(eng_pdev);783 if (!mxic) {784 platform_device_put(eng_pdev);785 return ERR_PTR(-EPROBE_DEFER);786 }787 788 return &mxic->pipelined_engine;789}790EXPORT_SYMBOL_GPL(mxic_ecc_get_pipelined_engine);791 792/*793 * Only the external ECC engine is exported as the pipelined is SoC specific, so794 * it is registered directly by the drivers that wrap it.795 */796static int mxic_ecc_probe(struct platform_device *pdev)797{798 struct device *dev = &pdev->dev;799 struct mxic_ecc_engine *mxic;800 int ret;801 802 mxic = devm_kzalloc(&pdev->dev, sizeof(*mxic), GFP_KERNEL);803 if (!mxic)804 return -ENOMEM;805 806 mxic->dev = &pdev->dev;807 808 /*809 * Both memory regions for the ECC engine itself and the AXI slave810 * address are mandatory.811 */812 mxic->regs = devm_platform_ioremap_resource(pdev, 0);813 if (IS_ERR(mxic->regs)) {814 dev_err(&pdev->dev, "Missing memory region\n");815 return PTR_ERR(mxic->regs);816 }817 818 mxic_ecc_disable_engine(mxic);819 mxic_ecc_disable_int(mxic);820 821 /* IRQ is optional yet much more efficient */822 mxic->irq = platform_get_irq_byname_optional(pdev, "ecc-engine");823 if (mxic->irq > 0) {824 ret = devm_request_irq(&pdev->dev, mxic->irq, mxic_ecc_isr, 0,825 "mxic-ecc", mxic);826 if (ret)827 return ret;828 } else {829 dev_info(dev, "Invalid or missing IRQ, fallback to polling\n");830 mxic->irq = 0;831 }832 833 mutex_init(&mxic->lock);834 835 /*836 * In external mode, the device is the ECC engine. In pipelined mode,837 * the device is the host controller. The device is used to match the838 * right ECC engine based on the DT properties.839 */840 mxic->external_engine.dev = &pdev->dev;841 mxic->external_engine.integration = NAND_ECC_ENGINE_INTEGRATION_EXTERNAL;842 mxic->external_engine.ops = &mxic_ecc_engine_external_ops;843 844 nand_ecc_register_on_host_hw_engine(&mxic->external_engine);845 846 platform_set_drvdata(pdev, mxic);847 848 return 0;849}850 851static void mxic_ecc_remove(struct platform_device *pdev)852{853 struct mxic_ecc_engine *mxic = platform_get_drvdata(pdev);854 855 nand_ecc_unregister_on_host_hw_engine(&mxic->external_engine);856}857 858static const struct of_device_id mxic_ecc_of_ids[] = {859 {860 .compatible = "mxicy,nand-ecc-engine-rev3",861 },862 { /* sentinel */ },863};864MODULE_DEVICE_TABLE(of, mxic_ecc_of_ids);865 866static struct platform_driver mxic_ecc_driver = {867 .driver = {868 .name = "mxic-nand-ecc-engine",869 .of_match_table = mxic_ecc_of_ids,870 },871 .probe = mxic_ecc_probe,872 .remove_new = mxic_ecc_remove,873};874module_platform_driver(mxic_ecc_driver);875 876MODULE_LICENSE("GPL");877MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");878MODULE_DESCRIPTION("Macronix NAND hardware ECC controller");879