407 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * This file provides ECC correction for more than 1 bit per block of data,4 * using binary BCH codes. It relies on the generic BCH library lib/bch.c.5 *6 * Copyright © 2011 Ivan Djelic <ivan.djelic@parrot.com>7 */8 9#include <linux/types.h>10#include <linux/kernel.h>11#include <linux/module.h>12#include <linux/slab.h>13#include <linux/bitops.h>14#include <linux/mtd/nand.h>15#include <linux/mtd/nand-ecc-sw-bch.h>16 17/**18 * nand_ecc_sw_bch_calculate - Calculate the ECC corresponding to a data block19 * @nand: NAND device20 * @buf: Input buffer with raw data21 * @code: Output buffer with ECC22 */23int nand_ecc_sw_bch_calculate(struct nand_device *nand,24 const unsigned char *buf, unsigned char *code)25{26 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;27 unsigned int i;28 29 memset(code, 0, engine_conf->code_size);30 bch_encode(engine_conf->bch, buf, nand->ecc.ctx.conf.step_size, code);31 32 /* apply mask so that an erased page is a valid codeword */33 for (i = 0; i < engine_conf->code_size; i++)34 code[i] ^= engine_conf->eccmask[i];35 36 return 0;37}38EXPORT_SYMBOL(nand_ecc_sw_bch_calculate);39 40/**41 * nand_ecc_sw_bch_correct - Detect, correct and report bit error(s)42 * @nand: NAND device43 * @buf: Raw data read from the chip44 * @read_ecc: ECC bytes from the chip45 * @calc_ecc: ECC calculated from the raw data46 *47 * Detect and correct bit errors for a data block.48 */49int nand_ecc_sw_bch_correct(struct nand_device *nand, unsigned char *buf,50 unsigned char *read_ecc, unsigned char *calc_ecc)51{52 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;53 unsigned int step_size = nand->ecc.ctx.conf.step_size;54 unsigned int *errloc = engine_conf->errloc;55 int i, count;56 57 count = bch_decode(engine_conf->bch, NULL, step_size, read_ecc,58 calc_ecc, NULL, errloc);59 if (count > 0) {60 for (i = 0; i < count; i++) {61 if (errloc[i] < (step_size * 8))62 /* The error is in the data area: correct it */63 buf[errloc[i] >> 3] ^= (1 << (errloc[i] & 7));64 65 /* Otherwise the error is in the ECC area: nothing to do */66 pr_debug("%s: corrected bitflip %u\n", __func__,67 errloc[i]);68 }69 } else if (count < 0) {70 pr_err("ECC unrecoverable error\n");71 count = -EBADMSG;72 }73 74 return count;75}76EXPORT_SYMBOL(nand_ecc_sw_bch_correct);77 78/**79 * nand_ecc_sw_bch_cleanup - Cleanup software BCH ECC resources80 * @nand: NAND device81 */82static void nand_ecc_sw_bch_cleanup(struct nand_device *nand)83{84 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;85 86 bch_free(engine_conf->bch);87 kfree(engine_conf->errloc);88 kfree(engine_conf->eccmask);89}90 91/**92 * nand_ecc_sw_bch_init - Initialize software BCH ECC engine93 * @nand: NAND device94 *95 * Returns: a pointer to a new NAND BCH control structure, or NULL upon failure96 *97 * Initialize NAND BCH error correction. @nand.ecc parameters 'step_size' and98 * 'bytes' are used to compute the following BCH parameters:99 * m, the Galois field order100 * t, the error correction capability101 * 'bytes' should be equal to the number of bytes required to store m * t102 * bits, where m is such that 2^m - 1 > step_size * 8.103 *104 * Example: to configure 4 bit correction per 512 bytes, you should pass105 * step_size = 512 (thus, m = 13 is the smallest integer such that 2^m - 1 > 512 * 8)106 * bytes = 7 (7 bytes are required to store m * t = 13 * 4 = 52 bits)107 */108static int nand_ecc_sw_bch_init(struct nand_device *nand)109{110 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;111 unsigned int eccsize = nand->ecc.ctx.conf.step_size;112 unsigned int eccbytes = engine_conf->code_size;113 unsigned int m, t, i;114 unsigned char *erased_page;115 int ret;116 117 m = fls(1 + (8 * eccsize));118 t = (eccbytes * 8) / m;119 120 engine_conf->bch = bch_init(m, t, 0, false);121 if (!engine_conf->bch)122 return -EINVAL;123 124 engine_conf->eccmask = kzalloc(eccbytes, GFP_KERNEL);125 engine_conf->errloc = kmalloc_array(t, sizeof(*engine_conf->errloc),126 GFP_KERNEL);127 if (!engine_conf->eccmask || !engine_conf->errloc) {128 ret = -ENOMEM;129 goto cleanup;130 }131 132 /* Compute and store the inverted ECC of an erased step */133 erased_page = kmalloc(eccsize, GFP_KERNEL);134 if (!erased_page) {135 ret = -ENOMEM;136 goto cleanup;137 }138 139 memset(erased_page, 0xff, eccsize);140 bch_encode(engine_conf->bch, erased_page, eccsize,141 engine_conf->eccmask);142 kfree(erased_page);143 144 for (i = 0; i < eccbytes; i++)145 engine_conf->eccmask[i] ^= 0xff;146 147 /* Verify that the number of code bytes has the expected value */148 if (engine_conf->bch->ecc_bytes != eccbytes) {149 pr_err("Invalid number of ECC bytes: %u, expected: %u\n",150 eccbytes, engine_conf->bch->ecc_bytes);151 ret = -EINVAL;152 goto cleanup;153 }154 155 /* Sanity checks */156 if (8 * (eccsize + eccbytes) >= (1 << m)) {157 pr_err("ECC step size is too large (%u)\n", eccsize);158 ret = -EINVAL;159 goto cleanup;160 }161 162 return 0;163 164cleanup:165 nand_ecc_sw_bch_cleanup(nand);166 167 return ret;168}169 170int nand_ecc_sw_bch_init_ctx(struct nand_device *nand)171{172 struct nand_ecc_props *conf = &nand->ecc.ctx.conf;173 struct mtd_info *mtd = nanddev_to_mtd(nand);174 struct nand_ecc_sw_bch_conf *engine_conf;175 unsigned int code_size = 0, nsteps;176 int ret;177 178 /* Only large page NAND chips may use BCH */179 if (mtd->oobsize < 64) {180 pr_err("BCH cannot be used with small page NAND chips\n");181 return -EINVAL;182 }183 184 if (!mtd->ooblayout)185 mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());186 187 conf->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;188 conf->algo = NAND_ECC_ALGO_BCH;189 conf->step_size = nand->ecc.user_conf.step_size;190 conf->strength = nand->ecc.user_conf.strength;191 192 /*193 * Board driver should supply ECC size and ECC strength194 * values to select how many bits are correctable.195 * Otherwise, default to 512 bytes for large page devices and 256 for196 * small page devices.197 */198 if (!conf->step_size) {199 if (mtd->oobsize >= 64)200 conf->step_size = 512;201 else202 conf->step_size = 256;203 204 conf->strength = 4;205 }206 207 nsteps = mtd->writesize / conf->step_size;208 209 /* Maximize */210 if (nand->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {211 conf->step_size = 1024;212 nsteps = mtd->writesize / conf->step_size;213 /* Reserve 2 bytes for the BBM */214 code_size = (mtd->oobsize - 2) / nsteps;215 conf->strength = code_size * 8 / fls(8 * conf->step_size);216 }217 218 if (!code_size)219 code_size = DIV_ROUND_UP(conf->strength *220 fls(8 * conf->step_size), 8);221 222 if (!conf->strength)223 conf->strength = (code_size * 8) / fls(8 * conf->step_size);224 225 if (!code_size && !conf->strength) {226 pr_err("Missing ECC parameters\n");227 return -EINVAL;228 }229 230 engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL);231 if (!engine_conf)232 return -ENOMEM;233 234 ret = nand_ecc_init_req_tweaking(&engine_conf->req_ctx, nand);235 if (ret)236 goto free_engine_conf;237 238 engine_conf->code_size = code_size;239 engine_conf->calc_buf = kzalloc(mtd->oobsize, GFP_KERNEL);240 engine_conf->code_buf = kzalloc(mtd->oobsize, GFP_KERNEL);241 if (!engine_conf->calc_buf || !engine_conf->code_buf) {242 ret = -ENOMEM;243 goto free_bufs;244 }245 246 nand->ecc.ctx.priv = engine_conf;247 nand->ecc.ctx.nsteps = nsteps;248 nand->ecc.ctx.total = nsteps * code_size;249 250 ret = nand_ecc_sw_bch_init(nand);251 if (ret)252 goto free_bufs;253 254 /* Verify the layout validity */255 if (mtd_ooblayout_count_eccbytes(mtd) !=256 nand->ecc.ctx.nsteps * engine_conf->code_size) {257 pr_err("Invalid ECC layout\n");258 ret = -EINVAL;259 goto cleanup_bch_ctx;260 }261 262 return 0;263 264cleanup_bch_ctx:265 nand_ecc_sw_bch_cleanup(nand);266free_bufs:267 nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);268 kfree(engine_conf->calc_buf);269 kfree(engine_conf->code_buf);270free_engine_conf:271 kfree(engine_conf);272 273 return ret;274}275EXPORT_SYMBOL(nand_ecc_sw_bch_init_ctx);276 277void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand)278{279 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;280 281 if (engine_conf) {282 nand_ecc_sw_bch_cleanup(nand);283 nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);284 kfree(engine_conf->calc_buf);285 kfree(engine_conf->code_buf);286 kfree(engine_conf);287 }288}289EXPORT_SYMBOL(nand_ecc_sw_bch_cleanup_ctx);290 291static int nand_ecc_sw_bch_prepare_io_req(struct nand_device *nand,292 struct nand_page_io_req *req)293{294 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;295 struct mtd_info *mtd = nanddev_to_mtd(nand);296 int eccsize = nand->ecc.ctx.conf.step_size;297 int eccbytes = engine_conf->code_size;298 int eccsteps = nand->ecc.ctx.nsteps;299 int total = nand->ecc.ctx.total;300 u8 *ecccalc = engine_conf->calc_buf;301 const u8 *data;302 int i;303 304 /* Nothing to do for a raw operation */305 if (req->mode == MTD_OPS_RAW)306 return 0;307 308 /* This engine does not provide BBM/free OOB bytes protection */309 if (!req->datalen)310 return 0;311 312 nand_ecc_tweak_req(&engine_conf->req_ctx, req);313 314 /* No more preparation for page read */315 if (req->type == NAND_PAGE_READ)316 return 0;317 318 /* Preparation for page write: derive the ECC bytes and place them */319 for (i = 0, data = req->databuf.out;320 eccsteps;321 eccsteps--, i += eccbytes, data += eccsize)322 nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);323 324 return mtd_ooblayout_set_eccbytes(mtd, ecccalc, (void *)req->oobbuf.out,325 0, total);326}327 328static int nand_ecc_sw_bch_finish_io_req(struct nand_device *nand,329 struct nand_page_io_req *req)330{331 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;332 struct mtd_info *mtd = nanddev_to_mtd(nand);333 int eccsize = nand->ecc.ctx.conf.step_size;334 int total = nand->ecc.ctx.total;335 int eccbytes = engine_conf->code_size;336 int eccsteps = nand->ecc.ctx.nsteps;337 u8 *ecccalc = engine_conf->calc_buf;338 u8 *ecccode = engine_conf->code_buf;339 unsigned int max_bitflips = 0;340 u8 *data = req->databuf.in;341 int i, ret;342 343 /* Nothing to do for a raw operation */344 if (req->mode == MTD_OPS_RAW)345 return 0;346 347 /* This engine does not provide BBM/free OOB bytes protection */348 if (!req->datalen)349 return 0;350 351 /* No more preparation for page write */352 if (req->type == NAND_PAGE_WRITE) {353 nand_ecc_restore_req(&engine_conf->req_ctx, req);354 return 0;355 }356 357 /* Finish a page read: retrieve the (raw) ECC bytes*/358 ret = mtd_ooblayout_get_eccbytes(mtd, ecccode, req->oobbuf.in, 0,359 total);360 if (ret)361 return ret;362 363 /* Calculate the ECC bytes */364 for (i = 0; eccsteps; eccsteps--, i += eccbytes, data += eccsize)365 nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);366 367 /* Finish a page read: compare and correct */368 for (eccsteps = nand->ecc.ctx.nsteps, i = 0, data = req->databuf.in;369 eccsteps;370 eccsteps--, i += eccbytes, data += eccsize) {371 int stat = nand_ecc_sw_bch_correct(nand, data,372 &ecccode[i],373 &ecccalc[i]);374 if (stat < 0) {375 mtd->ecc_stats.failed++;376 } else {377 mtd->ecc_stats.corrected += stat;378 max_bitflips = max_t(unsigned int, max_bitflips, stat);379 }380 }381 382 nand_ecc_restore_req(&engine_conf->req_ctx, req);383 384 return max_bitflips;385}386 387static struct nand_ecc_engine_ops nand_ecc_sw_bch_engine_ops = {388 .init_ctx = nand_ecc_sw_bch_init_ctx,389 .cleanup_ctx = nand_ecc_sw_bch_cleanup_ctx,390 .prepare_io_req = nand_ecc_sw_bch_prepare_io_req,391 .finish_io_req = nand_ecc_sw_bch_finish_io_req,392};393 394static struct nand_ecc_engine nand_ecc_sw_bch_engine = {395 .ops = &nand_ecc_sw_bch_engine_ops,396};397 398struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void)399{400 return &nand_ecc_sw_bch_engine;401}402EXPORT_SYMBOL(nand_ecc_sw_bch_get_engine);403 404MODULE_LICENSE("GPL");405MODULE_AUTHOR("Ivan Djelic <ivan.djelic@parrot.com>");406MODULE_DESCRIPTION("NAND software BCH ECC support");407