600 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright (C) 2017 Free Electrons4 * Copyright (C) 2017 NextThing Co5 *6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>7 */8 9#include <linux/slab.h>10 11#include "internals.h"12 13/*14 * Special Micron status bit 3 indicates that the block has been15 * corrected by on-die ECC and should be rewritten.16 */17#define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3)18 19/*20 * On chips with 8-bit ECC and additional bit can be used to distinguish21 * cases where a errors were corrected without needing a rewrite22 *23 * Bit 4 Bit 3 Bit 0 Description24 * ----- ----- ----- -----------25 * 0 0 0 No Errors26 * 0 0 1 Multiple uncorrected errors27 * 0 1 0 4 - 6 errors corrected, recommend rewrite28 * 0 1 1 Reserved29 * 1 0 0 1 - 3 errors corrected30 * 1 0 1 Reserved31 * 1 1 0 7 - 8 errors corrected, recommend rewrite32 */33#define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0))34#define NAND_ECC_STATUS_UNCORRECTABLE BIT(0)35#define NAND_ECC_STATUS_4_6_CORRECTED BIT(3)36#define NAND_ECC_STATUS_1_3_CORRECTED BIT(4)37#define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3))38 39struct nand_onfi_vendor_micron {40 u8 two_plane_read;41 u8 read_cache;42 u8 read_unique_id;43 u8 dq_imped;44 u8 dq_imped_num_settings;45 u8 dq_imped_feat_addr;46 u8 rb_pulldown_strength;47 u8 rb_pulldown_strength_feat_addr;48 u8 rb_pulldown_strength_num_settings;49 u8 otp_mode;50 u8 otp_page_start;51 u8 otp_data_prot_addr;52 u8 otp_num_pages;53 u8 otp_feat_addr;54 u8 read_retry_options;55 u8 reserved[72];56 u8 param_revision;57} __packed;58 59struct micron_on_die_ecc {60 bool forced;61 bool enabled;62 void *rawbuf;63};64 65struct micron_nand {66 struct micron_on_die_ecc ecc;67};68 69static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode)70{71 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode};72 73 return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature);74}75 76/*77 * Configure chip properties from Micron vendor-specific ONFI table78 */79static int micron_nand_onfi_init(struct nand_chip *chip)80{81 struct nand_parameters *p = &chip->parameters;82 83 if (p->onfi) {84 struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor;85 86 chip->read_retries = micron->read_retry_options;87 chip->ops.setup_read_retry = micron_nand_setup_read_retry;88 }89 90 if (p->supports_set_get_features) {91 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list);92 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list);93 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list);94 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list);95 }96 97 return 0;98}99 100static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd,101 int section,102 struct mtd_oob_region *oobregion)103{104 if (section >= 4)105 return -ERANGE;106 107 oobregion->offset = (section * 16) + 8;108 oobregion->length = 8;109 110 return 0;111}112 113static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd,114 int section,115 struct mtd_oob_region *oobregion)116{117 if (section >= 4)118 return -ERANGE;119 120 oobregion->offset = (section * 16) + 2;121 oobregion->length = 6;122 123 return 0;124}125 126static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = {127 .ecc = micron_nand_on_die_4_ooblayout_ecc,128 .free = micron_nand_on_die_4_ooblayout_free,129};130 131static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd,132 int section,133 struct mtd_oob_region *oobregion)134{135 struct nand_chip *chip = mtd_to_nand(mtd);136 137 if (section)138 return -ERANGE;139 140 oobregion->offset = mtd->oobsize - chip->ecc.total;141 oobregion->length = chip->ecc.total;142 143 return 0;144}145 146static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd,147 int section,148 struct mtd_oob_region *oobregion)149{150 struct nand_chip *chip = mtd_to_nand(mtd);151 152 if (section)153 return -ERANGE;154 155 oobregion->offset = 2;156 oobregion->length = mtd->oobsize - chip->ecc.total - 2;157 158 return 0;159}160 161static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = {162 .ecc = micron_nand_on_die_8_ooblayout_ecc,163 .free = micron_nand_on_die_8_ooblayout_free,164};165 166static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable)167{168 struct micron_nand *micron = nand_get_manufacturer_data(chip);169 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, };170 int ret;171 172 if (micron->ecc.forced)173 return 0;174 175 if (micron->ecc.enabled == enable)176 return 0;177 178 if (enable)179 feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN;180 181 ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature);182 if (!ret)183 micron->ecc.enabled = enable;184 185 return ret;186}187 188static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,189 void *buf, int page,190 int oob_required)191{192 struct micron_nand *micron = nand_get_manufacturer_data(chip);193 struct mtd_info *mtd = nand_to_mtd(chip);194 unsigned int step, max_bitflips = 0;195 bool use_datain = false;196 int ret;197 198 if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {199 if (status & NAND_STATUS_FAIL)200 mtd->ecc_stats.failed++;201 202 return 0;203 }204 205 /*206 * The internal ECC doesn't tell us the number of bitflips that have207 * been corrected, but tells us if it recommends to rewrite the block.208 * If it's the case, we need to read the page in raw mode and compare209 * its content to the corrected version to extract the actual number of210 * bitflips.211 * But before we do that, we must make sure we have all OOB bytes read212 * in non-raw mode, even if the user did not request those bytes.213 */214 if (!oob_required) {215 /*216 * We first check which operation is supported by the controller217 * before running it. This trick makes it possible to support218 * all controllers, even the most constraints, without almost219 * any performance hit.220 *221 * TODO: could be enhanced to avoid repeating the same check222 * over and over in the fast path.223 */224 if (!nand_has_exec_op(chip) ||225 !nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,226 true))227 use_datain = true;228 229 if (use_datain)230 ret = nand_read_data_op(chip, chip->oob_poi,231 mtd->oobsize, false, false);232 else233 ret = nand_change_read_column_op(chip, mtd->writesize,234 chip->oob_poi,235 mtd->oobsize, false);236 if (ret)237 return ret;238 }239 240 micron_nand_on_die_ecc_setup(chip, false);241 242 ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf,243 mtd->writesize + mtd->oobsize);244 if (ret)245 return ret;246 247 for (step = 0; step < chip->ecc.steps; step++) {248 unsigned int offs, i, nbitflips = 0;249 u8 *rawbuf, *corrbuf;250 251 offs = step * chip->ecc.size;252 rawbuf = micron->ecc.rawbuf + offs;253 corrbuf = buf + offs;254 255 for (i = 0; i < chip->ecc.size; i++)256 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);257 258 offs = (step * 16) + 4;259 rawbuf = micron->ecc.rawbuf + mtd->writesize + offs;260 corrbuf = chip->oob_poi + offs;261 262 for (i = 0; i < chip->ecc.bytes + 4; i++)263 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);264 265 if (WARN_ON(nbitflips > chip->ecc.strength))266 return -EINVAL;267 268 max_bitflips = max(nbitflips, max_bitflips);269 mtd->ecc_stats.corrected += nbitflips;270 }271 272 return max_bitflips;273}274 275static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status)276{277 struct mtd_info *mtd = nand_to_mtd(chip);278 279 /*280 * With 8/512 we have more information but still don't know precisely281 * how many bit-flips were seen.282 */283 switch (status & NAND_ECC_STATUS_MASK) {284 case NAND_ECC_STATUS_UNCORRECTABLE:285 mtd->ecc_stats.failed++;286 return 0;287 case NAND_ECC_STATUS_1_3_CORRECTED:288 mtd->ecc_stats.corrected += 3;289 return 3;290 case NAND_ECC_STATUS_4_6_CORRECTED:291 mtd->ecc_stats.corrected += 6;292 /* rewrite recommended */293 return 6;294 case NAND_ECC_STATUS_7_8_CORRECTED:295 mtd->ecc_stats.corrected += 8;296 /* rewrite recommended */297 return 8;298 default:299 return 0;300 }301}302 303static int304micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,305 int oob_required, int page)306{307 struct mtd_info *mtd = nand_to_mtd(chip);308 bool use_datain = false;309 u8 status;310 int ret, max_bitflips = 0;311 312 ret = micron_nand_on_die_ecc_setup(chip, true);313 if (ret)314 return ret;315 316 ret = nand_read_page_op(chip, page, 0, NULL, 0);317 if (ret)318 goto out;319 320 ret = nand_status_op(chip, &status);321 if (ret)322 goto out;323 324 /*325 * We first check which operation is supported by the controller before326 * running it. This trick makes it possible to support all controllers,327 * even the most constraints, without almost any performance hit.328 *329 * TODO: could be enhanced to avoid repeating the same check over and330 * over in the fast path.331 */332 if (!nand_has_exec_op(chip) ||333 !nand_read_data_op(chip, buf, mtd->writesize, false, true))334 use_datain = true;335 336 if (use_datain) {337 ret = nand_exit_status_op(chip);338 if (ret)339 goto out;340 341 ret = nand_read_data_op(chip, buf, mtd->writesize, false,342 false);343 if (!ret && oob_required)344 ret = nand_read_data_op(chip, chip->oob_poi,345 mtd->oobsize, false, false);346 } else {347 ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize,348 false);349 if (!ret && oob_required)350 ret = nand_change_read_column_op(chip, mtd->writesize,351 chip->oob_poi,352 mtd->oobsize, false);353 }354 355 if (chip->ecc.strength == 4)356 max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,357 buf, page,358 oob_required);359 else360 max_bitflips = micron_nand_on_die_ecc_status_8(chip, status);361 362out:363 micron_nand_on_die_ecc_setup(chip, false);364 365 return ret ? ret : max_bitflips;366}367 368static int369micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf,370 int oob_required, int page)371{372 int ret;373 374 ret = micron_nand_on_die_ecc_setup(chip, true);375 if (ret)376 return ret;377 378 ret = nand_write_page_raw(chip, buf, oob_required, page);379 micron_nand_on_die_ecc_setup(chip, false);380 381 return ret;382}383 384enum {385 /* The NAND flash doesn't support on-die ECC */386 MICRON_ON_DIE_UNSUPPORTED,387 388 /*389 * The NAND flash supports on-die ECC and it can be390 * enabled/disabled by a set features command.391 */392 MICRON_ON_DIE_SUPPORTED,393 394 /*395 * The NAND flash supports on-die ECC, and it cannot be396 * disabled.397 */398 MICRON_ON_DIE_MANDATORY,399};400 401#define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0)402#define MICRON_ID_ECC_ENABLED BIT(7)403 404/*405 * Try to detect if the NAND support on-die ECC. To do this, we enable406 * the feature, and read back if it has been enabled as expected. We407 * also check if it can be disabled, because some Micron NANDs do not408 * allow disabling the on-die ECC and we don't support such NANDs for409 * now.410 *411 * This function also has the side effect of disabling on-die ECC if412 * it had been left enabled by the firmware/bootloader.413 */414static int micron_supports_on_die_ecc(struct nand_chip *chip)415{416 const struct nand_ecc_props *requirements =417 nanddev_get_ecc_requirements(&chip->base);418 u8 id[5];419 int ret;420 421 if (!chip->parameters.onfi)422 return MICRON_ON_DIE_UNSUPPORTED;423 424 if (nanddev_bits_per_cell(&chip->base) != 1)425 return MICRON_ON_DIE_UNSUPPORTED;426 427 /*428 * We only support on-die ECC of 4/512 or 8/512429 */430 if (requirements->strength != 4 && requirements->strength != 8)431 return MICRON_ON_DIE_UNSUPPORTED;432 433 /* 0x2 means on-die ECC is available. */434 if (chip->id.len != 5 ||435 (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2)436 return MICRON_ON_DIE_UNSUPPORTED;437 438 /*439 * It seems that there are devices which do not support ECC officially.440 * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports441 * enabling the ECC feature but don't reflect that to the READ_ID table.442 * So we have to guarantee that we disable the ECC feature directly443 * after we did the READ_ID table command. Later we can evaluate the444 * ECC_ENABLE support.445 */446 ret = micron_nand_on_die_ecc_setup(chip, true);447 if (ret)448 return MICRON_ON_DIE_UNSUPPORTED;449 450 ret = nand_readid_op(chip, 0, id, sizeof(id));451 if (ret)452 return MICRON_ON_DIE_UNSUPPORTED;453 454 ret = micron_nand_on_die_ecc_setup(chip, false);455 if (ret)456 return MICRON_ON_DIE_UNSUPPORTED;457 458 if (!(id[4] & MICRON_ID_ECC_ENABLED))459 return MICRON_ON_DIE_UNSUPPORTED;460 461 ret = nand_readid_op(chip, 0, id, sizeof(id));462 if (ret)463 return MICRON_ON_DIE_UNSUPPORTED;464 465 if (id[4] & MICRON_ID_ECC_ENABLED)466 return MICRON_ON_DIE_MANDATORY;467 468 /*469 * We only support on-die ECC of 4/512 or 8/512470 */471 if (requirements->strength != 4 && requirements->strength != 8)472 return MICRON_ON_DIE_UNSUPPORTED;473 474 return MICRON_ON_DIE_SUPPORTED;475}476 477static int micron_nand_init(struct nand_chip *chip)478{479 struct nand_device *base = &chip->base;480 const struct nand_ecc_props *requirements =481 nanddev_get_ecc_requirements(base);482 struct mtd_info *mtd = nand_to_mtd(chip);483 struct micron_nand *micron;484 int ondie;485 int ret;486 487 micron = kzalloc(sizeof(*micron), GFP_KERNEL);488 if (!micron)489 return -ENOMEM;490 491 nand_set_manufacturer_data(chip, micron);492 493 ret = micron_nand_onfi_init(chip);494 if (ret)495 goto err_free_manuf_data;496 497 chip->options |= NAND_BBM_FIRSTPAGE;498 499 if (mtd->writesize == 2048)500 chip->options |= NAND_BBM_SECONDPAGE;501 502 ondie = micron_supports_on_die_ecc(chip);503 504 if (ondie == MICRON_ON_DIE_MANDATORY &&505 chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_DIE) {506 pr_err("On-die ECC forcefully enabled, not supported\n");507 ret = -EINVAL;508 goto err_free_manuf_data;509 }510 511 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE) {512 if (ondie == MICRON_ON_DIE_UNSUPPORTED) {513 pr_err("On-die ECC selected but not supported\n");514 ret = -EINVAL;515 goto err_free_manuf_data;516 }517 518 if (ondie == MICRON_ON_DIE_MANDATORY) {519 micron->ecc.forced = true;520 micron->ecc.enabled = true;521 }522 523 /*524 * In case of 4bit on-die ECC, we need a buffer to store a525 * page dumped in raw mode so that we can compare its content526 * to the same page after ECC correction happened and extract527 * the real number of bitflips from this comparison.528 * That's not needed for 8-bit ECC, because the status expose529 * a better approximation of the number of bitflips in a page.530 */531 if (requirements->strength == 4) {532 micron->ecc.rawbuf = kmalloc(mtd->writesize +533 mtd->oobsize,534 GFP_KERNEL);535 if (!micron->ecc.rawbuf) {536 ret = -ENOMEM;537 goto err_free_manuf_data;538 }539 }540 541 if (requirements->strength == 4)542 mtd_set_ooblayout(mtd,543 µn_nand_on_die_4_ooblayout_ops);544 else545 mtd_set_ooblayout(mtd,546 µn_nand_on_die_8_ooblayout_ops);547 548 chip->ecc.bytes = requirements->strength * 2;549 chip->ecc.size = 512;550 chip->ecc.strength = requirements->strength;551 chip->ecc.algo = NAND_ECC_ALGO_BCH;552 chip->ecc.read_page = micron_nand_read_page_on_die_ecc;553 chip->ecc.write_page = micron_nand_write_page_on_die_ecc;554 555 if (ondie == MICRON_ON_DIE_MANDATORY) {556 chip->ecc.read_page_raw = nand_read_page_raw_notsupp;557 chip->ecc.write_page_raw = nand_write_page_raw_notsupp;558 } else {559 if (!chip->ecc.read_page_raw)560 chip->ecc.read_page_raw = nand_read_page_raw;561 if (!chip->ecc.write_page_raw)562 chip->ecc.write_page_raw = nand_write_page_raw;563 }564 }565 566 return 0;567 568err_free_manuf_data:569 kfree(micron->ecc.rawbuf);570 kfree(micron);571 572 return ret;573}574 575static void micron_nand_cleanup(struct nand_chip *chip)576{577 struct micron_nand *micron = nand_get_manufacturer_data(chip);578 579 kfree(micron->ecc.rawbuf);580 kfree(micron);581}582 583static void micron_fixup_onfi_param_page(struct nand_chip *chip,584 struct nand_onfi_params *p)585{586 /*587 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the588 * revision number field of the ONFI parameter page. Assume ONFI589 * version 1.0 if the revision number is 00 00.590 */591 if (le16_to_cpu(p->revision) == 0)592 p->revision = cpu_to_le16(ONFI_VERSION_1_0);593}594 595const struct nand_manufacturer_ops micron_nand_manuf_ops = {596 .init = micron_nand_init,597 .cleanup = micron_nand_cleanup,598 .fixup_onfi_param_page = micron_fixup_onfi_param_page,599};600