1016 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright 2017 ATMEL4 * Copyright 2017 Free Electrons5 *6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>7 *8 * Derived from the atmel_nand.c driver which contained the following9 * copyrights:10 *11 * Copyright 2003 Rick Bronson12 *13 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)14 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de)15 *16 * Derived from drivers/mtd/spia.c (removed in v3.8)17 * Copyright 2000 Steven J. Hill (sjhill@cotw.com)18 *19 * Add Hardware ECC support for AT91SAM9260 / AT91SAM926320 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 200721 *22 * Derived from Das U-Boot source code23 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)24 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas25 *26 * Add Programmable Multibit ECC support for various AT91 SoC27 * Copyright 2012 ATMEL, Hong Xu28 *29 * Add Nand Flash Controller support for SAMA5 SoC30 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)31 *32 * The PMECC is an hardware assisted BCH engine, which means part of the33 * ECC algorithm is left to the software. The hardware/software repartition34 * is explained in the "PMECC Controller Functional Description" chapter in35 * Atmel datasheets, and some of the functions in this file are directly36 * implementing the algorithms described in the "Software Implementation"37 * sub-section.38 *39 * TODO: it seems that the software BCH implementation in lib/bch.c is already40 * providing some of the logic we are implementing here. It would be smart41 * to expose the needed lib/bch.c helpers/functions and re-use them here.42 */43 44#include <linux/genalloc.h>45#include <linux/iopoll.h>46#include <linux/module.h>47#include <linux/mtd/rawnand.h>48#include <linux/of_irq.h>49#include <linux/of_platform.h>50#include <linux/platform_device.h>51#include <linux/slab.h>52 53#include "pmecc.h"54 55/* Galois field dimension */56#define PMECC_GF_DIMENSION_13 1357#define PMECC_GF_DIMENSION_14 1458 59/* Primitive Polynomial used by PMECC */60#define PMECC_GF_13_PRIMITIVE_POLY 0x201b61#define PMECC_GF_14_PRIMITIVE_POLY 0x444362 63#define PMECC_LOOKUP_TABLE_SIZE_512 0x200064#define PMECC_LOOKUP_TABLE_SIZE_1024 0x400065 66/* Time out value for reading PMECC status register */67#define PMECC_MAX_TIMEOUT_MS 10068 69/* PMECC Register Definitions */70#define ATMEL_PMECC_CFG 0x071#define PMECC_CFG_BCH_STRENGTH(x) (x)72#define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0)73#define PMECC_CFG_SECTOR512 (0 << 4)74#define PMECC_CFG_SECTOR1024 (1 << 4)75#define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8)76#define PMECC_CFG_READ_OP (0 << 12)77#define PMECC_CFG_WRITE_OP (1 << 12)78#define PMECC_CFG_SPARE_ENABLE BIT(16)79#define PMECC_CFG_AUTO_ENABLE BIT(20)80 81#define ATMEL_PMECC_SAREA 0x482#define ATMEL_PMECC_SADDR 0x883#define ATMEL_PMECC_EADDR 0xc84 85#define ATMEL_PMECC_CLK 0x1086#define PMECC_CLK_133MHZ (2 << 0)87 88#define ATMEL_PMECC_CTRL 0x1489#define PMECC_CTRL_RST BIT(0)90#define PMECC_CTRL_DATA BIT(1)91#define PMECC_CTRL_USER BIT(2)92#define PMECC_CTRL_ENABLE BIT(4)93#define PMECC_CTRL_DISABLE BIT(5)94 95#define ATMEL_PMECC_SR 0x1896#define PMECC_SR_BUSY BIT(0)97#define PMECC_SR_ENABLE BIT(4)98 99#define ATMEL_PMECC_IER 0x1c100#define ATMEL_PMECC_IDR 0x20101#define ATMEL_PMECC_IMR 0x24102#define ATMEL_PMECC_ISR 0x28103#define PMECC_ERROR_INT BIT(0)104 105#define ATMEL_PMECC_ECC(sector, n) \106 ((((sector) + 1) * 0x40) + (n))107 108#define ATMEL_PMECC_REM(sector, n) \109 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)110 111/* PMERRLOC Register Definitions */112#define ATMEL_PMERRLOC_ELCFG 0x0113#define PMERRLOC_ELCFG_SECTOR_512 (0 << 0)114#define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0)115#define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16)116 117#define ATMEL_PMERRLOC_ELPRIM 0x4118#define ATMEL_PMERRLOC_ELEN 0x8119#define ATMEL_PMERRLOC_ELDIS 0xc120#define PMERRLOC_DISABLE BIT(0)121 122#define ATMEL_PMERRLOC_ELSR 0x10123#define PMERRLOC_ELSR_BUSY BIT(0)124 125#define ATMEL_PMERRLOC_ELIER 0x14126#define ATMEL_PMERRLOC_ELIDR 0x18127#define ATMEL_PMERRLOC_ELIMR 0x1c128#define ATMEL_PMERRLOC_ELISR 0x20129#define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8)130#define PMERRLOC_CALC_DONE BIT(0)131 132#define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28)133 134#define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs))135 136struct atmel_pmecc_gf_tables {137 u16 *alpha_to;138 u16 *index_of;139};140 141struct atmel_pmecc_caps {142 const int *strengths;143 int nstrengths;144 int el_offset;145 bool correct_erased_chunks;146};147 148struct atmel_pmecc {149 struct device *dev;150 const struct atmel_pmecc_caps *caps;151 152 struct {153 void __iomem *base;154 void __iomem *errloc;155 } regs;156 157 struct mutex lock;158};159 160struct atmel_pmecc_user_conf_cache {161 u32 cfg;162 u32 sarea;163 u32 saddr;164 u32 eaddr;165};166 167struct atmel_pmecc_user {168 struct atmel_pmecc_user_conf_cache cache;169 struct atmel_pmecc *pmecc;170 const struct atmel_pmecc_gf_tables *gf_tables;171 int eccbytes;172 s16 *partial_syn;173 s16 *si;174 s16 *lmu;175 s16 *smu;176 s32 *mu;177 s32 *dmu;178 s32 *delta;179 u32 isr;180};181 182static DEFINE_MUTEX(pmecc_gf_tables_lock);183static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;184static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;185 186static inline int deg(unsigned int poly)187{188 /* polynomial degree is the most-significant bit index */189 return fls(poly) - 1;190}191 192static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,193 struct atmel_pmecc_gf_tables *gf_tables)194{195 unsigned int i, x = 1;196 const unsigned int k = BIT(deg(poly));197 unsigned int nn = BIT(mm) - 1;198 199 /* primitive polynomial must be of degree m */200 if (k != (1u << mm))201 return -EINVAL;202 203 for (i = 0; i < nn; i++) {204 gf_tables->alpha_to[i] = x;205 gf_tables->index_of[x] = i;206 if (i && (x == 1))207 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */208 return -EINVAL;209 x <<= 1;210 if (x & k)211 x ^= poly;212 }213 gf_tables->alpha_to[nn] = 1;214 gf_tables->index_of[0] = 0;215 216 return 0;217}218 219static const struct atmel_pmecc_gf_tables *220atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)221{222 struct atmel_pmecc_gf_tables *gf_tables;223 unsigned int poly, degree, table_size;224 int ret;225 226 if (req->ecc.sectorsize == 512) {227 degree = PMECC_GF_DIMENSION_13;228 poly = PMECC_GF_13_PRIMITIVE_POLY;229 table_size = PMECC_LOOKUP_TABLE_SIZE_512;230 } else {231 degree = PMECC_GF_DIMENSION_14;232 poly = PMECC_GF_14_PRIMITIVE_POLY;233 table_size = PMECC_LOOKUP_TABLE_SIZE_1024;234 }235 236 gf_tables = kzalloc(sizeof(*gf_tables) +237 (2 * table_size * sizeof(u16)),238 GFP_KERNEL);239 if (!gf_tables)240 return ERR_PTR(-ENOMEM);241 242 gf_tables->alpha_to = (void *)(gf_tables + 1);243 gf_tables->index_of = gf_tables->alpha_to + table_size;244 245 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);246 if (ret) {247 kfree(gf_tables);248 return ERR_PTR(ret);249 }250 251 return gf_tables;252}253 254static const struct atmel_pmecc_gf_tables *255atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)256{257 const struct atmel_pmecc_gf_tables **gf_tables, *ret;258 259 mutex_lock(&pmecc_gf_tables_lock);260 if (req->ecc.sectorsize == 512)261 gf_tables = &pmecc_gf_tables_512;262 else263 gf_tables = &pmecc_gf_tables_1024;264 265 ret = *gf_tables;266 267 if (!ret) {268 ret = atmel_pmecc_create_gf_tables(req);269 if (!IS_ERR(ret))270 *gf_tables = ret;271 }272 mutex_unlock(&pmecc_gf_tables_lock);273 274 return ret;275}276 277static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,278 struct atmel_pmecc_user_req *req)279{280 int i, max_eccbytes, eccbytes = 0, eccstrength = 0;281 282 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)283 return -EINVAL;284 285 if (req->ecc.ooboffset >= 0 &&286 req->ecc.ooboffset + req->ecc.bytes > req->oobsize)287 return -EINVAL;288 289 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {290 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)291 return -EINVAL;292 293 if (req->pagesize > 512)294 req->ecc.sectorsize = 1024;295 else296 req->ecc.sectorsize = 512;297 }298 299 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)300 return -EINVAL;301 302 if (req->pagesize % req->ecc.sectorsize)303 return -EINVAL;304 305 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;306 307 max_eccbytes = req->ecc.bytes;308 309 for (i = 0; i < pmecc->caps->nstrengths; i++) {310 int nbytes, strength = pmecc->caps->strengths[i];311 312 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&313 strength < req->ecc.strength)314 continue;315 316 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),317 8);318 nbytes *= req->ecc.nsectors;319 320 if (nbytes > max_eccbytes)321 break;322 323 eccstrength = strength;324 eccbytes = nbytes;325 326 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)327 break;328 }329 330 if (!eccstrength)331 return -EINVAL;332 333 req->ecc.bytes = eccbytes;334 req->ecc.strength = eccstrength;335 336 if (req->ecc.ooboffset < 0)337 req->ecc.ooboffset = req->oobsize - eccbytes;338 339 return 0;340}341 342struct atmel_pmecc_user *343atmel_pmecc_create_user(struct atmel_pmecc *pmecc,344 struct atmel_pmecc_user_req *req)345{346 struct atmel_pmecc_user *user;347 const struct atmel_pmecc_gf_tables *gf_tables;348 int strength, size, ret;349 350 ret = atmel_pmecc_prepare_user_req(pmecc, req);351 if (ret)352 return ERR_PTR(ret);353 354 size = sizeof(*user);355 size = ALIGN(size, sizeof(u16));356 /* Reserve space for partial_syn, si and smu */357 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *358 (2 + req->ecc.strength + 2);359 /* Reserve space for lmu. */360 size += (req->ecc.strength + 1) * sizeof(u16);361 /* Reserve space for mu, dmu and delta. */362 size = ALIGN(size, sizeof(s32));363 size += (req->ecc.strength + 1) * sizeof(s32) * 3;364 365 user = kzalloc(size, GFP_KERNEL);366 if (!user)367 return ERR_PTR(-ENOMEM);368 369 user->pmecc = pmecc;370 371 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));372 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);373 user->lmu = user->si + ((2 * req->ecc.strength) + 1);374 user->smu = user->lmu + (req->ecc.strength + 1);375 user->mu = (s32 *)PTR_ALIGN(user->smu +376 (((2 * req->ecc.strength) + 1) *377 (req->ecc.strength + 2)),378 sizeof(s32));379 user->dmu = user->mu + req->ecc.strength + 1;380 user->delta = user->dmu + req->ecc.strength + 1;381 382 gf_tables = atmel_pmecc_get_gf_tables(req);383 if (IS_ERR(gf_tables)) {384 kfree(user);385 return ERR_CAST(gf_tables);386 }387 388 user->gf_tables = gf_tables;389 390 user->eccbytes = req->ecc.bytes / req->ecc.nsectors;391 392 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {393 if (pmecc->caps->strengths[strength] == req->ecc.strength)394 break;395 }396 397 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |398 PMECC_CFG_NSECTORS(req->ecc.nsectors);399 400 if (req->ecc.sectorsize == 1024)401 user->cache.cfg |= PMECC_CFG_SECTOR1024;402 403 user->cache.sarea = req->oobsize - 1;404 user->cache.saddr = req->ecc.ooboffset;405 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;406 407 return user;408}409EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);410 411void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user)412{413 kfree(user);414}415EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user);416 417static int get_strength(struct atmel_pmecc_user *user)418{419 const int *strengths = user->pmecc->caps->strengths;420 421 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];422}423 424static int get_sectorsize(struct atmel_pmecc_user *user)425{426 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;427}428 429static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)430{431 int strength = get_strength(user);432 u32 value;433 int i;434 435 /* Fill odd syndromes */436 for (i = 0; i < strength; i++) {437 value = readl_relaxed(user->pmecc->regs.base +438 ATMEL_PMECC_REM(sector, i / 2));439 if (i & 1)440 value >>= 16;441 442 user->partial_syn[(2 * i) + 1] = value;443 }444}445 446static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)447{448 int degree = get_sectorsize(user) == 512 ? 13 : 14;449 int cw_len = BIT(degree) - 1;450 int strength = get_strength(user);451 s16 *alpha_to = user->gf_tables->alpha_to;452 s16 *index_of = user->gf_tables->index_of;453 s16 *partial_syn = user->partial_syn;454 s16 *si;455 int i, j;456 457 /*458 * si[] is a table that holds the current syndrome value,459 * an element of that table belongs to the field460 */461 si = user->si;462 463 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));464 465 /* Computation 2t syndromes based on S(x) */466 /* Odd syndromes */467 for (i = 1; i < 2 * strength; i += 2) {468 for (j = 0; j < degree; j++) {469 if (partial_syn[i] & BIT(j))470 si[i] = alpha_to[i * j] ^ si[i];471 }472 }473 /* Even syndrome = (Odd syndrome) ** 2 */474 for (i = 2, j = 1; j <= strength; i = ++j << 1) {475 if (si[j] == 0) {476 si[i] = 0;477 } else {478 s16 tmp;479 480 tmp = index_of[si[j]];481 tmp = (tmp * 2) % cw_len;482 si[i] = alpha_to[tmp];483 }484 }485}486 487static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)488{489 s16 *lmu = user->lmu;490 s16 *si = user->si;491 s32 *mu = user->mu;492 s32 *dmu = user->dmu;493 s32 *delta = user->delta;494 int degree = get_sectorsize(user) == 512 ? 13 : 14;495 int cw_len = BIT(degree) - 1;496 int strength = get_strength(user);497 int num = 2 * strength + 1;498 s16 *index_of = user->gf_tables->index_of;499 s16 *alpha_to = user->gf_tables->alpha_to;500 int i, j, k;501 u32 dmu_0_count, tmp;502 s16 *smu = user->smu;503 504 /* index of largest delta */505 int ro;506 int largest;507 int diff;508 509 dmu_0_count = 0;510 511 /* First Row */512 513 /* Mu */514 mu[0] = -1;515 516 memset(smu, 0, sizeof(s16) * num);517 smu[0] = 1;518 519 /* discrepancy set to 1 */520 dmu[0] = 1;521 /* polynom order set to 0 */522 lmu[0] = 0;523 delta[0] = (mu[0] * 2 - lmu[0]) >> 1;524 525 /* Second Row */526 527 /* Mu */528 mu[1] = 0;529 /* Sigma(x) set to 1 */530 memset(&smu[num], 0, sizeof(s16) * num);531 smu[num] = 1;532 533 /* discrepancy set to S1 */534 dmu[1] = si[1];535 536 /* polynom order set to 0 */537 lmu[1] = 0;538 539 delta[1] = (mu[1] * 2 - lmu[1]) >> 1;540 541 /* Init the Sigma(x) last row */542 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);543 544 for (i = 1; i <= strength; i++) {545 mu[i + 1] = i << 1;546 /* Begin Computing Sigma (Mu+1) and L(mu) */547 /* check if discrepancy is set to 0 */548 if (dmu[i] == 0) {549 dmu_0_count++;550 551 tmp = ((strength - (lmu[i] >> 1) - 1) / 2);552 if ((strength - (lmu[i] >> 1) - 1) & 0x1)553 tmp += 2;554 else555 tmp += 1;556 557 if (dmu_0_count == tmp) {558 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)559 smu[(strength + 1) * num + j] =560 smu[i * num + j];561 562 lmu[strength + 1] = lmu[i];563 return;564 }565 566 /* copy polynom */567 for (j = 0; j <= lmu[i] >> 1; j++)568 smu[(i + 1) * num + j] = smu[i * num + j];569 570 /* copy previous polynom order to the next */571 lmu[i + 1] = lmu[i];572 } else {573 ro = 0;574 largest = -1;575 /* find largest delta with dmu != 0 */576 for (j = 0; j < i; j++) {577 if ((dmu[j]) && (delta[j] > largest)) {578 largest = delta[j];579 ro = j;580 }581 }582 583 /* compute difference */584 diff = (mu[i] - mu[ro]);585 586 /* Compute degree of the new smu polynomial */587 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))588 lmu[i + 1] = lmu[i];589 else590 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;591 592 /* Init smu[i+1] with 0 */593 for (k = 0; k < num; k++)594 smu[(i + 1) * num + k] = 0;595 596 /* Compute smu[i+1] */597 for (k = 0; k <= lmu[ro] >> 1; k++) {598 s16 a, b, c;599 600 if (!(smu[ro * num + k] && dmu[i]))601 continue;602 603 a = index_of[dmu[i]];604 b = index_of[dmu[ro]];605 c = index_of[smu[ro * num + k]];606 tmp = a + (cw_len - b) + c;607 a = alpha_to[tmp % cw_len];608 smu[(i + 1) * num + (k + diff)] = a;609 }610 611 for (k = 0; k <= lmu[i] >> 1; k++)612 smu[(i + 1) * num + k] ^= smu[i * num + k];613 }614 615 /* End Computing Sigma (Mu+1) and L(mu) */616 /* In either case compute delta */617 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;618 619 /* Do not compute discrepancy for the last iteration */620 if (i >= strength)621 continue;622 623 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {624 tmp = 2 * (i - 1);625 if (k == 0) {626 dmu[i + 1] = si[tmp + 3];627 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {628 s16 a, b, c;629 630 a = index_of[smu[(i + 1) * num + k]];631 b = si[2 * (i - 1) + 3 - k];632 c = index_of[b];633 tmp = a + c;634 tmp %= cw_len;635 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];636 }637 }638 }639}640 641static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)642{643 int sector_size = get_sectorsize(user);644 int degree = sector_size == 512 ? 13 : 14;645 struct atmel_pmecc *pmecc = user->pmecc;646 int strength = get_strength(user);647 int ret, roots_nbr, i, err_nbr = 0;648 int num = (2 * strength) + 1;649 s16 *smu = user->smu;650 u32 val;651 652 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);653 654 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {655 writel_relaxed(smu[(strength + 1) * num + i],656 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));657 err_nbr++;658 }659 660 val = (err_nbr - 1) << 16;661 if (sector_size == 1024)662 val |= 1;663 664 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);665 writel((sector_size * 8) + (degree * strength),666 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);667 668 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +669 ATMEL_PMERRLOC_ELISR,670 val, val & PMERRLOC_CALC_DONE, 0,671 PMECC_MAX_TIMEOUT_MS * 1000);672 if (ret) {673 dev_err(pmecc->dev,674 "PMECC: Timeout to calculate error location.\n");675 return ret;676 }677 678 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;679 /* Number of roots == degree of smu hence <= cap */680 if (roots_nbr == user->lmu[strength + 1] >> 1)681 return err_nbr - 1;682 683 /*684 * Number of roots does not match the degree of smu685 * unable to correct error.686 */687 return -EBADMSG;688}689 690int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,691 void *data, void *ecc)692{693 struct atmel_pmecc *pmecc = user->pmecc;694 int sectorsize = get_sectorsize(user);695 int eccbytes = user->eccbytes;696 int i, nerrors;697 698 if (!(user->isr & BIT(sector)))699 return 0;700 701 atmel_pmecc_gen_syndrome(user, sector);702 atmel_pmecc_substitute(user);703 atmel_pmecc_get_sigma(user);704 705 nerrors = atmel_pmecc_err_location(user);706 if (nerrors < 0)707 return nerrors;708 709 for (i = 0; i < nerrors; i++) {710 const char *area;711 int byte, bit;712 u32 errpos;713 u8 *ptr;714 715 errpos = readl_relaxed(pmecc->regs.errloc +716 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));717 errpos--;718 719 byte = errpos / 8;720 bit = errpos % 8;721 722 if (byte < sectorsize) {723 ptr = data + byte;724 area = "data";725 } else if (byte < sectorsize + eccbytes) {726 ptr = ecc + byte - sectorsize;727 area = "ECC";728 } else {729 dev_dbg(pmecc->dev,730 "Invalid errpos value (%d, max is %d)\n",731 errpos, (sectorsize + eccbytes) * 8);732 return -EINVAL;733 }734 735 dev_dbg(pmecc->dev,736 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",737 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));738 739 *ptr ^= BIT(bit);740 }741 742 return nerrors;743}744EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);745 746bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)747{748 return user->pmecc->caps->correct_erased_chunks;749}750EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);751 752void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,753 int sector, void *ecc)754{755 struct atmel_pmecc *pmecc = user->pmecc;756 u8 *ptr = ecc;757 int i;758 759 for (i = 0; i < user->eccbytes; i++)760 ptr[i] = readb_relaxed(pmecc->regs.base +761 ATMEL_PMECC_ECC(sector, i));762}763EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);764 765void atmel_pmecc_reset(struct atmel_pmecc *pmecc)766{767 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);768 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);769}770EXPORT_SYMBOL_GPL(atmel_pmecc_reset);771 772int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)773{774 struct atmel_pmecc *pmecc = user->pmecc;775 u32 cfg;776 777 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {778 dev_err(pmecc->dev, "Bad ECC operation!");779 return -EINVAL;780 }781 782 mutex_lock(&user->pmecc->lock);783 784 cfg = user->cache.cfg;785 if (op == NAND_ECC_WRITE)786 cfg |= PMECC_CFG_WRITE_OP;787 else788 cfg |= PMECC_CFG_AUTO_ENABLE;789 790 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);791 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);792 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);793 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);794 795 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);796 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);797 798 return 0;799}800EXPORT_SYMBOL_GPL(atmel_pmecc_enable);801 802void atmel_pmecc_disable(struct atmel_pmecc_user *user)803{804 atmel_pmecc_reset(user->pmecc);805 mutex_unlock(&user->pmecc->lock);806}807EXPORT_SYMBOL_GPL(atmel_pmecc_disable);808 809int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)810{811 struct atmel_pmecc *pmecc = user->pmecc;812 u32 status;813 int ret;814 815 ret = readl_relaxed_poll_timeout(pmecc->regs.base +816 ATMEL_PMECC_SR,817 status, !(status & PMECC_SR_BUSY), 0,818 PMECC_MAX_TIMEOUT_MS * 1000);819 if (ret) {820 dev_err(pmecc->dev,821 "Timeout while waiting for PMECC ready.\n");822 return ret;823 }824 825 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);826 827 return 0;828}829EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);830 831static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev,832 const struct atmel_pmecc_caps *caps,833 int pmecc_res_idx, int errloc_res_idx)834{835 struct device *dev = &pdev->dev;836 struct atmel_pmecc *pmecc;837 838 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);839 if (!pmecc)840 return ERR_PTR(-ENOMEM);841 842 pmecc->caps = caps;843 pmecc->dev = dev;844 mutex_init(&pmecc->lock);845 846 pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx);847 if (IS_ERR(pmecc->regs.base))848 return ERR_CAST(pmecc->regs.base);849 850 pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx);851 if (IS_ERR(pmecc->regs.errloc))852 return ERR_CAST(pmecc->regs.errloc);853 854 /* Disable all interrupts before registering the PMECC handler. */855 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);856 atmel_pmecc_reset(pmecc);857 858 return pmecc;859}860 861static void devm_atmel_pmecc_put(struct device *dev, void *res)862{863 struct atmel_pmecc **pmecc = res;864 865 put_device((*pmecc)->dev);866}867 868static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev,869 struct device_node *np)870{871 struct platform_device *pdev;872 struct atmel_pmecc *pmecc, **ptr;873 int ret;874 875 pdev = of_find_device_by_node(np);876 if (!pdev)877 return ERR_PTR(-EPROBE_DEFER);878 pmecc = platform_get_drvdata(pdev);879 if (!pmecc) {880 ret = -EPROBE_DEFER;881 goto err_put_device;882 }883 884 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);885 if (!ptr) {886 ret = -ENOMEM;887 goto err_put_device;888 }889 890 *ptr = pmecc;891 892 devres_add(userdev, ptr);893 894 return pmecc;895 896err_put_device:897 put_device(&pdev->dev);898 return ERR_PTR(ret);899}900 901static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };902 903static struct atmel_pmecc_caps at91sam9g45_caps = {904 .strengths = atmel_pmecc_strengths,905 .nstrengths = 5,906 .el_offset = 0x8c,907};908 909static struct atmel_pmecc_caps sama5d4_caps = {910 .strengths = atmel_pmecc_strengths,911 .nstrengths = 5,912 .el_offset = 0x8c,913 .correct_erased_chunks = true,914};915 916static struct atmel_pmecc_caps sama5d2_caps = {917 .strengths = atmel_pmecc_strengths,918 .nstrengths = 6,919 .el_offset = 0xac,920 .correct_erased_chunks = true,921};922 923static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = {924 { .compatible = "atmel,sama5d4-nand", &sama5d4_caps },925 { .compatible = "atmel,sama5d2-nand", &sama5d2_caps },926 { /* sentinel */ }927};928 929struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev)930{931 struct atmel_pmecc *pmecc;932 struct device_node *np;933 934 if (!userdev)935 return ERR_PTR(-EINVAL);936 937 if (!userdev->of_node)938 return NULL;939 940 np = of_parse_phandle(userdev->of_node, "ecc-engine", 0);941 if (np) {942 pmecc = atmel_pmecc_get_by_node(userdev, np);943 of_node_put(np);944 } else {945 /*946 * Support old DT bindings: in this case the PMECC iomem947 * resources are directly defined in the user pdev at position948 * 1 and 2. Extract all relevant information from there.949 */950 struct platform_device *pdev = to_platform_device(userdev);951 const struct atmel_pmecc_caps *caps;952 const struct of_device_id *match;953 954 /* No PMECC engine available. */955 if (!of_property_read_bool(userdev->of_node,956 "atmel,has-pmecc"))957 return NULL;958 959 caps = &at91sam9g45_caps;960 961 /* Find the caps associated to the NAND dev node. */962 match = of_match_node(atmel_pmecc_legacy_match,963 userdev->of_node);964 if (match && match->data)965 caps = match->data;966 967 pmecc = atmel_pmecc_create(pdev, caps, 1, 2);968 }969 970 return pmecc;971}972EXPORT_SYMBOL(devm_atmel_pmecc_get);973 974static const struct of_device_id atmel_pmecc_match[] = {975 { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps },976 { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps },977 { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps },978 { /* sentinel */ }979};980MODULE_DEVICE_TABLE(of, atmel_pmecc_match);981 982static int atmel_pmecc_probe(struct platform_device *pdev)983{984 struct device *dev = &pdev->dev;985 const struct atmel_pmecc_caps *caps;986 struct atmel_pmecc *pmecc;987 988 caps = of_device_get_match_data(&pdev->dev);989 if (!caps) {990 dev_err(dev, "Invalid caps\n");991 return -EINVAL;992 }993 994 pmecc = atmel_pmecc_create(pdev, caps, 0, 1);995 if (IS_ERR(pmecc))996 return PTR_ERR(pmecc);997 998 platform_set_drvdata(pdev, pmecc);999 1000 return 0;1001}1002 1003static struct platform_driver atmel_pmecc_driver = {1004 .driver = {1005 .name = "atmel-pmecc",1006 .of_match_table = atmel_pmecc_match,1007 },1008 .probe = atmel_pmecc_probe,1009};1010module_platform_driver(atmel_pmecc_driver);1011 1012MODULE_LICENSE("GPL");1013MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");1014MODULE_DESCRIPTION("PMECC engine driver");1015MODULE_ALIAS("platform:atmel_pmecc");1016