2207 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>4 *5 * Derived from:6 * https://github.com/yuq/sunxi-nfc-mtd7 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>8 *9 * https://github.com/hno/Allwinner-Info10 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>11 *12 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>13 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>14 */15 16#include <linux/dma-mapping.h>17#include <linux/slab.h>18#include <linux/module.h>19#include <linux/moduleparam.h>20#include <linux/platform_device.h>21#include <linux/of.h>22#include <linux/mtd/mtd.h>23#include <linux/mtd/rawnand.h>24#include <linux/mtd/partitions.h>25#include <linux/clk.h>26#include <linux/delay.h>27#include <linux/dmaengine.h>28#include <linux/interrupt.h>29#include <linux/iopoll.h>30#include <linux/reset.h>31 32#define NFC_REG_CTL 0x000033#define NFC_REG_ST 0x000434#define NFC_REG_INT 0x000835#define NFC_REG_TIMING_CTL 0x000C36#define NFC_REG_TIMING_CFG 0x001037#define NFC_REG_ADDR_LOW 0x001438#define NFC_REG_ADDR_HIGH 0x001839#define NFC_REG_SECTOR_NUM 0x001C40#define NFC_REG_CNT 0x002041#define NFC_REG_CMD 0x002442#define NFC_REG_RCMD_SET 0x002843#define NFC_REG_WCMD_SET 0x002C44#define NFC_REG_A10_IO_DATA 0x003045#define NFC_REG_A23_IO_DATA 0x030046#define NFC_REG_ECC_CTL 0x003447#define NFC_REG_ECC_ST 0x003848#define NFC_REG_DEBUG 0x003C49#define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)50#define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))51#define NFC_REG_SPARE_AREA 0x00A052#define NFC_REG_PAT_ID 0x00A453#define NFC_REG_MDMA_ADDR 0x00C054#define NFC_REG_MDMA_CNT 0x00C455#define NFC_RAM0_BASE 0x040056#define NFC_RAM1_BASE 0x080057 58/* define bit use in NFC_CTL */59#define NFC_EN BIT(0)60#define NFC_RESET BIT(1)61#define NFC_BUS_WIDTH_MSK BIT(2)62#define NFC_BUS_WIDTH_8 (0 << 2)63#define NFC_BUS_WIDTH_16 (1 << 2)64#define NFC_RB_SEL_MSK BIT(3)65#define NFC_RB_SEL(x) ((x) << 3)66#define NFC_CE_SEL_MSK GENMASK(26, 24)67#define NFC_CE_SEL(x) ((x) << 24)68#define NFC_CE_CTL BIT(6)69#define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)70#define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)71#define NFC_SAM BIT(12)72#define NFC_RAM_METHOD BIT(14)73#define NFC_DMA_TYPE_NORMAL BIT(15)74#define NFC_DEBUG_CTL BIT(31)75 76/* define bit use in NFC_ST */77#define NFC_RB_B2R BIT(0)78#define NFC_CMD_INT_FLAG BIT(1)79#define NFC_DMA_INT_FLAG BIT(2)80#define NFC_CMD_FIFO_STATUS BIT(3)81#define NFC_STA BIT(4)82#define NFC_NATCH_INT_FLAG BIT(5)83#define NFC_RB_STATE(x) BIT(x + 8)84 85/* define bit use in NFC_INT */86#define NFC_B2R_INT_ENABLE BIT(0)87#define NFC_CMD_INT_ENABLE BIT(1)88#define NFC_DMA_INT_ENABLE BIT(2)89#define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \90 NFC_CMD_INT_ENABLE | \91 NFC_DMA_INT_ENABLE)92 93/* define bit use in NFC_TIMING_CTL */94#define NFC_TIMING_CTL_EDO BIT(8)95 96/* define NFC_TIMING_CFG register layout */97#define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \98 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \99 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \100 (((tCAD) & 0x7) << 8))101 102/* define bit use in NFC_CMD */103#define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)104#define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)105#define NFC_CMD(x) (x)106#define NFC_ADR_NUM_MSK GENMASK(18, 16)107#define NFC_ADR_NUM(x) (((x) - 1) << 16)108#define NFC_SEND_ADR BIT(19)109#define NFC_ACCESS_DIR BIT(20)110#define NFC_DATA_TRANS BIT(21)111#define NFC_SEND_CMD1 BIT(22)112#define NFC_WAIT_FLAG BIT(23)113#define NFC_SEND_CMD2 BIT(24)114#define NFC_SEQ BIT(25)115#define NFC_DATA_SWAP_METHOD BIT(26)116#define NFC_ROW_AUTO_INC BIT(27)117#define NFC_SEND_CMD3 BIT(28)118#define NFC_SEND_CMD4 BIT(29)119#define NFC_CMD_TYPE_MSK GENMASK(31, 30)120#define NFC_NORMAL_OP (0 << 30)121#define NFC_ECC_OP (1 << 30)122#define NFC_PAGE_OP (2U << 30)123 124/* define bit use in NFC_RCMD_SET */125#define NFC_READ_CMD_MSK GENMASK(7, 0)126#define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)127#define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)128 129/* define bit use in NFC_WCMD_SET */130#define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)131#define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)132#define NFC_READ_CMD0_MSK GENMASK(23, 16)133#define NFC_READ_CMD1_MSK GENMASK(31, 24)134 135/* define bit use in NFC_ECC_CTL */136#define NFC_ECC_EN BIT(0)137#define NFC_ECC_PIPELINE BIT(3)138#define NFC_ECC_EXCEPTION BIT(4)139#define NFC_ECC_BLOCK_SIZE_MSK BIT(5)140#define NFC_ECC_BLOCK_512 BIT(5)141#define NFC_RANDOM_EN BIT(9)142#define NFC_RANDOM_DIRECTION BIT(10)143#define NFC_ECC_MODE_MSK GENMASK(15, 12)144#define NFC_ECC_MODE(x) ((x) << 12)145#define NFC_RANDOM_SEED_MSK GENMASK(30, 16)146#define NFC_RANDOM_SEED(x) ((x) << 16)147 148/* define bit use in NFC_ECC_ST */149#define NFC_ECC_ERR(x) BIT(x)150#define NFC_ECC_ERR_MSK GENMASK(15, 0)151#define NFC_ECC_PAT_FOUND(x) BIT(x + 16)152#define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)153 154#define NFC_DEFAULT_TIMEOUT_MS 1000155 156#define NFC_SRAM_SIZE 1024157 158#define NFC_MAX_CS 7159 160/**161 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select162 *163 * @cs: the NAND CS id used to communicate with a NAND Chip164 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC165 */166struct sunxi_nand_chip_sel {167 u8 cs;168 s8 rb;169};170 171/**172 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support173 *174 * @ecc_ctl: ECC_CTL register value for this NAND chip175 */176struct sunxi_nand_hw_ecc {177 u32 ecc_ctl;178};179 180/**181 * struct sunxi_nand_chip - stores NAND chip device related information182 *183 * @node: used to store NAND chips into a list184 * @nand: base NAND chip structure185 * @ecc: ECC controller structure186 * @clk_rate: clk_rate required for this NAND chip187 * @timing_cfg: TIMING_CFG register value for this NAND chip188 * @timing_ctl: TIMING_CTL register value for this NAND chip189 * @nsels: number of CS lines required by the NAND chip190 * @sels: array of CS lines descriptions191 */192struct sunxi_nand_chip {193 struct list_head node;194 struct nand_chip nand;195 struct sunxi_nand_hw_ecc ecc;196 unsigned long clk_rate;197 u32 timing_cfg;198 u32 timing_ctl;199 int nsels;200 struct sunxi_nand_chip_sel sels[] __counted_by(nsels);201};202 203static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)204{205 return container_of(nand, struct sunxi_nand_chip, nand);206}207 208/*209 * NAND Controller capabilities structure: stores NAND controller capabilities210 * for distinction between compatible strings.211 *212 * @has_mdma: Use mbus dma mode, otherwise general dma213 * through MBUS on A23/A33 needs extra configuration.214 * @reg_io_data: I/O data register215 * @dma_maxburst: DMA maxburst216 */217struct sunxi_nfc_caps {218 bool has_mdma;219 unsigned int reg_io_data;220 unsigned int dma_maxburst;221};222 223/**224 * struct sunxi_nfc - stores sunxi NAND controller information225 *226 * @controller: base controller structure227 * @dev: parent device (used to print error messages)228 * @regs: NAND controller registers229 * @ahb_clk: NAND controller AHB clock230 * @mod_clk: NAND controller mod clock231 * @reset: NAND controller reset line232 * @assigned_cs: bitmask describing already assigned CS lines233 * @clk_rate: NAND controller current clock rate234 * @chips: a list containing all the NAND chips attached to this NAND235 * controller236 * @complete: a completion object used to wait for NAND controller events237 * @dmac: the DMA channel attached to the NAND controller238 * @caps: NAND Controller capabilities239 */240struct sunxi_nfc {241 struct nand_controller controller;242 struct device *dev;243 void __iomem *regs;244 struct clk *ahb_clk;245 struct clk *mod_clk;246 struct reset_control *reset;247 unsigned long assigned_cs;248 unsigned long clk_rate;249 struct list_head chips;250 struct completion complete;251 struct dma_chan *dmac;252 const struct sunxi_nfc_caps *caps;253};254 255static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)256{257 return container_of(ctrl, struct sunxi_nfc, controller);258}259 260static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)261{262 struct sunxi_nfc *nfc = dev_id;263 u32 st = readl(nfc->regs + NFC_REG_ST);264 u32 ien = readl(nfc->regs + NFC_REG_INT);265 266 if (!(ien & st))267 return IRQ_NONE;268 269 if ((ien & st) == ien)270 complete(&nfc->complete);271 272 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);273 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);274 275 return IRQ_HANDLED;276}277 278static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,279 bool use_polling, unsigned int timeout_ms)280{281 int ret;282 283 if (events & ~NFC_INT_MASK)284 return -EINVAL;285 286 if (!timeout_ms)287 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;288 289 if (!use_polling) {290 init_completion(&nfc->complete);291 292 writel(events, nfc->regs + NFC_REG_INT);293 294 ret = wait_for_completion_timeout(&nfc->complete,295 msecs_to_jiffies(timeout_ms));296 if (!ret)297 ret = -ETIMEDOUT;298 else299 ret = 0;300 301 writel(0, nfc->regs + NFC_REG_INT);302 } else {303 u32 status;304 305 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,306 (status & events) == events, 1,307 timeout_ms * 1000);308 }309 310 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);311 312 if (ret)313 dev_err(nfc->dev, "wait interrupt timedout\n");314 315 return ret;316}317 318static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)319{320 u32 status;321 int ret;322 323 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,324 !(status & NFC_CMD_FIFO_STATUS), 1,325 NFC_DEFAULT_TIMEOUT_MS * 1000);326 if (ret)327 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");328 329 return ret;330}331 332static int sunxi_nfc_rst(struct sunxi_nfc *nfc)333{334 u32 ctl;335 int ret;336 337 writel(0, nfc->regs + NFC_REG_ECC_CTL);338 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);339 340 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,341 !(ctl & NFC_RESET), 1,342 NFC_DEFAULT_TIMEOUT_MS * 1000);343 if (ret)344 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");345 346 return ret;347}348 349static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,350 int chunksize, int nchunks,351 enum dma_data_direction ddir,352 struct scatterlist *sg)353{354 struct dma_async_tx_descriptor *dmad;355 enum dma_transfer_direction tdir;356 dma_cookie_t dmat;357 int ret;358 359 if (ddir == DMA_FROM_DEVICE)360 tdir = DMA_DEV_TO_MEM;361 else362 tdir = DMA_MEM_TO_DEV;363 364 sg_init_one(sg, buf, nchunks * chunksize);365 ret = dma_map_sg(nfc->dev, sg, 1, ddir);366 if (!ret)367 return -ENOMEM;368 369 if (!nfc->caps->has_mdma) {370 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);371 if (!dmad) {372 ret = -EINVAL;373 goto err_unmap_buf;374 }375 }376 377 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,378 nfc->regs + NFC_REG_CTL);379 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);380 writel(chunksize, nfc->regs + NFC_REG_CNT);381 382 if (nfc->caps->has_mdma) {383 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL,384 nfc->regs + NFC_REG_CTL);385 writel(chunksize * nchunks, nfc->regs + NFC_REG_MDMA_CNT);386 writel(sg_dma_address(sg), nfc->regs + NFC_REG_MDMA_ADDR);387 } else {388 dmat = dmaengine_submit(dmad);389 390 ret = dma_submit_error(dmat);391 if (ret)392 goto err_clr_dma_flag;393 }394 395 return 0;396 397err_clr_dma_flag:398 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,399 nfc->regs + NFC_REG_CTL);400 401err_unmap_buf:402 dma_unmap_sg(nfc->dev, sg, 1, ddir);403 return ret;404}405 406static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,407 enum dma_data_direction ddir,408 struct scatterlist *sg)409{410 dma_unmap_sg(nfc->dev, sg, 1, ddir);411 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,412 nfc->regs + NFC_REG_CTL);413}414 415static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)416{417 struct mtd_info *mtd = nand_to_mtd(nand);418 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);419 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);420 struct sunxi_nand_chip_sel *sel;421 u32 ctl;422 423 if (cs >= sunxi_nand->nsels)424 return;425 426 ctl = readl(nfc->regs + NFC_REG_CTL) &427 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);428 429 sel = &sunxi_nand->sels[cs];430 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);431 if (sel->rb >= 0)432 ctl |= NFC_RB_SEL(sel->rb);433 434 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);435 436 if (nfc->clk_rate != sunxi_nand->clk_rate) {437 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);438 nfc->clk_rate = sunxi_nand->clk_rate;439 }440 441 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);442 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);443 writel(ctl, nfc->regs + NFC_REG_CTL);444}445 446static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)447{448 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);449 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);450 int ret;451 int cnt;452 int offs = 0;453 u32 tmp;454 455 while (len > offs) {456 bool poll = false;457 458 cnt = min(len - offs, NFC_SRAM_SIZE);459 460 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);461 if (ret)462 break;463 464 writel(cnt, nfc->regs + NFC_REG_CNT);465 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;466 writel(tmp, nfc->regs + NFC_REG_CMD);467 468 /* Arbitrary limit for polling mode */469 if (cnt < 64)470 poll = true;471 472 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);473 if (ret)474 break;475 476 if (buf)477 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,478 cnt);479 offs += cnt;480 }481}482 483static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,484 int len)485{486 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);487 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);488 int ret;489 int cnt;490 int offs = 0;491 u32 tmp;492 493 while (len > offs) {494 bool poll = false;495 496 cnt = min(len - offs, NFC_SRAM_SIZE);497 498 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);499 if (ret)500 break;501 502 writel(cnt, nfc->regs + NFC_REG_CNT);503 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);504 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |505 NFC_ACCESS_DIR;506 writel(tmp, nfc->regs + NFC_REG_CMD);507 508 /* Arbitrary limit for polling mode */509 if (cnt < 64)510 poll = true;511 512 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);513 if (ret)514 break;515 516 offs += cnt;517 }518}519 520/* These seed values have been extracted from Allwinner's BSP */521static const u16 sunxi_nfc_randomizer_page_seeds[] = {522 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,523 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,524 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,525 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,526 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,527 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,528 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,529 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,530 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,531 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,532 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,533 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,534 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,535 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,536 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,537 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,538};539 540/*541 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds542 * have been generated using543 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what544 * the randomizer engine does internally before de/scrambling OOB data.545 *546 * Those tables are statically defined to avoid calculating randomizer state547 * at runtime.548 */549static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {550 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,551 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,552 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,553 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,554 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,555 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,556 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,557 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,558 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,559 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,560 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,561 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,562 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,563 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,564 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,565 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,566};567 568static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {569 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,570 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,571 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,572 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,573 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,574 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,575 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,576 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,577 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,578 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,579 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,580 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,581 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,582 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,583 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,584 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,585};586 587static u16 sunxi_nfc_randomizer_step(u16 state, int count)588{589 state &= 0x7fff;590 591 /*592 * This loop is just a simple implementation of a Fibonacci LFSR using593 * the x16 + x15 + 1 polynomial.594 */595 while (count--)596 state = ((state >> 1) |597 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;598 599 return state;600}601 602static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,603 bool ecc)604{605 struct mtd_info *mtd = nand_to_mtd(nand);606 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;607 int mod = mtd_div_by_ws(mtd->erasesize, mtd);608 609 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))610 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);611 612 if (ecc) {613 if (mtd->ecc_step_size == 512)614 seeds = sunxi_nfc_randomizer_ecc512_seeds;615 else616 seeds = sunxi_nfc_randomizer_ecc1024_seeds;617 }618 619 return seeds[page % mod];620}621 622static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,623 bool ecc)624{625 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);626 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);627 u16 state;628 629 if (!(nand->options & NAND_NEED_SCRAMBLING))630 return;631 632 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);633 state = sunxi_nfc_randomizer_state(nand, page, ecc);634 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;635 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);636}637 638static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)639{640 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);641 642 if (!(nand->options & NAND_NEED_SCRAMBLING))643 return;644 645 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,646 nfc->regs + NFC_REG_ECC_CTL);647}648 649static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)650{651 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);652 653 if (!(nand->options & NAND_NEED_SCRAMBLING))654 return;655 656 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,657 nfc->regs + NFC_REG_ECC_CTL);658}659 660static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)661{662 u16 state = sunxi_nfc_randomizer_state(nand, page, true);663 664 bbm[0] ^= state;665 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);666}667 668static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,669 const uint8_t *buf, int len,670 bool ecc, int page)671{672 sunxi_nfc_randomizer_config(nand, page, ecc);673 sunxi_nfc_randomizer_enable(nand);674 sunxi_nfc_write_buf(nand, buf, len);675 sunxi_nfc_randomizer_disable(nand);676}677 678static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,679 int len, bool ecc, int page)680{681 sunxi_nfc_randomizer_config(nand, page, ecc);682 sunxi_nfc_randomizer_enable(nand);683 sunxi_nfc_read_buf(nand, buf, len);684 sunxi_nfc_randomizer_disable(nand);685}686 687static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)688{689 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);690 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);691 692 writel(sunxi_nand->ecc.ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);693}694 695static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)696{697 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);698 699 writel(0, nfc->regs + NFC_REG_ECC_CTL);700}701 702static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)703{704 buf[0] = user_data;705 buf[1] = user_data >> 8;706 buf[2] = user_data >> 16;707 buf[3] = user_data >> 24;708}709 710static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)711{712 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);713}714 715static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,716 int step, bool bbm, int page)717{718 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);719 720 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),721 oob);722 723 /* De-randomize the Bad Block Marker. */724 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))725 sunxi_nfc_randomize_bbm(nand, page, oob);726}727 728static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,729 const u8 *oob, int step,730 bool bbm, int page)731{732 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);733 u8 user_data[4];734 735 /* Randomize the Bad Block Marker. */736 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {737 memcpy(user_data, oob, sizeof(user_data));738 sunxi_nfc_randomize_bbm(nand, page, user_data);739 oob = user_data;740 }741 742 writel(sunxi_nfc_buf_to_user_data(oob),743 nfc->regs + NFC_REG_USER_DATA(step));744}745 746static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,747 unsigned int *max_bitflips, int ret)748{749 struct mtd_info *mtd = nand_to_mtd(nand);750 751 if (ret < 0) {752 mtd->ecc_stats.failed++;753 } else {754 mtd->ecc_stats.corrected += ret;755 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);756 }757}758 759static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,760 int step, u32 status, bool *erased)761{762 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);763 struct nand_ecc_ctrl *ecc = &nand->ecc;764 u32 tmp;765 766 *erased = false;767 768 if (status & NFC_ECC_ERR(step))769 return -EBADMSG;770 771 if (status & NFC_ECC_PAT_FOUND(step)) {772 u8 pattern;773 774 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {775 pattern = 0x0;776 } else {777 pattern = 0xff;778 *erased = true;779 }780 781 if (data)782 memset(data, pattern, ecc->size);783 784 if (oob)785 memset(oob, pattern, ecc->bytes + 4);786 787 return 0;788 }789 790 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));791 792 return NFC_ECC_ERR_CNT(step, tmp);793}794 795static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,796 u8 *data, int data_off,797 u8 *oob, int oob_off,798 int *cur_off,799 unsigned int *max_bitflips,800 bool bbm, bool oob_required, int page)801{802 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);803 struct nand_ecc_ctrl *ecc = &nand->ecc;804 int raw_mode = 0;805 bool erased;806 int ret;807 808 if (*cur_off != data_off)809 nand_change_read_column_op(nand, data_off, NULL, 0, false);810 811 sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);812 813 if (data_off + ecc->size != oob_off)814 nand_change_read_column_op(nand, oob_off, NULL, 0, false);815 816 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);817 if (ret)818 return ret;819 820 sunxi_nfc_randomizer_enable(nand);821 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,822 nfc->regs + NFC_REG_CMD);823 824 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);825 sunxi_nfc_randomizer_disable(nand);826 if (ret)827 return ret;828 829 *cur_off = oob_off + ecc->bytes + 4;830 831 ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,832 readl(nfc->regs + NFC_REG_ECC_ST),833 &erased);834 if (erased)835 return 1;836 837 if (ret < 0) {838 /*839 * Re-read the data with the randomizer disabled to identify840 * bitflips in erased pages.841 */842 if (nand->options & NAND_NEED_SCRAMBLING)843 nand_change_read_column_op(nand, data_off, data,844 ecc->size, false);845 else846 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,847 ecc->size);848 849 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,850 false);851 852 ret = nand_check_erased_ecc_chunk(data, ecc->size,853 oob, ecc->bytes + 4,854 NULL, 0, ecc->strength);855 if (ret >= 0)856 raw_mode = 1;857 } else {858 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);859 860 if (oob_required) {861 nand_change_read_column_op(nand, oob_off, NULL, 0,862 false);863 sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,864 true, page);865 866 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,867 bbm, page);868 }869 }870 871 sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);872 873 return raw_mode;874}875 876static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,877 u8 *oob, int *cur_off,878 bool randomize, int page)879{880 struct mtd_info *mtd = nand_to_mtd(nand);881 struct nand_ecc_ctrl *ecc = &nand->ecc;882 int offset = ((ecc->bytes + 4) * ecc->steps);883 int len = mtd->oobsize - offset;884 885 if (len <= 0)886 return;887 888 if (!cur_off || *cur_off != offset)889 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,890 false);891 892 if (!randomize)893 sunxi_nfc_read_buf(nand, oob + offset, len);894 else895 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,896 false, page);897 898 if (cur_off)899 *cur_off = mtd->oobsize + mtd->writesize;900}901 902static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,903 int oob_required, int page,904 int nchunks)905{906 bool randomized = nand->options & NAND_NEED_SCRAMBLING;907 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);908 struct mtd_info *mtd = nand_to_mtd(nand);909 struct nand_ecc_ctrl *ecc = &nand->ecc;910 unsigned int max_bitflips = 0;911 int ret, i, raw_mode = 0;912 struct scatterlist sg;913 u32 status, wait;914 915 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);916 if (ret)917 return ret;918 919 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,920 DMA_FROM_DEVICE, &sg);921 if (ret)922 return ret;923 924 sunxi_nfc_hw_ecc_enable(nand);925 sunxi_nfc_randomizer_config(nand, page, false);926 sunxi_nfc_randomizer_enable(nand);927 928 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |929 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);930 931 wait = NFC_CMD_INT_FLAG;932 933 if (nfc->caps->has_mdma)934 wait |= NFC_DMA_INT_FLAG;935 else936 dma_async_issue_pending(nfc->dmac);937 938 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,939 nfc->regs + NFC_REG_CMD);940 941 ret = sunxi_nfc_wait_events(nfc, wait, false, 0);942 if (ret && !nfc->caps->has_mdma)943 dmaengine_terminate_all(nfc->dmac);944 945 sunxi_nfc_randomizer_disable(nand);946 sunxi_nfc_hw_ecc_disable(nand);947 948 sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);949 950 if (ret)951 return ret;952 953 status = readl(nfc->regs + NFC_REG_ECC_ST);954 955 for (i = 0; i < nchunks; i++) {956 int data_off = i * ecc->size;957 int oob_off = i * (ecc->bytes + 4);958 u8 *data = buf + data_off;959 u8 *oob = nand->oob_poi + oob_off;960 bool erased;961 962 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,963 oob_required ? oob : NULL,964 i, status, &erased);965 966 /* ECC errors are handled in the second loop. */967 if (ret < 0)968 continue;969 970 if (oob_required && !erased) {971 /* TODO: use DMA to retrieve OOB */972 nand_change_read_column_op(nand,973 mtd->writesize + oob_off,974 oob, ecc->bytes + 4, false);975 976 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,977 !i, page);978 }979 980 if (erased)981 raw_mode = 1;982 983 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);984 }985 986 if (status & NFC_ECC_ERR_MSK) {987 for (i = 0; i < nchunks; i++) {988 int data_off = i * ecc->size;989 int oob_off = i * (ecc->bytes + 4);990 u8 *data = buf + data_off;991 u8 *oob = nand->oob_poi + oob_off;992 993 if (!(status & NFC_ECC_ERR(i)))994 continue;995 996 /*997 * Re-read the data with the randomizer disabled to998 * identify bitflips in erased pages.999 * TODO: use DMA to read page in raw mode1000 */1001 if (randomized)1002 nand_change_read_column_op(nand, data_off,1003 data, ecc->size,1004 false);1005 1006 /* TODO: use DMA to retrieve OOB */1007 nand_change_read_column_op(nand,1008 mtd->writesize + oob_off,1009 oob, ecc->bytes + 4, false);1010 1011 ret = nand_check_erased_ecc_chunk(data, ecc->size,1012 oob, ecc->bytes + 4,1013 NULL, 0,1014 ecc->strength);1015 if (ret >= 0)1016 raw_mode = 1;1017 1018 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);1019 }1020 }1021 1022 if (oob_required)1023 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,1024 NULL, !raw_mode,1025 page);1026 1027 return max_bitflips;1028}1029 1030static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,1031 const u8 *data, int data_off,1032 const u8 *oob, int oob_off,1033 int *cur_off, bool bbm,1034 int page)1035{1036 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);1037 struct nand_ecc_ctrl *ecc = &nand->ecc;1038 int ret;1039 1040 if (data_off != *cur_off)1041 nand_change_write_column_op(nand, data_off, NULL, 0, false);1042 1043 sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);1044 1045 if (data_off + ecc->size != oob_off)1046 nand_change_write_column_op(nand, oob_off, NULL, 0, false);1047 1048 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);1049 if (ret)1050 return ret;1051 1052 sunxi_nfc_randomizer_enable(nand);1053 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);1054 1055 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |1056 NFC_ACCESS_DIR | NFC_ECC_OP,1057 nfc->regs + NFC_REG_CMD);1058 1059 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);1060 sunxi_nfc_randomizer_disable(nand);1061 if (ret)1062 return ret;1063 1064 *cur_off = oob_off + ecc->bytes + 4;1065 1066 return 0;1067}1068 1069static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,1070 u8 *oob, int *cur_off,1071 int page)1072{1073 struct mtd_info *mtd = nand_to_mtd(nand);1074 struct nand_ecc_ctrl *ecc = &nand->ecc;1075 int offset = ((ecc->bytes + 4) * ecc->steps);1076 int len = mtd->oobsize - offset;1077 1078 if (len <= 0)1079 return;1080 1081 if (!cur_off || *cur_off != offset)1082 nand_change_write_column_op(nand, offset + mtd->writesize,1083 NULL, 0, false);1084 1085 sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);1086 1087 if (cur_off)1088 *cur_off = mtd->oobsize + mtd->writesize;1089}1090 1091static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,1092 int oob_required, int page)1093{1094 struct mtd_info *mtd = nand_to_mtd(nand);1095 struct nand_ecc_ctrl *ecc = &nand->ecc;1096 unsigned int max_bitflips = 0;1097 int ret, i, cur_off = 0;1098 bool raw_mode = false;1099 1100 sunxi_nfc_select_chip(nand, nand->cur_cs);1101 1102 nand_read_page_op(nand, page, 0, NULL, 0);1103 1104 sunxi_nfc_hw_ecc_enable(nand);1105 1106 for (i = 0; i < ecc->steps; i++) {1107 int data_off = i * ecc->size;1108 int oob_off = i * (ecc->bytes + 4);1109 u8 *data = buf + data_off;1110 u8 *oob = nand->oob_poi + oob_off;1111 1112 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,1113 oob_off + mtd->writesize,1114 &cur_off, &max_bitflips,1115 !i, oob_required, page);1116 if (ret < 0)1117 return ret;1118 else if (ret)1119 raw_mode = true;1120 }1121 1122 if (oob_required)1123 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,1124 !raw_mode, page);1125 1126 sunxi_nfc_hw_ecc_disable(nand);1127 1128 return max_bitflips;1129}1130 1131static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,1132 int oob_required, int page)1133{1134 int ret;1135 1136 sunxi_nfc_select_chip(nand, nand->cur_cs);1137 1138 nand_read_page_op(nand, page, 0, NULL, 0);1139 1140 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,1141 nand->ecc.steps);1142 if (ret >= 0)1143 return ret;1144 1145 /* Fallback to PIO mode */1146 return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);1147}1148 1149static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,1150 u32 data_offs, u32 readlen,1151 u8 *bufpoi, int page)1152{1153 struct mtd_info *mtd = nand_to_mtd(nand);1154 struct nand_ecc_ctrl *ecc = &nand->ecc;1155 int ret, i, cur_off = 0;1156 unsigned int max_bitflips = 0;1157 1158 sunxi_nfc_select_chip(nand, nand->cur_cs);1159 1160 nand_read_page_op(nand, page, 0, NULL, 0);1161 1162 sunxi_nfc_hw_ecc_enable(nand);1163 1164 for (i = data_offs / ecc->size;1165 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {1166 int data_off = i * ecc->size;1167 int oob_off = i * (ecc->bytes + 4);1168 u8 *data = bufpoi + data_off;1169 u8 *oob = nand->oob_poi + oob_off;1170 1171 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,1172 oob,1173 oob_off + mtd->writesize,1174 &cur_off, &max_bitflips, !i,1175 false, page);1176 if (ret < 0)1177 return ret;1178 }1179 1180 sunxi_nfc_hw_ecc_disable(nand);1181 1182 return max_bitflips;1183}1184 1185static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,1186 u32 data_offs, u32 readlen,1187 u8 *buf, int page)1188{1189 int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);1190 int ret;1191 1192 sunxi_nfc_select_chip(nand, nand->cur_cs);1193 1194 nand_read_page_op(nand, page, 0, NULL, 0);1195 1196 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);1197 if (ret >= 0)1198 return ret;1199 1200 /* Fallback to PIO mode */1201 return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,1202 buf, page);1203}1204 1205static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,1206 const uint8_t *buf, int oob_required,1207 int page)1208{1209 struct mtd_info *mtd = nand_to_mtd(nand);1210 struct nand_ecc_ctrl *ecc = &nand->ecc;1211 int ret, i, cur_off = 0;1212 1213 sunxi_nfc_select_chip(nand, nand->cur_cs);1214 1215 nand_prog_page_begin_op(nand, page, 0, NULL, 0);1216 1217 sunxi_nfc_hw_ecc_enable(nand);1218 1219 for (i = 0; i < ecc->steps; i++) {1220 int data_off = i * ecc->size;1221 int oob_off = i * (ecc->bytes + 4);1222 const u8 *data = buf + data_off;1223 const u8 *oob = nand->oob_poi + oob_off;1224 1225 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,1226 oob_off + mtd->writesize,1227 &cur_off, !i, page);1228 if (ret)1229 return ret;1230 }1231 1232 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))1233 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,1234 &cur_off, page);1235 1236 sunxi_nfc_hw_ecc_disable(nand);1237 1238 return nand_prog_page_end_op(nand);1239}1240 1241static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,1242 u32 data_offs, u32 data_len,1243 const u8 *buf, int oob_required,1244 int page)1245{1246 struct mtd_info *mtd = nand_to_mtd(nand);1247 struct nand_ecc_ctrl *ecc = &nand->ecc;1248 int ret, i, cur_off = 0;1249 1250 sunxi_nfc_select_chip(nand, nand->cur_cs);1251 1252 nand_prog_page_begin_op(nand, page, 0, NULL, 0);1253 1254 sunxi_nfc_hw_ecc_enable(nand);1255 1256 for (i = data_offs / ecc->size;1257 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {1258 int data_off = i * ecc->size;1259 int oob_off = i * (ecc->bytes + 4);1260 const u8 *data = buf + data_off;1261 const u8 *oob = nand->oob_poi + oob_off;1262 1263 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,1264 oob_off + mtd->writesize,1265 &cur_off, !i, page);1266 if (ret)1267 return ret;1268 }1269 1270 sunxi_nfc_hw_ecc_disable(nand);1271 1272 return nand_prog_page_end_op(nand);1273}1274 1275static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,1276 const u8 *buf,1277 int oob_required,1278 int page)1279{1280 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);1281 struct nand_ecc_ctrl *ecc = &nand->ecc;1282 struct scatterlist sg;1283 u32 wait;1284 int ret, i;1285 1286 sunxi_nfc_select_chip(nand, nand->cur_cs);1287 1288 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);1289 if (ret)1290 return ret;1291 1292 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,1293 DMA_TO_DEVICE, &sg);1294 if (ret)1295 goto pio_fallback;1296 1297 for (i = 0; i < ecc->steps; i++) {1298 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));1299 1300 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);1301 }1302 1303 nand_prog_page_begin_op(nand, page, 0, NULL, 0);1304 1305 sunxi_nfc_hw_ecc_enable(nand);1306 sunxi_nfc_randomizer_config(nand, page, false);1307 sunxi_nfc_randomizer_enable(nand);1308 1309 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,1310 nfc->regs + NFC_REG_WCMD_SET);1311 1312 wait = NFC_CMD_INT_FLAG;1313 1314 if (nfc->caps->has_mdma)1315 wait |= NFC_DMA_INT_FLAG;1316 else1317 dma_async_issue_pending(nfc->dmac);1318 1319 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |1320 NFC_DATA_TRANS | NFC_ACCESS_DIR,1321 nfc->regs + NFC_REG_CMD);1322 1323 ret = sunxi_nfc_wait_events(nfc, wait, false, 0);1324 if (ret && !nfc->caps->has_mdma)1325 dmaengine_terminate_all(nfc->dmac);1326 1327 sunxi_nfc_randomizer_disable(nand);1328 sunxi_nfc_hw_ecc_disable(nand);1329 1330 sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);1331 1332 if (ret)1333 return ret;1334 1335 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))1336 /* TODO: use DMA to transfer extra OOB bytes ? */1337 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,1338 NULL, page);1339 1340 return nand_prog_page_end_op(nand);1341 1342pio_fallback:1343 return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);1344}1345 1346static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)1347{1348 u8 *buf = nand_get_data_buf(nand);1349 1350 return nand->ecc.read_page(nand, buf, 1, page);1351}1352 1353static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)1354{1355 struct mtd_info *mtd = nand_to_mtd(nand);1356 u8 *buf = nand_get_data_buf(nand);1357 int ret;1358 1359 memset(buf, 0xff, mtd->writesize);1360 ret = nand->ecc.write_page(nand, buf, 1, page);1361 if (ret)1362 return ret;1363 1364 /* Send command to program the OOB data */1365 return nand_prog_page_end_op(nand);1366}1367 1368static const s32 tWB_lut[] = {6, 12, 16, 20};1369static const s32 tRHW_lut[] = {4, 8, 12, 20};1370 1371static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,1372 u32 clk_period)1373{1374 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);1375 int i;1376 1377 for (i = 0; i < lut_size; i++) {1378 if (clk_cycles <= lut[i])1379 return i;1380 }1381 1382 /* Doesn't fit */1383 return -EINVAL;1384}1385 1386#define sunxi_nand_lookup_timing(l, p, c) \1387 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)1388 1389static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,1390 const struct nand_interface_config *conf)1391{1392 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);1393 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);1394 const struct nand_sdr_timings *timings;1395 u32 min_clk_period = 0;1396 s32 tWB, tADL, tWHR, tRHW, tCAD;1397 long real_clk_rate;1398 1399 timings = nand_get_sdr_timings(conf);1400 if (IS_ERR(timings))1401 return -ENOTSUPP;1402 1403 /* T1 <=> tCLS */1404 if (timings->tCLS_min > min_clk_period)1405 min_clk_period = timings->tCLS_min;1406 1407 /* T2 <=> tCLH */1408 if (timings->tCLH_min > min_clk_period)1409 min_clk_period = timings->tCLH_min;1410 1411 /* T3 <=> tCS */1412 if (timings->tCS_min > min_clk_period)1413 min_clk_period = timings->tCS_min;1414 1415 /* T4 <=> tCH */1416 if (timings->tCH_min > min_clk_period)1417 min_clk_period = timings->tCH_min;1418 1419 /* T5 <=> tWP */1420 if (timings->tWP_min > min_clk_period)1421 min_clk_period = timings->tWP_min;1422 1423 /* T6 <=> tWH */1424 if (timings->tWH_min > min_clk_period)1425 min_clk_period = timings->tWH_min;1426 1427 /* T7 <=> tALS */1428 if (timings->tALS_min > min_clk_period)1429 min_clk_period = timings->tALS_min;1430 1431 /* T8 <=> tDS */1432 if (timings->tDS_min > min_clk_period)1433 min_clk_period = timings->tDS_min;1434 1435 /* T9 <=> tDH */1436 if (timings->tDH_min > min_clk_period)1437 min_clk_period = timings->tDH_min;1438 1439 /* T10 <=> tRR */1440 if (timings->tRR_min > (min_clk_period * 3))1441 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);1442 1443 /* T11 <=> tALH */1444 if (timings->tALH_min > min_clk_period)1445 min_clk_period = timings->tALH_min;1446 1447 /* T12 <=> tRP */1448 if (timings->tRP_min > min_clk_period)1449 min_clk_period = timings->tRP_min;1450 1451 /* T13 <=> tREH */1452 if (timings->tREH_min > min_clk_period)1453 min_clk_period = timings->tREH_min;1454 1455 /* T14 <=> tRC */1456 if (timings->tRC_min > (min_clk_period * 2))1457 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);1458 1459 /* T15 <=> tWC */1460 if (timings->tWC_min > (min_clk_period * 2))1461 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);1462 1463 /* T16 - T19 + tCAD */1464 if (timings->tWB_max > (min_clk_period * 20))1465 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);1466 1467 if (timings->tADL_min > (min_clk_period * 32))1468 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);1469 1470 if (timings->tWHR_min > (min_clk_period * 32))1471 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);1472 1473 if (timings->tRHW_min > (min_clk_period * 20))1474 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);1475 1476 /*1477 * In non-EDO, tREA should be less than tRP to guarantee that the1478 * controller does not sample the IO lines too early. Unfortunately,1479 * the sunxi NAND controller does not allow us to have different1480 * values for tRP and tREH (tRP = tREH = tRW / 2).1481 *1482 * We have 2 options to overcome this limitation:1483 *1484 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint1485 * 2/ Use EDO mode (only works if timings->tRLOH > 0)1486 */1487 if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)1488 min_clk_period = timings->tREA_max;1489 1490 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,1491 min_clk_period);1492 if (tWB < 0) {1493 dev_err(nfc->dev, "unsupported tWB\n");1494 return tWB;1495 }1496 1497 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;1498 if (tADL > 3) {1499 dev_err(nfc->dev, "unsupported tADL\n");1500 return -EINVAL;1501 }1502 1503 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;1504 if (tWHR > 3) {1505 dev_err(nfc->dev, "unsupported tWHR\n");1506 return -EINVAL;1507 }1508 1509 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,1510 min_clk_period);1511 if (tRHW < 0) {1512 dev_err(nfc->dev, "unsupported tRHW\n");1513 return tRHW;1514 }1515 1516 if (csline == NAND_DATA_IFACE_CHECK_ONLY)1517 return 0;1518 1519 /*1520 * TODO: according to ONFI specs this value only applies for DDR NAND,1521 * but Allwinner seems to set this to 0x7. Mimic them for now.1522 */1523 tCAD = 0x7;1524 1525 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */1526 sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);1527 1528 /* Convert min_clk_period from picoseconds to nanoseconds */1529 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);1530 1531 /*1532 * Unlike what is stated in Allwinner datasheet, the clk_rate should1533 * be set to (1 / min_clk_period), and not (2 / min_clk_period).1534 * This new formula was verified with a scope and validated by1535 * Allwinner engineers.1536 */1537 sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;1538 real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);1539 if (real_clk_rate <= 0) {1540 dev_err(nfc->dev, "Unable to round clk %lu\n",1541 sunxi_nand->clk_rate);1542 return -EINVAL;1543 }1544 1545 sunxi_nand->timing_ctl = 0;1546 1547 /*1548 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data1549 * output cycle timings shall be used if the host drives tRC less than1550 * 30 ns. We should also use EDO mode if tREA is bigger than tRP.1551 */1552 min_clk_period = NSEC_PER_SEC / real_clk_rate;1553 if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)1554 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;1555 1556 return 0;1557}1558 1559static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,1560 struct mtd_oob_region *oobregion)1561{1562 struct nand_chip *nand = mtd_to_nand(mtd);1563 struct nand_ecc_ctrl *ecc = &nand->ecc;1564 1565 if (section >= ecc->steps)1566 return -ERANGE;1567 1568 oobregion->offset = section * (ecc->bytes + 4) + 4;1569 oobregion->length = ecc->bytes;1570 1571 return 0;1572}1573 1574static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,1575 struct mtd_oob_region *oobregion)1576{1577 struct nand_chip *nand = mtd_to_nand(mtd);1578 struct nand_ecc_ctrl *ecc = &nand->ecc;1579 1580 if (section > ecc->steps)1581 return -ERANGE;1582 1583 /*1584 * The first 2 bytes are used for BB markers, hence we1585 * only have 2 bytes available in the first user data1586 * section.1587 */1588 if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {1589 oobregion->offset = 2;1590 oobregion->length = 2;1591 1592 return 0;1593 }1594 1595 /*1596 * The controller does not provide access to OOB bytes1597 * past the end of the ECC data.1598 */1599 if (section == ecc->steps && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST)1600 return -ERANGE;1601 1602 oobregion->offset = section * (ecc->bytes + 4);1603 1604 if (section < ecc->steps)1605 oobregion->length = 4;1606 else1607 oobregion->length = mtd->oobsize - oobregion->offset;1608 1609 return 0;1610}1611 1612static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {1613 .ecc = sunxi_nand_ooblayout_ecc,1614 .free = sunxi_nand_ooblayout_free,1615};1616 1617static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,1618 struct nand_ecc_ctrl *ecc,1619 struct device_node *np)1620{1621 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };1622 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);1623 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);1624 struct mtd_info *mtd = nand_to_mtd(nand);1625 struct nand_device *nanddev = mtd_to_nanddev(mtd);1626 int nsectors;1627 int i;1628 1629 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {1630 int bytes;1631 1632 ecc->size = 1024;1633 nsectors = mtd->writesize / ecc->size;1634 1635 /* Reserve 2 bytes for the BBM */1636 bytes = (mtd->oobsize - 2) / nsectors;1637 1638 /* 4 non-ECC bytes are added before each ECC bytes section */1639 bytes -= 4;1640 1641 /* and bytes has to be even. */1642 if (bytes % 2)1643 bytes--;1644 1645 ecc->strength = bytes * 8 / fls(8 * ecc->size);1646 1647 for (i = 0; i < ARRAY_SIZE(strengths); i++) {1648 if (strengths[i] > ecc->strength)1649 break;1650 }1651 1652 if (!i)1653 ecc->strength = 0;1654 else1655 ecc->strength = strengths[i - 1];1656 }1657 1658 if (ecc->size != 512 && ecc->size != 1024)1659 return -EINVAL;1660 1661 /* Prefer 1k ECC chunk over 512 ones */1662 if (ecc->size == 512 && mtd->writesize > 512) {1663 ecc->size = 1024;1664 ecc->strength *= 2;1665 }1666 1667 /* Add ECC info retrieval from DT */1668 for (i = 0; i < ARRAY_SIZE(strengths); i++) {1669 if (ecc->strength <= strengths[i]) {1670 /*1671 * Update ecc->strength value with the actual strength1672 * that will be used by the ECC engine.1673 */1674 ecc->strength = strengths[i];1675 break;1676 }1677 }1678 1679 if (i >= ARRAY_SIZE(strengths)) {1680 dev_err(nfc->dev, "unsupported strength\n");1681 return -ENOTSUPP;1682 }1683 1684 /* HW ECC always request ECC bytes for 1024 bytes blocks */1685 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);1686 1687 /* HW ECC always work with even numbers of ECC bytes */1688 ecc->bytes = ALIGN(ecc->bytes, 2);1689 1690 nsectors = mtd->writesize / ecc->size;1691 1692 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors))1693 return -EINVAL;1694 1695 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;1696 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;1697 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);1698 1699 if (nfc->dmac || nfc->caps->has_mdma) {1700 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;1701 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;1702 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;1703 nand->options |= NAND_USES_DMA;1704 } else {1705 ecc->read_page = sunxi_nfc_hw_ecc_read_page;1706 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;1707 ecc->write_page = sunxi_nfc_hw_ecc_write_page;1708 }1709 1710 /* TODO: support DMA for raw accesses and subpage write */1711 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;1712 ecc->read_oob_raw = nand_read_oob_std;1713 ecc->write_oob_raw = nand_write_oob_std;1714 1715 sunxi_nand->ecc.ecc_ctl = NFC_ECC_MODE(i) | NFC_ECC_EXCEPTION |1716 NFC_ECC_PIPELINE | NFC_ECC_EN;1717 1718 if (ecc->size == 512)1719 sunxi_nand->ecc.ecc_ctl |= NFC_ECC_BLOCK_512;1720 1721 return 0;1722}1723 1724static int sunxi_nand_attach_chip(struct nand_chip *nand)1725{1726 const struct nand_ecc_props *requirements =1727 nanddev_get_ecc_requirements(&nand->base);1728 struct nand_ecc_ctrl *ecc = &nand->ecc;1729 struct device_node *np = nand_get_flash_node(nand);1730 int ret;1731 1732 if (nand->bbt_options & NAND_BBT_USE_FLASH)1733 nand->bbt_options |= NAND_BBT_NO_OOB;1734 1735 if (nand->options & NAND_NEED_SCRAMBLING)1736 nand->options |= NAND_NO_SUBPAGE_WRITE;1737 1738 nand->options |= NAND_SUBPAGE_READ;1739 1740 if (!ecc->size) {1741 ecc->size = requirements->step_size;1742 ecc->strength = requirements->strength;1743 }1744 1745 if (!ecc->size || !ecc->strength)1746 return -EINVAL;1747 1748 switch (ecc->engine_type) {1749 case NAND_ECC_ENGINE_TYPE_ON_HOST:1750 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);1751 if (ret)1752 return ret;1753 break;1754 case NAND_ECC_ENGINE_TYPE_NONE:1755 case NAND_ECC_ENGINE_TYPE_SOFT:1756 break;1757 default:1758 return -EINVAL;1759 }1760 1761 return 0;1762}1763 1764static int sunxi_nfc_exec_subop(struct nand_chip *nand,1765 const struct nand_subop *subop)1766{1767 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);1768 u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };1769 unsigned int i, j, remaining, start;1770 void *inbuf = NULL;1771 int ret;1772 1773 for (i = 0; i < subop->ninstrs; i++) {1774 const struct nand_op_instr *instr = &subop->instrs[i];1775 1776 switch (instr->type) {1777 case NAND_OP_CMD_INSTR:1778 if (cmd & NFC_SEND_CMD1) {1779 if (WARN_ON(cmd & NFC_SEND_CMD2))1780 return -EINVAL;1781 1782 cmd |= NFC_SEND_CMD2;1783 extcmd |= instr->ctx.cmd.opcode;1784 } else {1785 cmd |= NFC_SEND_CMD1 |1786 NFC_CMD(instr->ctx.cmd.opcode);1787 }1788 break;1789 1790 case NAND_OP_ADDR_INSTR:1791 remaining = nand_subop_get_num_addr_cyc(subop, i);1792 start = nand_subop_get_addr_start_off(subop, i);1793 for (j = 0; j < 8 && j + start < remaining; j++) {1794 u32 addr = instr->ctx.addr.addrs[j + start];1795 1796 addrs[j / 4] |= addr << (j % 4) * 8;1797 }1798 1799 if (j)1800 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);1801 1802 break;1803 1804 case NAND_OP_DATA_IN_INSTR:1805 case NAND_OP_DATA_OUT_INSTR:1806 start = nand_subop_get_data_start_off(subop, i);1807 remaining = nand_subop_get_data_len(subop, i);1808 cnt = min_t(u32, remaining, NFC_SRAM_SIZE);1809 cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;1810 1811 if (instr->type == NAND_OP_DATA_OUT_INSTR) {1812 cmd |= NFC_ACCESS_DIR;1813 memcpy_toio(nfc->regs + NFC_RAM0_BASE,1814 instr->ctx.data.buf.out + start,1815 cnt);1816 } else {1817 inbuf = instr->ctx.data.buf.in + start;1818 }1819 1820 break;1821 1822 case NAND_OP_WAITRDY_INSTR:1823 cmd |= NFC_WAIT_FLAG;1824 break;1825 }1826 }1827 1828 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);1829 if (ret)1830 return ret;1831 1832 if (cmd & NFC_SEND_ADR) {1833 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);1834 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);1835 }1836 1837 if (cmd & NFC_SEND_CMD2)1838 writel(extcmd,1839 nfc->regs +1840 (cmd & NFC_ACCESS_DIR ?1841 NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));1842 1843 if (cmd & NFC_DATA_TRANS)1844 writel(cnt, nfc->regs + NFC_REG_CNT);1845 1846 writel(cmd, nfc->regs + NFC_REG_CMD);1847 1848 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,1849 !(cmd & NFC_WAIT_FLAG) && cnt < 64,1850 0);1851 if (ret)1852 return ret;1853 1854 if (inbuf)1855 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);1856 1857 return 0;1858}1859 1860static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,1861 const struct nand_subop *subop)1862{1863 return nand_soft_waitrdy(nand,1864 subop->instrs[0].ctx.waitrdy.timeout_ms);1865}1866 1867static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(1868 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,1869 NAND_OP_PARSER_PAT_CMD_ELEM(true),1870 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),1871 NAND_OP_PARSER_PAT_CMD_ELEM(true),1872 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),1873 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),1874 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,1875 NAND_OP_PARSER_PAT_CMD_ELEM(true),1876 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),1877 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),1878 NAND_OP_PARSER_PAT_CMD_ELEM(true),1879 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),1880);1881 1882static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(1883 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,1884 NAND_OP_PARSER_PAT_CMD_ELEM(true),1885 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),1886 NAND_OP_PARSER_PAT_CMD_ELEM(true),1887 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),1888 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,1889 NAND_OP_PARSER_PAT_CMD_ELEM(true),1890 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),1891 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),1892 NAND_OP_PARSER_PAT_CMD_ELEM(true)),1893 NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,1894 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),1895);1896 1897static int sunxi_nfc_exec_op(struct nand_chip *nand,1898 const struct nand_operation *op, bool check_only)1899{1900 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);1901 const struct nand_op_parser *parser;1902 1903 if (!check_only)1904 sunxi_nfc_select_chip(nand, op->cs);1905 1906 if (sunxi_nand->sels[op->cs].rb >= 0)1907 parser = &sunxi_nfc_op_parser;1908 else1909 parser = &sunxi_nfc_norb_op_parser;1910 1911 return nand_op_parser_exec_op(nand, parser, op, check_only);1912}1913 1914static const struct nand_controller_ops sunxi_nand_controller_ops = {1915 .attach_chip = sunxi_nand_attach_chip,1916 .setup_interface = sunxi_nfc_setup_interface,1917 .exec_op = sunxi_nfc_exec_op,1918};1919 1920static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)1921{1922 struct sunxi_nand_chip *sunxi_nand;1923 struct nand_chip *chip;1924 int ret;1925 1926 while (!list_empty(&nfc->chips)) {1927 sunxi_nand = list_first_entry(&nfc->chips,1928 struct sunxi_nand_chip,1929 node);1930 chip = &sunxi_nand->nand;1931 ret = mtd_device_unregister(nand_to_mtd(chip));1932 WARN_ON(ret);1933 nand_cleanup(chip);1934 list_del(&sunxi_nand->node);1935 }1936}1937 1938static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,1939 struct device_node *np)1940{1941 struct sunxi_nand_chip *sunxi_nand;1942 struct mtd_info *mtd;1943 struct nand_chip *nand;1944 int nsels;1945 int ret;1946 int i;1947 u32 tmp;1948 1949 if (!of_get_property(np, "reg", &nsels))1950 return -EINVAL;1951 1952 nsels /= sizeof(u32);1953 if (!nsels) {1954 dev_err(dev, "invalid reg property size\n");1955 return -EINVAL;1956 }1957 1958 sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),1959 GFP_KERNEL);1960 if (!sunxi_nand)1961 return -ENOMEM;1962 1963 sunxi_nand->nsels = nsels;1964 1965 for (i = 0; i < nsels; i++) {1966 ret = of_property_read_u32_index(np, "reg", i, &tmp);1967 if (ret) {1968 dev_err(dev, "could not retrieve reg property: %d\n",1969 ret);1970 return ret;1971 }1972 1973 if (tmp > NFC_MAX_CS) {1974 dev_err(dev,1975 "invalid reg value: %u (max CS = 7)\n",1976 tmp);1977 return -EINVAL;1978 }1979 1980 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {1981 dev_err(dev, "CS %d already assigned\n", tmp);1982 return -EINVAL;1983 }1984 1985 sunxi_nand->sels[i].cs = tmp;1986 1987 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&1988 tmp < 2)1989 sunxi_nand->sels[i].rb = tmp;1990 else1991 sunxi_nand->sels[i].rb = -1;1992 }1993 1994 nand = &sunxi_nand->nand;1995 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */1996 nand->controller = &nfc->controller;1997 nand->controller->ops = &sunxi_nand_controller_ops;1998 1999 /*2000 * Set the ECC mode to the default value in case nothing is specified2001 * in the DT.2002 */2003 nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;2004 nand_set_flash_node(nand, np);2005 2006 mtd = nand_to_mtd(nand);2007 mtd->dev.parent = dev;2008 2009 ret = nand_scan(nand, nsels);2010 if (ret)2011 return ret;2012 2013 ret = mtd_device_register(mtd, NULL, 0);2014 if (ret) {2015 dev_err(dev, "failed to register mtd device: %d\n", ret);2016 nand_cleanup(nand);2017 return ret;2018 }2019 2020 list_add_tail(&sunxi_nand->node, &nfc->chips);2021 2022 return 0;2023}2024 2025static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)2026{2027 struct device_node *np = dev->of_node;2028 int ret;2029 2030 for_each_child_of_node_scoped(np, nand_np) {2031 ret = sunxi_nand_chip_init(dev, nfc, nand_np);2032 if (ret) {2033 sunxi_nand_chips_cleanup(nfc);2034 return ret;2035 }2036 }2037 2038 return 0;2039}2040 2041static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r)2042{2043 int ret;2044 2045 if (nfc->caps->has_mdma)2046 return 0;2047 2048 nfc->dmac = dma_request_chan(nfc->dev, "rxtx");2049 if (IS_ERR(nfc->dmac)) {2050 ret = PTR_ERR(nfc->dmac);2051 if (ret == -EPROBE_DEFER)2052 return ret;2053 2054 /* Ignore errors to fall back to PIO mode */2055 dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret);2056 nfc->dmac = NULL;2057 } else {2058 struct dma_slave_config dmac_cfg = { };2059 2060 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;2061 dmac_cfg.dst_addr = dmac_cfg.src_addr;2062 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;2063 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;2064 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;2065 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;2066 dmaengine_slave_config(nfc->dmac, &dmac_cfg);2067 }2068 return 0;2069}2070 2071static int sunxi_nfc_probe(struct platform_device *pdev)2072{2073 struct device *dev = &pdev->dev;2074 struct resource *r;2075 struct sunxi_nfc *nfc;2076 int irq;2077 int ret;2078 2079 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);2080 if (!nfc)2081 return -ENOMEM;2082 2083 nfc->dev = dev;2084 nand_controller_init(&nfc->controller);2085 INIT_LIST_HEAD(&nfc->chips);2086 2087 nfc->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &r);2088 if (IS_ERR(nfc->regs))2089 return PTR_ERR(nfc->regs);2090 2091 irq = platform_get_irq(pdev, 0);2092 if (irq < 0)2093 return irq;2094 2095 nfc->ahb_clk = devm_clk_get_enabled(dev, "ahb");2096 if (IS_ERR(nfc->ahb_clk)) {2097 dev_err(dev, "failed to retrieve ahb clk\n");2098 return PTR_ERR(nfc->ahb_clk);2099 }2100 2101 nfc->mod_clk = devm_clk_get_enabled(dev, "mod");2102 if (IS_ERR(nfc->mod_clk)) {2103 dev_err(dev, "failed to retrieve mod clk\n");2104 return PTR_ERR(nfc->mod_clk);2105 }2106 2107 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");2108 if (IS_ERR(nfc->reset))2109 return PTR_ERR(nfc->reset);2110 2111 ret = reset_control_deassert(nfc->reset);2112 if (ret) {2113 dev_err(dev, "reset err %d\n", ret);2114 return ret;2115 }2116 2117 nfc->caps = of_device_get_match_data(&pdev->dev);2118 if (!nfc->caps) {2119 ret = -EINVAL;2120 goto out_ahb_reset_reassert;2121 }2122 2123 ret = sunxi_nfc_rst(nfc);2124 if (ret)2125 goto out_ahb_reset_reassert;2126 2127 writel(0, nfc->regs + NFC_REG_INT);2128 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,2129 0, "sunxi-nand", nfc);2130 if (ret)2131 goto out_ahb_reset_reassert;2132 2133 ret = sunxi_nfc_dma_init(nfc, r);2134 2135 if (ret)2136 goto out_ahb_reset_reassert;2137 2138 platform_set_drvdata(pdev, nfc);2139 2140 ret = sunxi_nand_chips_init(dev, nfc);2141 if (ret) {2142 dev_err(dev, "failed to init nand chips\n");2143 goto out_release_dmac;2144 }2145 2146 return 0;2147 2148out_release_dmac:2149 if (nfc->dmac)2150 dma_release_channel(nfc->dmac);2151out_ahb_reset_reassert:2152 reset_control_assert(nfc->reset);2153 2154 return ret;2155}2156 2157static void sunxi_nfc_remove(struct platform_device *pdev)2158{2159 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);2160 2161 sunxi_nand_chips_cleanup(nfc);2162 2163 reset_control_assert(nfc->reset);2164 2165 if (nfc->dmac)2166 dma_release_channel(nfc->dmac);2167}2168 2169static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {2170 .reg_io_data = NFC_REG_A10_IO_DATA,2171 .dma_maxburst = 4,2172};2173 2174static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {2175 .has_mdma = true,2176 .reg_io_data = NFC_REG_A23_IO_DATA,2177 .dma_maxburst = 8,2178};2179 2180static const struct of_device_id sunxi_nfc_ids[] = {2181 {2182 .compatible = "allwinner,sun4i-a10-nand",2183 .data = &sunxi_nfc_a10_caps,2184 },2185 {2186 .compatible = "allwinner,sun8i-a23-nand-controller",2187 .data = &sunxi_nfc_a23_caps,2188 },2189 { /* sentinel */ }2190};2191MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);2192 2193static struct platform_driver sunxi_nfc_driver = {2194 .driver = {2195 .name = "sunxi_nand",2196 .of_match_table = sunxi_nfc_ids,2197 },2198 .probe = sunxi_nfc_probe,2199 .remove_new = sunxi_nfc_remove,2200};2201module_platform_driver(sunxi_nfc_driver);2202 2203MODULE_LICENSE("GPL");2204MODULE_AUTHOR("Boris BREZILLON");2205MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");2206MODULE_ALIAS("platform:sunxi_nand");2207