3071 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * Cadence NAND flash controller driver4 *5 * Copyright (C) 2019 Cadence6 *7 * Author: Piotr Sroka <piotrs@cadence.com>8 */9 10#include <linux/bitfield.h>11#include <linux/clk.h>12#include <linux/dma-mapping.h>13#include <linux/dmaengine.h>14#include <linux/interrupt.h>15#include <linux/module.h>16#include <linux/mtd/mtd.h>17#include <linux/mtd/rawnand.h>18#include <linux/iopoll.h>19#include <linux/of.h>20#include <linux/platform_device.h>21#include <linux/property.h>22#include <linux/slab.h>23 24/*25 * HPNFC can work in 3 modes:26 * - PIO - can work in master or slave DMA27 * - CDMA - needs Master DMA for accessing command descriptors.28 * - Generic mode - can use only slave DMA.29 * CDMA and PIO modes can be used to execute only base commands.30 * Generic mode can be used to execute any command31 * on NAND flash memory. Driver uses CDMA mode for32 * block erasing, page reading, page programing.33 * Generic mode is used for executing rest of commands.34 */35 36#define MAX_ADDRESS_CYC 637#define MAX_ERASE_ADDRESS_CYC 338#define MAX_DATA_SIZE 0xFFFC39#define DMA_DATA_SIZE_ALIGN 840 41/* Register definition. */42/*43 * Command register 0.44 * Writing data to this register will initiate a new transaction45 * of the NF controller.46 */47#define CMD_REG0 0x000048/* Command type field mask. */49#define CMD_REG0_CT GENMASK(31, 30)50/* Command type CDMA. */51#define CMD_REG0_CT_CDMA 0uL52/* Command type generic. */53#define CMD_REG0_CT_GEN 3uL54/* Command thread number field mask. */55#define CMD_REG0_TN GENMASK(27, 24)56 57/* Command register 2. */58#define CMD_REG2 0x000859/* Command register 3. */60#define CMD_REG3 0x000C61/* Pointer register to select which thread status will be selected. */62#define CMD_STATUS_PTR 0x001063/* Command status register for selected thread. */64#define CMD_STATUS 0x001465 66/* Interrupt status register. */67#define INTR_STATUS 0x011068#define INTR_STATUS_SDMA_ERR BIT(22)69#define INTR_STATUS_SDMA_TRIGG BIT(21)70#define INTR_STATUS_UNSUPP_CMD BIT(19)71#define INTR_STATUS_DDMA_TERR BIT(18)72#define INTR_STATUS_CDMA_TERR BIT(17)73#define INTR_STATUS_CDMA_IDL BIT(16)74 75/* Interrupt enable register. */76#define INTR_ENABLE 0x011477#define INTR_ENABLE_INTR_EN BIT(31)78#define INTR_ENABLE_SDMA_ERR_EN BIT(22)79#define INTR_ENABLE_SDMA_TRIGG_EN BIT(21)80#define INTR_ENABLE_UNSUPP_CMD_EN BIT(19)81#define INTR_ENABLE_DDMA_TERR_EN BIT(18)82#define INTR_ENABLE_CDMA_TERR_EN BIT(17)83#define INTR_ENABLE_CDMA_IDLE_EN BIT(16)84 85/* Controller internal state. */86#define CTRL_STATUS 0x011887#define CTRL_STATUS_INIT_COMP BIT(9)88#define CTRL_STATUS_CTRL_BUSY BIT(8)89 90/* Command Engine threads state. */91#define TRD_STATUS 0x012092 93/* Command Engine interrupt thread error status. */94#define TRD_ERR_INT_STATUS 0x012895/* Command Engine interrupt thread error enable. */96#define TRD_ERR_INT_STATUS_EN 0x013097/* Command Engine interrupt thread complete status. */98#define TRD_COMP_INT_STATUS 0x013899 100/*101 * Transfer config 0 register.102 * Configures data transfer parameters.103 */104#define TRAN_CFG_0 0x0400105/* Offset value from the beginning of the page. */106#define TRAN_CFG_0_OFFSET GENMASK(31, 16)107/* Numbers of sectors to transfer within singlNF device's page. */108#define TRAN_CFG_0_SEC_CNT GENMASK(7, 0)109 110/*111 * Transfer config 1 register.112 * Configures data transfer parameters.113 */114#define TRAN_CFG_1 0x0404115/* Size of last data sector. */116#define TRAN_CFG_1_LAST_SEC_SIZE GENMASK(31, 16)117/* Size of not-last data sector. */118#define TRAN_CFG_1_SECTOR_SIZE GENMASK(15, 0)119 120/* ECC engine configuration register 0. */121#define ECC_CONFIG_0 0x0428122/* Correction strength. */123#define ECC_CONFIG_0_CORR_STR GENMASK(10, 8)124/* Enable erased pages detection mechanism. */125#define ECC_CONFIG_0_ERASE_DET_EN BIT(1)126/* Enable controller ECC check bits generation and correction. */127#define ECC_CONFIG_0_ECC_EN BIT(0)128 129/* ECC engine configuration register 1. */130#define ECC_CONFIG_1 0x042C131 132/* Multiplane settings register. */133#define MULTIPLANE_CFG 0x0434134/* Cache operation settings. */135#define CACHE_CFG 0x0438136 137/* DMA settings register. */138#define DMA_SETINGS 0x043C139/* Enable SDMA error report on access unprepared slave DMA interface. */140#define DMA_SETINGS_SDMA_ERR_RSP BIT(17)141 142/* Transferred data block size for the slave DMA module. */143#define SDMA_SIZE 0x0440144 145/* Thread number associated with transferred data block146 * for the slave DMA module.147 */148#define SDMA_TRD_NUM 0x0444149/* Thread number mask. */150#define SDMA_TRD_NUM_SDMA_TRD GENMASK(2, 0)151 152#define CONTROL_DATA_CTRL 0x0494153/* Thread number mask. */154#define CONTROL_DATA_CTRL_SIZE GENMASK(15, 0)155 156#define CTRL_VERSION 0x800157#define CTRL_VERSION_REV GENMASK(7, 0)158 159/* Available hardware features of the controller. */160#define CTRL_FEATURES 0x804161/* Support for NV-DDR2/3 work mode. */162#define CTRL_FEATURES_NVDDR_2_3 BIT(28)163/* Support for NV-DDR work mode. */164#define CTRL_FEATURES_NVDDR BIT(27)165/* Support for asynchronous work mode. */166#define CTRL_FEATURES_ASYNC BIT(26)167/* Support for asynchronous work mode. */168#define CTRL_FEATURES_N_BANKS GENMASK(25, 24)169/* Slave and Master DMA data width. */170#define CTRL_FEATURES_DMA_DWITH64 BIT(21)171/* Availability of Control Data feature.*/172#define CTRL_FEATURES_CONTROL_DATA BIT(10)173 174/* BCH Engine identification register 0 - correction strengths. */175#define BCH_CFG_0 0x838176#define BCH_CFG_0_CORR_CAP_0 GENMASK(7, 0)177#define BCH_CFG_0_CORR_CAP_1 GENMASK(15, 8)178#define BCH_CFG_0_CORR_CAP_2 GENMASK(23, 16)179#define BCH_CFG_0_CORR_CAP_3 GENMASK(31, 24)180 181/* BCH Engine identification register 1 - correction strengths. */182#define BCH_CFG_1 0x83C183#define BCH_CFG_1_CORR_CAP_4 GENMASK(7, 0)184#define BCH_CFG_1_CORR_CAP_5 GENMASK(15, 8)185#define BCH_CFG_1_CORR_CAP_6 GENMASK(23, 16)186#define BCH_CFG_1_CORR_CAP_7 GENMASK(31, 24)187 188/* BCH Engine identification register 2 - sector sizes. */189#define BCH_CFG_2 0x840190#define BCH_CFG_2_SECT_0 GENMASK(15, 0)191#define BCH_CFG_2_SECT_1 GENMASK(31, 16)192 193/* BCH Engine identification register 3. */194#define BCH_CFG_3 0x844195#define BCH_CFG_3_METADATA_SIZE GENMASK(23, 16)196 197/* Ready/Busy# line status. */198#define RBN_SETINGS 0x1004199 200/* Common settings. */201#define COMMON_SET 0x1008202/* 16 bit device connected to the NAND Flash interface. */203#define COMMON_SET_DEVICE_16BIT BIT(8)204 205/* Skip_bytes registers. */206#define SKIP_BYTES_CONF 0x100C207#define SKIP_BYTES_MARKER_VALUE GENMASK(31, 16)208#define SKIP_BYTES_NUM_OF_BYTES GENMASK(7, 0)209 210#define SKIP_BYTES_OFFSET 0x1010211#define SKIP_BYTES_OFFSET_VALUE GENMASK(23, 0)212 213/* Timings configuration. */214#define ASYNC_TOGGLE_TIMINGS 0x101c215#define ASYNC_TOGGLE_TIMINGS_TRH GENMASK(28, 24)216#define ASYNC_TOGGLE_TIMINGS_TRP GENMASK(20, 16)217#define ASYNC_TOGGLE_TIMINGS_TWH GENMASK(12, 8)218#define ASYNC_TOGGLE_TIMINGS_TWP GENMASK(4, 0)219 220#define TIMINGS0 0x1024221#define TIMINGS0_TADL GENMASK(31, 24)222#define TIMINGS0_TCCS GENMASK(23, 16)223#define TIMINGS0_TWHR GENMASK(15, 8)224#define TIMINGS0_TRHW GENMASK(7, 0)225 226#define TIMINGS1 0x1028227#define TIMINGS1_TRHZ GENMASK(31, 24)228#define TIMINGS1_TWB GENMASK(23, 16)229#define TIMINGS1_TVDLY GENMASK(7, 0)230 231#define TIMINGS2 0x102c232#define TIMINGS2_TFEAT GENMASK(25, 16)233#define TIMINGS2_CS_HOLD_TIME GENMASK(13, 8)234#define TIMINGS2_CS_SETUP_TIME GENMASK(5, 0)235 236/* Configuration of the resynchronization of slave DLL of PHY. */237#define DLL_PHY_CTRL 0x1034238#define DLL_PHY_CTRL_DLL_RST_N BIT(24)239#define DLL_PHY_CTRL_EXTENDED_WR_MODE BIT(17)240#define DLL_PHY_CTRL_EXTENDED_RD_MODE BIT(16)241#define DLL_PHY_CTRL_RS_HIGH_WAIT_CNT GENMASK(11, 8)242#define DLL_PHY_CTRL_RS_IDLE_CNT GENMASK(7, 0)243 244/* Register controlling DQ related timing. */245#define PHY_DQ_TIMING 0x2000246/* Register controlling DSQ related timing. */247#define PHY_DQS_TIMING 0x2004248#define PHY_DQS_TIMING_DQS_SEL_OE_END GENMASK(3, 0)249#define PHY_DQS_TIMING_PHONY_DQS_SEL BIT(16)250#define PHY_DQS_TIMING_USE_PHONY_DQS BIT(20)251 252/* Register controlling the gate and loopback control related timing. */253#define PHY_GATE_LPBK_CTRL 0x2008254#define PHY_GATE_LPBK_CTRL_RDS GENMASK(24, 19)255 256/* Register holds the control for the master DLL logic. */257#define PHY_DLL_MASTER_CTRL 0x200C258#define PHY_DLL_MASTER_CTRL_BYPASS_MODE BIT(23)259 260/* Register holds the control for the slave DLL logic. */261#define PHY_DLL_SLAVE_CTRL 0x2010262 263/* This register handles the global control settings for the PHY. */264#define PHY_CTRL 0x2080265#define PHY_CTRL_SDR_DQS BIT(14)266#define PHY_CTRL_PHONY_DQS GENMASK(9, 4)267 268/*269 * This register handles the global control settings270 * for the termination selects for reads.271 */272#define PHY_TSEL 0x2084273 274/* Generic command layout. */275#define GCMD_LAY_CS GENMASK_ULL(11, 8)276/*277 * This bit informs the minicotroller if it has to wait for tWB278 * after sending the last CMD/ADDR/DATA in the sequence.279 */280#define GCMD_LAY_TWB BIT_ULL(6)281/* Type of generic instruction. */282#define GCMD_LAY_INSTR GENMASK_ULL(5, 0)283 284/* Generic CMD sequence type. */285#define GCMD_LAY_INSTR_CMD 0286/* Generic ADDR sequence type. */287#define GCMD_LAY_INSTR_ADDR 1288/* Generic data transfer sequence type. */289#define GCMD_LAY_INSTR_DATA 2290 291/* Input part of generic command type of input is command. */292#define GCMD_LAY_INPUT_CMD GENMASK_ULL(23, 16)293 294/* Generic command address sequence - address fields. */295#define GCMD_LAY_INPUT_ADDR GENMASK_ULL(63, 16)296/* Generic command address sequence - address size. */297#define GCMD_LAY_INPUT_ADDR_SIZE GENMASK_ULL(13, 11)298 299/* Transfer direction field of generic command data sequence. */300#define GCMD_DIR BIT_ULL(11)301/* Read transfer direction of generic command data sequence. */302#define GCMD_DIR_READ 0303/* Write transfer direction of generic command data sequence. */304#define GCMD_DIR_WRITE 1305 306/* ECC enabled flag of generic command data sequence - ECC enabled. */307#define GCMD_ECC_EN BIT_ULL(12)308/* Generic command data sequence - sector size. */309#define GCMD_SECT_SIZE GENMASK_ULL(31, 16)310/* Generic command data sequence - sector count. */311#define GCMD_SECT_CNT GENMASK_ULL(39, 32)312/* Generic command data sequence - last sector size. */313#define GCMD_LAST_SIZE GENMASK_ULL(55, 40)314 315/* CDMA descriptor fields. */316/* Erase command type of CDMA descriptor. */317#define CDMA_CT_ERASE 0x1000318/* Program page command type of CDMA descriptor. */319#define CDMA_CT_WR 0x2100320/* Read page command type of CDMA descriptor. */321#define CDMA_CT_RD 0x2200322 323/* Flash pointer memory shift. */324#define CDMA_CFPTR_MEM_SHIFT 24325/* Flash pointer memory mask. */326#define CDMA_CFPTR_MEM GENMASK(26, 24)327 328/*329 * Command DMA descriptor flags. If set causes issue interrupt after330 * the completion of descriptor processing.331 */332#define CDMA_CF_INT BIT(8)333/*334 * Command DMA descriptor flags - the next descriptor335 * address field is valid and descriptor processing should continue.336 */337#define CDMA_CF_CONT BIT(9)338/* DMA master flag of command DMA descriptor. */339#define CDMA_CF_DMA_MASTER BIT(10)340 341/* Operation complete status of command descriptor. */342#define CDMA_CS_COMP BIT(15)343/* Operation complete status of command descriptor. */344/* Command descriptor status - operation fail. */345#define CDMA_CS_FAIL BIT(14)346/* Command descriptor status - page erased. */347#define CDMA_CS_ERP BIT(11)348/* Command descriptor status - timeout occurred. */349#define CDMA_CS_TOUT BIT(10)350/*351 * Maximum amount of correction applied to one ECC sector.352 * It is part of command descriptor status.353 */354#define CDMA_CS_MAXERR GENMASK(9, 2)355/* Command descriptor status - uncorrectable ECC error. */356#define CDMA_CS_UNCE BIT(1)357/* Command descriptor status - descriptor error. */358#define CDMA_CS_ERR BIT(0)359 360/* Status of operation - OK. */361#define STAT_OK 0362/* Status of operation - FAIL. */363#define STAT_FAIL 2364/* Status of operation - uncorrectable ECC error. */365#define STAT_ECC_UNCORR 3366/* Status of operation - page erased. */367#define STAT_ERASED 5368/* Status of operation - correctable ECC error. */369#define STAT_ECC_CORR 6370/* Status of operation - unsuspected state. */371#define STAT_UNKNOWN 7372/* Status of operation - operation is not completed yet. */373#define STAT_BUSY 0xFF374 375#define BCH_MAX_NUM_CORR_CAPS 8376#define BCH_MAX_NUM_SECTOR_SIZES 2377 378struct cadence_nand_timings {379 u32 async_toggle_timings;380 u32 timings0;381 u32 timings1;382 u32 timings2;383 u32 dll_phy_ctrl;384 u32 phy_ctrl;385 u32 phy_dqs_timing;386 u32 phy_gate_lpbk_ctrl;387};388 389/* Command DMA descriptor. */390struct cadence_nand_cdma_desc {391 /* Next descriptor address. */392 u64 next_pointer;393 394 /* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */395 u32 flash_pointer;396 /*field appears in HPNFC version 13*/397 u16 bank;398 u16 rsvd0;399 400 /* Operation the controller needs to perform. */401 u16 command_type;402 u16 rsvd1;403 /* Flags for operation of this command. */404 u16 command_flags;405 u16 rsvd2;406 407 /* System/host memory address required for data DMA commands. */408 u64 memory_pointer;409 410 /* Status of operation. */411 u32 status;412 u32 rsvd3;413 414 /* Address pointer to sync buffer location. */415 u64 sync_flag_pointer;416 417 /* Controls the buffer sync mechanism. */418 u32 sync_arguments;419 u32 rsvd4;420 421 /* Control data pointer. */422 u64 ctrl_data_ptr;423};424 425/* Interrupt status. */426struct cadence_nand_irq_status {427 /* Thread operation complete status. */428 u32 trd_status;429 /* Thread operation error. */430 u32 trd_error;431 /* Controller status. */432 u32 status;433};434 435/* Cadence NAND flash controller capabilities get from driver data. */436struct cadence_nand_dt_devdata {437 /* Skew value of the output signals of the NAND Flash interface. */438 u32 if_skew;439 /* It informs if slave DMA interface is connected to DMA engine. */440 unsigned int has_dma:1;441};442 443/* Cadence NAND flash controller capabilities read from registers. */444struct cdns_nand_caps {445 /* Maximum number of banks supported by hardware. */446 u8 max_banks;447 /* Slave and Master DMA data width in bytes (4 or 8). */448 u8 data_dma_width;449 /* Control Data feature supported. */450 bool data_control_supp;451 /* Is PHY type DLL. */452 bool is_phy_type_dll;453};454 455struct cdns_nand_ctrl {456 struct device *dev;457 struct nand_controller controller;458 struct cadence_nand_cdma_desc *cdma_desc;459 /* IP capability. */460 const struct cadence_nand_dt_devdata *caps1;461 struct cdns_nand_caps caps2;462 u8 ctrl_rev;463 dma_addr_t dma_cdma_desc;464 u8 *buf;465 u32 buf_size;466 u8 curr_corr_str_idx;467 468 /* Register interface. */469 void __iomem *reg;470 471 struct {472 void __iomem *virt;473 dma_addr_t dma;474 } io;475 476 int irq;477 /* Interrupts that have happened. */478 struct cadence_nand_irq_status irq_status;479 /* Interrupts we are waiting for. */480 struct cadence_nand_irq_status irq_mask;481 struct completion complete;482 /* Protect irq_mask and irq_status. */483 spinlock_t irq_lock;484 485 int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];486 struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];487 struct nand_ecc_caps ecc_caps;488 489 int curr_trans_type;490 491 struct dma_chan *dmac;492 493 u32 nf_clk_rate;494 /*495 * Estimated Board delay. The value includes the total496 * round trip delay for the signals and is used for deciding on values497 * associated with data read capture.498 */499 u32 board_delay;500 501 struct nand_chip *selected_chip;502 503 unsigned long assigned_cs;504 struct list_head chips;505 u8 bch_metadata_size;506};507 508struct cdns_nand_chip {509 struct cadence_nand_timings timings;510 struct nand_chip chip;511 u8 nsels;512 struct list_head node;513 514 /*515 * part of oob area of NAND flash memory page.516 * This part is available for user to read or write.517 */518 u32 avail_oob_size;519 520 /* Sector size. There are few sectors per mtd->writesize */521 u32 sector_size;522 u32 sector_count;523 524 /* Offset of BBM. */525 u8 bbm_offs;526 /* Number of bytes reserved for BBM. */527 u8 bbm_len;528 /* ECC strength index. */529 u8 corr_str_idx;530 531 u8 cs[] __counted_by(nsels);532};533 534static inline struct535cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)536{537 return container_of(chip, struct cdns_nand_chip, chip);538}539 540static inline struct541cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)542{543 return container_of(controller, struct cdns_nand_ctrl, controller);544}545 546static bool547cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,548 u32 buf_len)549{550 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;551 552 return buf && virt_addr_valid(buf) &&553 likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&554 likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));555}556 557static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,558 u32 reg_offset, u32 timeout_us,559 u32 mask, bool is_clear)560{561 u32 val;562 int ret;563 564 ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,565 val, !(val & mask) == is_clear,566 10, timeout_us);567 568 if (ret < 0) {569 dev_err(cdns_ctrl->dev,570 "Timeout while waiting for reg %x with mask %x is clear %d\n",571 reg_offset, mask, is_clear);572 }573 574 return ret;575}576 577static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,578 bool enable)579{580 u32 reg;581 582 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,583 1000000,584 CTRL_STATUS_CTRL_BUSY, true))585 return -ETIMEDOUT;586 587 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);588 589 if (enable)590 reg |= ECC_CONFIG_0_ECC_EN;591 else592 reg &= ~ECC_CONFIG_0_ECC_EN;593 594 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);595 596 return 0;597}598 599static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,600 u8 corr_str_idx)601{602 u32 reg;603 604 if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)605 return;606 607 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);608 reg &= ~ECC_CONFIG_0_CORR_STR;609 reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);610 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);611 612 cdns_ctrl->curr_corr_str_idx = corr_str_idx;613}614 615static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,616 u8 strength)617{618 int i, corr_str_idx = -1;619 620 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {621 if (cdns_ctrl->ecc_strengths[i] == strength) {622 corr_str_idx = i;623 break;624 }625 }626 627 return corr_str_idx;628}629 630static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,631 u16 marker_value)632{633 u32 reg;634 635 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,636 1000000,637 CTRL_STATUS_CTRL_BUSY, true))638 return -ETIMEDOUT;639 640 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);641 reg &= ~SKIP_BYTES_MARKER_VALUE;642 reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,643 marker_value);644 645 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);646 647 return 0;648}649 650static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,651 u8 num_of_bytes,652 u32 offset_value,653 int enable)654{655 u32 reg, skip_bytes_offset;656 657 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,658 1000000,659 CTRL_STATUS_CTRL_BUSY, true))660 return -ETIMEDOUT;661 662 if (!enable) {663 num_of_bytes = 0;664 offset_value = 0;665 }666 667 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);668 reg &= ~SKIP_BYTES_NUM_OF_BYTES;669 reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,670 num_of_bytes);671 skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,672 offset_value);673 674 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);675 writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);676 677 return 0;678}679 680/* Functions enables/disables hardware detection of erased data */681static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,682 bool enable,683 u8 bitflips_threshold)684{685 u32 reg;686 687 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);688 689 if (enable)690 reg |= ECC_CONFIG_0_ERASE_DET_EN;691 else692 reg &= ~ECC_CONFIG_0_ERASE_DET_EN;693 694 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);695 writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);696}697 698static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,699 bool bit_bus16)700{701 u32 reg;702 703 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,704 1000000,705 CTRL_STATUS_CTRL_BUSY, true))706 return -ETIMEDOUT;707 708 reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);709 710 if (!bit_bus16)711 reg &= ~COMMON_SET_DEVICE_16BIT;712 else713 reg |= COMMON_SET_DEVICE_16BIT;714 writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);715 716 return 0;717}718 719static void720cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,721 struct cadence_nand_irq_status *irq_status)722{723 writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);724 writel_relaxed(irq_status->trd_status,725 cdns_ctrl->reg + TRD_COMP_INT_STATUS);726 writel_relaxed(irq_status->trd_error,727 cdns_ctrl->reg + TRD_ERR_INT_STATUS);728}729 730static void731cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,732 struct cadence_nand_irq_status *irq_status)733{734 irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);735 irq_status->trd_status = readl_relaxed(cdns_ctrl->reg736 + TRD_COMP_INT_STATUS);737 irq_status->trd_error = readl_relaxed(cdns_ctrl->reg738 + TRD_ERR_INT_STATUS);739}740 741static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,742 struct cadence_nand_irq_status *irq_status)743{744 cadence_nand_read_int_status(cdns_ctrl, irq_status);745 746 return irq_status->status || irq_status->trd_status ||747 irq_status->trd_error;748}749 750static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)751{752 unsigned long flags;753 754 spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);755 memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));756 memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));757 spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);758}759 760/*761 * This is the interrupt service routine. It handles all interrupts762 * sent to this device.763 */764static irqreturn_t cadence_nand_isr(int irq, void *dev_id)765{766 struct cdns_nand_ctrl *cdns_ctrl = dev_id;767 struct cadence_nand_irq_status irq_status;768 irqreturn_t result = IRQ_NONE;769 770 spin_lock(&cdns_ctrl->irq_lock);771 772 if (irq_detected(cdns_ctrl, &irq_status)) {773 /* Handle interrupt. */774 /* First acknowledge it. */775 cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);776 /* Status in the device context for someone to read. */777 cdns_ctrl->irq_status.status |= irq_status.status;778 cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;779 cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;780 /* Notify anyone who cares that it happened. */781 complete(&cdns_ctrl->complete);782 /* Tell the OS that we've handled this. */783 result = IRQ_HANDLED;784 }785 spin_unlock(&cdns_ctrl->irq_lock);786 787 return result;788}789 790static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,791 struct cadence_nand_irq_status *irq_mask)792{793 writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,794 cdns_ctrl->reg + INTR_ENABLE);795 796 writel_relaxed(irq_mask->trd_error,797 cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);798}799 800static void801cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,802 struct cadence_nand_irq_status *irq_mask,803 struct cadence_nand_irq_status *irq_status)804{805 unsigned long timeout = msecs_to_jiffies(10000);806 unsigned long time_left;807 808 time_left = wait_for_completion_timeout(&cdns_ctrl->complete,809 timeout);810 811 *irq_status = cdns_ctrl->irq_status;812 if (time_left == 0) {813 /* Timeout error. */814 dev_err(cdns_ctrl->dev, "timeout occurred:\n");815 dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",816 irq_status->status, irq_mask->status);817 dev_err(cdns_ctrl->dev,818 "\ttrd_status = 0x%x, trd_status mask = 0x%x\n",819 irq_status->trd_status, irq_mask->trd_status);820 dev_err(cdns_ctrl->dev,821 "\t trd_error = 0x%x, trd_error mask = 0x%x\n",822 irq_status->trd_error, irq_mask->trd_error);823 }824}825 826/* Execute generic command on NAND controller. */827static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,828 u8 chip_nr,829 u64 mini_ctrl_cmd)830{831 u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;832 833 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);834 mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;835 mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;836 837 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,838 1000000,839 CTRL_STATUS_CTRL_BUSY, true))840 return -ETIMEDOUT;841 842 cadence_nand_reset_irq(cdns_ctrl);843 844 writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);845 writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);846 847 /* Select generic command. */848 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);849 /* Thread number. */850 reg |= FIELD_PREP(CMD_REG0_TN, 0);851 852 /* Issue command. */853 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);854 855 return 0;856}857 858/* Wait for data on slave DMA interface. */859static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,860 u8 *out_sdma_trd,861 u32 *out_sdma_size)862{863 struct cadence_nand_irq_status irq_mask, irq_status;864 865 irq_mask.trd_status = 0;866 irq_mask.trd_error = 0;867 irq_mask.status = INTR_STATUS_SDMA_TRIGG868 | INTR_STATUS_SDMA_ERR869 | INTR_STATUS_UNSUPP_CMD;870 871 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);872 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);873 if (irq_status.status == 0) {874 dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");875 return -ETIMEDOUT;876 }877 878 if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {879 *out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);880 *out_sdma_trd = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);881 *out_sdma_trd =882 FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);883 } else {884 dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",885 irq_status.status);886 return -EIO;887 }888 889 return 0;890}891 892static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)893{894 u32 reg;895 896 reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);897 898 cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);899 900 if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))901 cdns_ctrl->caps2.data_dma_width = 8;902 else903 cdns_ctrl->caps2.data_dma_width = 4;904 905 if (reg & CTRL_FEATURES_CONTROL_DATA)906 cdns_ctrl->caps2.data_control_supp = true;907 908 if (reg & (CTRL_FEATURES_NVDDR_2_3909 | CTRL_FEATURES_NVDDR))910 cdns_ctrl->caps2.is_phy_type_dll = true;911}912 913/* Prepare CDMA descriptor. */914static void915cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,916 char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,917 dma_addr_t ctrl_data_ptr, u16 ctype)918{919 struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;920 921 memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));922 923 /* Set fields for one descriptor. */924 cdma_desc->flash_pointer = flash_ptr;925 if (cdns_ctrl->ctrl_rev >= 13)926 cdma_desc->bank = nf_mem;927 else928 cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);929 930 cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;931 cdma_desc->command_flags |= CDMA_CF_INT;932 933 cdma_desc->memory_pointer = mem_ptr;934 cdma_desc->status = 0;935 cdma_desc->sync_flag_pointer = 0;936 cdma_desc->sync_arguments = 0;937 938 cdma_desc->command_type = ctype;939 cdma_desc->ctrl_data_ptr = ctrl_data_ptr;940}941 942static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,943 u32 desc_status)944{945 if (desc_status & CDMA_CS_ERP)946 return STAT_ERASED;947 948 if (desc_status & CDMA_CS_UNCE)949 return STAT_ECC_UNCORR;950 951 if (desc_status & CDMA_CS_ERR) {952 dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");953 return STAT_FAIL;954 }955 956 if (FIELD_GET(CDMA_CS_MAXERR, desc_status))957 return STAT_ECC_CORR;958 959 return STAT_FAIL;960}961 962static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)963{964 struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;965 u8 status = STAT_BUSY;966 967 if (desc_ptr->status & CDMA_CS_FAIL) {968 status = cadence_nand_check_desc_error(cdns_ctrl,969 desc_ptr->status);970 dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);971 } else if (desc_ptr->status & CDMA_CS_COMP) {972 /* Descriptor finished with no errors. */973 if (desc_ptr->command_flags & CDMA_CF_CONT) {974 dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");975 status = STAT_UNKNOWN;976 } else {977 /* Last descriptor. */978 status = STAT_OK;979 }980 }981 982 return status;983}984 985static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,986 u8 thread)987{988 u32 reg;989 int status;990 991 /* Wait for thread ready. */992 status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,993 1000000,994 BIT(thread), true);995 if (status)996 return status;997 998 cadence_nand_reset_irq(cdns_ctrl);999 reinit_completion(&cdns_ctrl->complete);1000 1001 writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,1002 cdns_ctrl->reg + CMD_REG2);1003 writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);1004 1005 /* Select CDMA mode. */1006 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);1007 /* Thread number. */1008 reg |= FIELD_PREP(CMD_REG0_TN, thread);1009 /* Issue command. */1010 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);1011 1012 return 0;1013}1014 1015/* Send SDMA command and wait for finish. */1016static u321017cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,1018 u8 thread)1019{1020 struct cadence_nand_irq_status irq_mask, irq_status = {0};1021 int status;1022 1023 irq_mask.trd_status = BIT(thread);1024 irq_mask.trd_error = BIT(thread);1025 irq_mask.status = INTR_STATUS_CDMA_TERR;1026 1027 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);1028 1029 status = cadence_nand_cdma_send(cdns_ctrl, thread);1030 if (status)1031 return status;1032 1033 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);1034 1035 if (irq_status.status == 0 && irq_status.trd_status == 0 &&1036 irq_status.trd_error == 0) {1037 dev_err(cdns_ctrl->dev, "CDMA command timeout\n");1038 return -ETIMEDOUT;1039 }1040 if (irq_status.status & irq_mask.status) {1041 dev_err(cdns_ctrl->dev, "CDMA command failed\n");1042 return -EIO;1043 }1044 1045 return 0;1046}1047 1048/*1049 * ECC size depends on configured ECC strength and on maximum supported1050 * ECC step size.1051 */1052static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)1053{1054 int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);1055 1056 return ALIGN(nbytes, 2);1057}1058 1059#define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \1060 static int \1061 cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \1062 int strength)\1063 {\1064 return cadence_nand_calc_ecc_bytes(max_step_size, strength);\1065 }1066 1067CADENCE_NAND_CALC_ECC_BYTES(256)1068CADENCE_NAND_CALC_ECC_BYTES(512)1069CADENCE_NAND_CALC_ECC_BYTES(1024)1070CADENCE_NAND_CALC_ECC_BYTES(2048)1071CADENCE_NAND_CALC_ECC_BYTES(4096)1072 1073/* Function reads BCH capabilities. */1074static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)1075{1076 struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;1077 int max_step_size = 0, nstrengths, i;1078 u32 reg;1079 1080 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);1081 cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);1082 if (cdns_ctrl->bch_metadata_size < 4) {1083 dev_err(cdns_ctrl->dev,1084 "Driver needs at least 4 bytes of BCH meta data\n");1085 return -EIO;1086 }1087 1088 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);1089 cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);1090 cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);1091 cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);1092 cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);1093 1094 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);1095 cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);1096 cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);1097 cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);1098 cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);1099 1100 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);1101 cdns_ctrl->ecc_stepinfos[0].stepsize =1102 FIELD_GET(BCH_CFG_2_SECT_0, reg);1103 1104 cdns_ctrl->ecc_stepinfos[1].stepsize =1105 FIELD_GET(BCH_CFG_2_SECT_1, reg);1106 1107 nstrengths = 0;1108 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {1109 if (cdns_ctrl->ecc_strengths[i] != 0)1110 nstrengths++;1111 }1112 1113 ecc_caps->nstepinfos = 0;1114 for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {1115 /* ECC strengths are common for all step infos. */1116 cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;1117 cdns_ctrl->ecc_stepinfos[i].strengths =1118 cdns_ctrl->ecc_strengths;1119 1120 if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)1121 ecc_caps->nstepinfos++;1122 1123 if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)1124 max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;1125 }1126 ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];1127 1128 switch (max_step_size) {1129 case 256:1130 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;1131 break;1132 case 512:1133 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;1134 break;1135 case 1024:1136 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;1137 break;1138 case 2048:1139 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;1140 break;1141 case 4096:1142 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;1143 break;1144 default:1145 dev_err(cdns_ctrl->dev,1146 "Unsupported sector size(ecc step size) %d\n",1147 max_step_size);1148 return -EIO;1149 }1150 1151 return 0;1152}1153 1154/* Hardware initialization. */1155static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)1156{1157 int status;1158 u32 reg;1159 1160 status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,1161 1000000,1162 CTRL_STATUS_INIT_COMP, false);1163 if (status)1164 return status;1165 1166 reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);1167 cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);1168 1169 dev_info(cdns_ctrl->dev,1170 "%s: cadence nand controller version reg %x\n",1171 __func__, reg);1172 1173 /* Disable cache and multiplane. */1174 writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);1175 writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);1176 1177 /* Clear all interrupts. */1178 writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);1179 1180 cadence_nand_get_caps(cdns_ctrl);1181 if (cadence_nand_read_bch_caps(cdns_ctrl))1182 return -EIO;1183 1184#ifndef CONFIG_64BIT1185 if (cdns_ctrl->caps2.data_dma_width == 8) {1186 dev_err(cdns_ctrl->dev,1187 "cannot access 64-bit dma on !64-bit architectures");1188 return -EIO;1189 }1190#endif1191 1192 /*1193 * Set IO width access to 8.1194 * It is because during SW device discovering width access1195 * is expected to be 8.1196 */1197 status = cadence_nand_set_access_width16(cdns_ctrl, false);1198 1199 return status;1200}1201 1202#define TT_MAIN_OOB_AREAS 21203#define TT_RAW_PAGE 31204#define TT_BBM 41205#define TT_MAIN_OOB_AREA_EXT 51206 1207/* Prepare size of data to transfer. */1208static void1209cadence_nand_prepare_data_size(struct nand_chip *chip,1210 int transfer_type)1211{1212 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1213 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1214 struct mtd_info *mtd = nand_to_mtd(chip);1215 u32 sec_size = 0, offset = 0, sec_cnt = 1;1216 u32 last_sec_size = cdns_chip->sector_size;1217 u32 data_ctrl_size = 0;1218 u32 reg = 0;1219 1220 if (cdns_ctrl->curr_trans_type == transfer_type)1221 return;1222 1223 switch (transfer_type) {1224 case TT_MAIN_OOB_AREA_EXT:1225 sec_cnt = cdns_chip->sector_count;1226 sec_size = cdns_chip->sector_size;1227 data_ctrl_size = cdns_chip->avail_oob_size;1228 break;1229 case TT_MAIN_OOB_AREAS:1230 sec_cnt = cdns_chip->sector_count;1231 last_sec_size = cdns_chip->sector_size1232 + cdns_chip->avail_oob_size;1233 sec_size = cdns_chip->sector_size;1234 break;1235 case TT_RAW_PAGE:1236 last_sec_size = mtd->writesize + mtd->oobsize;1237 break;1238 case TT_BBM:1239 offset = mtd->writesize + cdns_chip->bbm_offs;1240 last_sec_size = 8;1241 break;1242 }1243 1244 reg = 0;1245 reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);1246 reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);1247 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);1248 1249 reg = 0;1250 reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);1251 reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);1252 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);1253 1254 if (cdns_ctrl->caps2.data_control_supp) {1255 reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);1256 reg &= ~CONTROL_DATA_CTRL_SIZE;1257 reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);1258 writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);1259 }1260 1261 cdns_ctrl->curr_trans_type = transfer_type;1262}1263 1264static int1265cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,1266 int page, void *buf, void *ctrl_dat, u32 buf_size,1267 u32 ctrl_dat_size, enum dma_data_direction dir,1268 bool with_ecc)1269{1270 dma_addr_t dma_buf, dma_ctrl_dat = 0;1271 u8 thread_nr = chip_nr;1272 int status;1273 u16 ctype;1274 1275 if (dir == DMA_FROM_DEVICE)1276 ctype = CDMA_CT_RD;1277 else1278 ctype = CDMA_CT_WR;1279 1280 cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);1281 1282 dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);1283 if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {1284 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");1285 return -EIO;1286 }1287 1288 if (ctrl_dat && ctrl_dat_size) {1289 dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,1290 ctrl_dat_size, dir);1291 if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {1292 dma_unmap_single(cdns_ctrl->dev, dma_buf,1293 buf_size, dir);1294 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");1295 return -EIO;1296 }1297 }1298 1299 cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,1300 dma_buf, dma_ctrl_dat, ctype);1301 1302 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);1303 1304 dma_unmap_single(cdns_ctrl->dev, dma_buf,1305 buf_size, dir);1306 1307 if (ctrl_dat && ctrl_dat_size)1308 dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,1309 ctrl_dat_size, dir);1310 if (status)1311 return status;1312 1313 return cadence_nand_cdma_finish(cdns_ctrl);1314}1315 1316static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,1317 struct cadence_nand_timings *t)1318{1319 writel_relaxed(t->async_toggle_timings,1320 cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);1321 writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);1322 writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);1323 writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);1324 1325 if (cdns_ctrl->caps2.is_phy_type_dll)1326 writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);1327 1328 writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);1329 1330 if (cdns_ctrl->caps2.is_phy_type_dll) {1331 writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);1332 writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);1333 writel_relaxed(t->phy_dqs_timing,1334 cdns_ctrl->reg + PHY_DQS_TIMING);1335 writel_relaxed(t->phy_gate_lpbk_ctrl,1336 cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);1337 writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,1338 cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);1339 writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);1340 }1341}1342 1343static int cadence_nand_select_target(struct nand_chip *chip)1344{1345 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1346 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1347 1348 if (chip == cdns_ctrl->selected_chip)1349 return 0;1350 1351 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,1352 1000000,1353 CTRL_STATUS_CTRL_BUSY, true))1354 return -ETIMEDOUT;1355 1356 cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);1357 1358 cadence_nand_set_ecc_strength(cdns_ctrl,1359 cdns_chip->corr_str_idx);1360 1361 cadence_nand_set_erase_detection(cdns_ctrl, true,1362 chip->ecc.strength);1363 1364 cdns_ctrl->curr_trans_type = -1;1365 cdns_ctrl->selected_chip = chip;1366 1367 return 0;1368}1369 1370static int cadence_nand_erase(struct nand_chip *chip, u32 page)1371{1372 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1373 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1374 int status;1375 u8 thread_nr = cdns_chip->cs[chip->cur_cs];1376 1377 cadence_nand_cdma_desc_prepare(cdns_ctrl,1378 cdns_chip->cs[chip->cur_cs],1379 page, 0, 0,1380 CDMA_CT_ERASE);1381 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);1382 if (status) {1383 dev_err(cdns_ctrl->dev, "erase operation failed\n");1384 return -EIO;1385 }1386 1387 status = cadence_nand_cdma_finish(cdns_ctrl);1388 if (status)1389 return status;1390 1391 return 0;1392}1393 1394static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)1395{1396 int status;1397 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1398 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1399 struct mtd_info *mtd = nand_to_mtd(chip);1400 1401 cadence_nand_prepare_data_size(chip, TT_BBM);1402 1403 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);1404 1405 /*1406 * Read only bad block marker from offset1407 * defined by a memory manufacturer.1408 */1409 status = cadence_nand_cdma_transfer(cdns_ctrl,1410 cdns_chip->cs[chip->cur_cs],1411 page, cdns_ctrl->buf, NULL,1412 mtd->oobsize,1413 0, DMA_FROM_DEVICE, false);1414 if (status) {1415 dev_err(cdns_ctrl->dev, "read BBM failed\n");1416 return -EIO;1417 }1418 1419 memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);1420 1421 return 0;1422}1423 1424static int cadence_nand_write_page(struct nand_chip *chip,1425 const u8 *buf, int oob_required,1426 int page)1427{1428 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1429 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1430 struct mtd_info *mtd = nand_to_mtd(chip);1431 int status;1432 u16 marker_val = 0xFFFF;1433 1434 status = cadence_nand_select_target(chip);1435 if (status)1436 return status;1437 1438 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,1439 mtd->writesize1440 + cdns_chip->bbm_offs,1441 1);1442 1443 if (oob_required) {1444 marker_val = *(u16 *)(chip->oob_poi1445 + cdns_chip->bbm_offs);1446 } else {1447 /* Set oob data to 0xFF. */1448 memset(cdns_ctrl->buf + mtd->writesize, 0xFF,1449 cdns_chip->avail_oob_size);1450 }1451 1452 cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);1453 1454 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);1455 1456 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&1457 cdns_ctrl->caps2.data_control_supp) {1458 u8 *oob;1459 1460 if (oob_required)1461 oob = chip->oob_poi;1462 else1463 oob = cdns_ctrl->buf + mtd->writesize;1464 1465 status = cadence_nand_cdma_transfer(cdns_ctrl,1466 cdns_chip->cs[chip->cur_cs],1467 page, (void *)buf, oob,1468 mtd->writesize,1469 cdns_chip->avail_oob_size,1470 DMA_TO_DEVICE, true);1471 if (status) {1472 dev_err(cdns_ctrl->dev, "write page failed\n");1473 return -EIO;1474 }1475 1476 return 0;1477 }1478 1479 if (oob_required) {1480 /* Transfer the data to the oob area. */1481 memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,1482 cdns_chip->avail_oob_size);1483 }1484 1485 memcpy(cdns_ctrl->buf, buf, mtd->writesize);1486 1487 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);1488 1489 return cadence_nand_cdma_transfer(cdns_ctrl,1490 cdns_chip->cs[chip->cur_cs],1491 page, cdns_ctrl->buf, NULL,1492 mtd->writesize1493 + cdns_chip->avail_oob_size,1494 0, DMA_TO_DEVICE, true);1495}1496 1497static int cadence_nand_write_oob(struct nand_chip *chip, int page)1498{1499 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1500 struct mtd_info *mtd = nand_to_mtd(chip);1501 1502 memset(cdns_ctrl->buf, 0xFF, mtd->writesize);1503 1504 return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);1505}1506 1507static int cadence_nand_write_page_raw(struct nand_chip *chip,1508 const u8 *buf, int oob_required,1509 int page)1510{1511 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1512 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1513 struct mtd_info *mtd = nand_to_mtd(chip);1514 int writesize = mtd->writesize;1515 int oobsize = mtd->oobsize;1516 int ecc_steps = chip->ecc.steps;1517 int ecc_size = chip->ecc.size;1518 int ecc_bytes = chip->ecc.bytes;1519 void *tmp_buf = cdns_ctrl->buf;1520 int oob_skip = cdns_chip->bbm_len;1521 size_t size = writesize + oobsize;1522 int i, pos, len;1523 int status = 0;1524 1525 status = cadence_nand_select_target(chip);1526 if (status)1527 return status;1528 1529 /*1530 * Fill the buffer with 0xff first except the full page transfer.1531 * This simplifies the logic.1532 */1533 if (!buf || !oob_required)1534 memset(tmp_buf, 0xff, size);1535 1536 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);1537 1538 /* Arrange the buffer for syndrome payload/ecc layout. */1539 if (buf) {1540 for (i = 0; i < ecc_steps; i++) {1541 pos = i * (ecc_size + ecc_bytes);1542 len = ecc_size;1543 1544 if (pos >= writesize)1545 pos += oob_skip;1546 else if (pos + len > writesize)1547 len = writesize - pos;1548 1549 memcpy(tmp_buf + pos, buf, len);1550 buf += len;1551 if (len < ecc_size) {1552 len = ecc_size - len;1553 memcpy(tmp_buf + writesize + oob_skip, buf,1554 len);1555 buf += len;1556 }1557 }1558 }1559 1560 if (oob_required) {1561 const u8 *oob = chip->oob_poi;1562 u32 oob_data_offset = (cdns_chip->sector_count - 1) *1563 (cdns_chip->sector_size + chip->ecc.bytes)1564 + cdns_chip->sector_size + oob_skip;1565 1566 /* BBM at the beginning of the OOB area. */1567 memcpy(tmp_buf + writesize, oob, oob_skip);1568 1569 /* OOB free. */1570 memcpy(tmp_buf + oob_data_offset, oob,1571 cdns_chip->avail_oob_size);1572 oob += cdns_chip->avail_oob_size;1573 1574 /* OOB ECC. */1575 for (i = 0; i < ecc_steps; i++) {1576 pos = ecc_size + i * (ecc_size + ecc_bytes);1577 if (i == (ecc_steps - 1))1578 pos += cdns_chip->avail_oob_size;1579 1580 len = ecc_bytes;1581 1582 if (pos >= writesize)1583 pos += oob_skip;1584 else if (pos + len > writesize)1585 len = writesize - pos;1586 1587 memcpy(tmp_buf + pos, oob, len);1588 oob += len;1589 if (len < ecc_bytes) {1590 len = ecc_bytes - len;1591 memcpy(tmp_buf + writesize + oob_skip, oob,1592 len);1593 oob += len;1594 }1595 }1596 }1597 1598 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);1599 1600 return cadence_nand_cdma_transfer(cdns_ctrl,1601 cdns_chip->cs[chip->cur_cs],1602 page, cdns_ctrl->buf, NULL,1603 mtd->writesize +1604 mtd->oobsize,1605 0, DMA_TO_DEVICE, false);1606}1607 1608static int cadence_nand_write_oob_raw(struct nand_chip *chip,1609 int page)1610{1611 return cadence_nand_write_page_raw(chip, NULL, true, page);1612}1613 1614static int cadence_nand_read_page(struct nand_chip *chip,1615 u8 *buf, int oob_required, int page)1616{1617 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1618 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1619 struct mtd_info *mtd = nand_to_mtd(chip);1620 int status = 0;1621 int ecc_err_count = 0;1622 1623 status = cadence_nand_select_target(chip);1624 if (status)1625 return status;1626 1627 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,1628 mtd->writesize1629 + cdns_chip->bbm_offs, 1);1630 1631 /*1632 * If data buffer can be accessed by DMA and data_control feature1633 * is supported then transfer data and oob directly.1634 */1635 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&1636 cdns_ctrl->caps2.data_control_supp) {1637 u8 *oob;1638 1639 if (oob_required)1640 oob = chip->oob_poi;1641 else1642 oob = cdns_ctrl->buf + mtd->writesize;1643 1644 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);1645 status = cadence_nand_cdma_transfer(cdns_ctrl,1646 cdns_chip->cs[chip->cur_cs],1647 page, buf, oob,1648 mtd->writesize,1649 cdns_chip->avail_oob_size,1650 DMA_FROM_DEVICE, true);1651 /* Otherwise use bounce buffer. */1652 } else {1653 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);1654 status = cadence_nand_cdma_transfer(cdns_ctrl,1655 cdns_chip->cs[chip->cur_cs],1656 page, cdns_ctrl->buf,1657 NULL, mtd->writesize1658 + cdns_chip->avail_oob_size,1659 0, DMA_FROM_DEVICE, true);1660 1661 memcpy(buf, cdns_ctrl->buf, mtd->writesize);1662 if (oob_required)1663 memcpy(chip->oob_poi,1664 cdns_ctrl->buf + mtd->writesize,1665 mtd->oobsize);1666 }1667 1668 switch (status) {1669 case STAT_ECC_UNCORR:1670 mtd->ecc_stats.failed++;1671 ecc_err_count++;1672 break;1673 case STAT_ECC_CORR:1674 ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,1675 cdns_ctrl->cdma_desc->status);1676 mtd->ecc_stats.corrected += ecc_err_count;1677 break;1678 case STAT_ERASED:1679 case STAT_OK:1680 break;1681 default:1682 dev_err(cdns_ctrl->dev, "read page failed\n");1683 return -EIO;1684 }1685 1686 if (oob_required)1687 if (cadence_nand_read_bbm(chip, page, chip->oob_poi))1688 return -EIO;1689 1690 return ecc_err_count;1691}1692 1693/* Reads OOB data from the device. */1694static int cadence_nand_read_oob(struct nand_chip *chip, int page)1695{1696 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1697 1698 return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);1699}1700 1701static int cadence_nand_read_page_raw(struct nand_chip *chip,1702 u8 *buf, int oob_required, int page)1703{1704 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);1705 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);1706 struct mtd_info *mtd = nand_to_mtd(chip);1707 int oob_skip = cdns_chip->bbm_len;1708 int writesize = mtd->writesize;1709 int ecc_steps = chip->ecc.steps;1710 int ecc_size = chip->ecc.size;1711 int ecc_bytes = chip->ecc.bytes;1712 void *tmp_buf = cdns_ctrl->buf;1713 int i, pos, len;1714 int status = 0;1715 1716 status = cadence_nand_select_target(chip);1717 if (status)1718 return status;1719 1720 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);1721 1722 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);1723 status = cadence_nand_cdma_transfer(cdns_ctrl,1724 cdns_chip->cs[chip->cur_cs],1725 page, cdns_ctrl->buf, NULL,1726 mtd->writesize1727 + mtd->oobsize,1728 0, DMA_FROM_DEVICE, false);1729 1730 switch (status) {1731 case STAT_ERASED:1732 case STAT_OK:1733 break;1734 default:1735 dev_err(cdns_ctrl->dev, "read raw page failed\n");1736 return -EIO;1737 }1738 1739 /* Arrange the buffer for syndrome payload/ecc layout. */1740 if (buf) {1741 for (i = 0; i < ecc_steps; i++) {1742 pos = i * (ecc_size + ecc_bytes);1743 len = ecc_size;1744 1745 if (pos >= writesize)1746 pos += oob_skip;1747 else if (pos + len > writesize)1748 len = writesize - pos;1749 1750 memcpy(buf, tmp_buf + pos, len);1751 buf += len;1752 if (len < ecc_size) {1753 len = ecc_size - len;1754 memcpy(buf, tmp_buf + writesize + oob_skip,1755 len);1756 buf += len;1757 }1758 }1759 }1760 1761 if (oob_required) {1762 u8 *oob = chip->oob_poi;1763 u32 oob_data_offset = (cdns_chip->sector_count - 1) *1764 (cdns_chip->sector_size + chip->ecc.bytes)1765 + cdns_chip->sector_size + oob_skip;1766 1767 /* OOB free. */1768 memcpy(oob, tmp_buf + oob_data_offset,1769 cdns_chip->avail_oob_size);1770 1771 /* BBM at the beginning of the OOB area. */1772 memcpy(oob, tmp_buf + writesize, oob_skip);1773 1774 oob += cdns_chip->avail_oob_size;1775 1776 /* OOB ECC */1777 for (i = 0; i < ecc_steps; i++) {1778 pos = ecc_size + i * (ecc_size + ecc_bytes);1779 len = ecc_bytes;1780 1781 if (i == (ecc_steps - 1))1782 pos += cdns_chip->avail_oob_size;1783 1784 if (pos >= writesize)1785 pos += oob_skip;1786 else if (pos + len > writesize)1787 len = writesize - pos;1788 1789 memcpy(oob, tmp_buf + pos, len);1790 oob += len;1791 if (len < ecc_bytes) {1792 len = ecc_bytes - len;1793 memcpy(oob, tmp_buf + writesize + oob_skip,1794 len);1795 oob += len;1796 }1797 }1798 }1799 1800 return 0;1801}1802 1803static int cadence_nand_read_oob_raw(struct nand_chip *chip,1804 int page)1805{1806 return cadence_nand_read_page_raw(chip, NULL, true, page);1807}1808 1809static void cadence_nand_slave_dma_transfer_finished(void *data)1810{1811 struct completion *finished = data;1812 1813 complete(finished);1814}1815 1816static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,1817 void *buf,1818 dma_addr_t dev_dma, size_t len,1819 enum dma_data_direction dir)1820{1821 DECLARE_COMPLETION_ONSTACK(finished);1822 struct dma_chan *chan;1823 struct dma_device *dma_dev;1824 dma_addr_t src_dma, dst_dma, buf_dma;1825 struct dma_async_tx_descriptor *tx;1826 dma_cookie_t cookie;1827 1828 chan = cdns_ctrl->dmac;1829 dma_dev = chan->device;1830 1831 buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);1832 if (dma_mapping_error(dma_dev->dev, buf_dma)) {1833 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");1834 goto err;1835 }1836 1837 if (dir == DMA_FROM_DEVICE) {1838 src_dma = cdns_ctrl->io.dma;1839 dst_dma = buf_dma;1840 } else {1841 src_dma = buf_dma;1842 dst_dma = cdns_ctrl->io.dma;1843 }1844 1845 tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,1846 DMA_CTRL_ACK | DMA_PREP_INTERRUPT);1847 if (!tx) {1848 dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");1849 goto err_unmap;1850 }1851 1852 tx->callback = cadence_nand_slave_dma_transfer_finished;1853 tx->callback_param = &finished;1854 1855 cookie = dmaengine_submit(tx);1856 if (dma_submit_error(cookie)) {1857 dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");1858 goto err_unmap;1859 }1860 1861 dma_async_issue_pending(cdns_ctrl->dmac);1862 wait_for_completion(&finished);1863 1864 dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);1865 1866 return 0;1867 1868err_unmap:1869 dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);1870 1871err:1872 dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");1873 1874 return -EIO;1875}1876 1877static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,1878 u8 *buf, int len)1879{1880 u8 thread_nr = 0;1881 u32 sdma_size;1882 int status;1883 1884 /* Wait until slave DMA interface is ready to data transfer. */1885 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);1886 if (status)1887 return status;1888 1889 if (!cdns_ctrl->caps1->has_dma) {1890 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;1891 1892 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;1893 1894 /* read alingment data */1895 if (data_dma_width == 4)1896 ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);1897#ifdef CONFIG_64BIT1898 else1899 readsq(cdns_ctrl->io.virt, buf, len_in_words);1900#endif1901 1902 if (sdma_size > len) {1903 int read_bytes = (data_dma_width == 4) ?1904 len_in_words << 2 : len_in_words << 3;1905 1906 /* read rest data from slave DMA interface if any */1907 if (data_dma_width == 4)1908 ioread32_rep(cdns_ctrl->io.virt,1909 cdns_ctrl->buf,1910 sdma_size / 4 - len_in_words);1911#ifdef CONFIG_64BIT1912 else1913 readsq(cdns_ctrl->io.virt, cdns_ctrl->buf,1914 sdma_size / 8 - len_in_words);1915#endif1916 1917 /* copy rest of data */1918 memcpy(buf + read_bytes, cdns_ctrl->buf,1919 len - read_bytes);1920 }1921 return 0;1922 }1923 1924 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {1925 status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,1926 cdns_ctrl->io.dma,1927 len, DMA_FROM_DEVICE);1928 if (status == 0)1929 return 0;1930 1931 dev_warn(cdns_ctrl->dev,1932 "Slave DMA transfer failed. Try again using bounce buffer.");1933 }1934 1935 /* If DMA transfer is not possible or failed then use bounce buffer. */1936 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,1937 cdns_ctrl->io.dma,1938 sdma_size, DMA_FROM_DEVICE);1939 1940 if (status) {1941 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");1942 return status;1943 }1944 1945 memcpy(buf, cdns_ctrl->buf, len);1946 1947 return 0;1948}1949 1950static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,1951 const u8 *buf, int len)1952{1953 u8 thread_nr = 0;1954 u32 sdma_size;1955 int status;1956 1957 /* Wait until slave DMA interface is ready to data transfer. */1958 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);1959 if (status)1960 return status;1961 1962 if (!cdns_ctrl->caps1->has_dma) {1963 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;1964 1965 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;1966 1967 if (data_dma_width == 4)1968 iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);1969#ifdef CONFIG_64BIT1970 else1971 writesq(cdns_ctrl->io.virt, buf, len_in_words);1972#endif1973 1974 if (sdma_size > len) {1975 int written_bytes = (data_dma_width == 4) ?1976 len_in_words << 2 : len_in_words << 3;1977 1978 /* copy rest of data */1979 memcpy(cdns_ctrl->buf, buf + written_bytes,1980 len - written_bytes);1981 1982 /* write all expected by nand controller data */1983 if (data_dma_width == 4)1984 iowrite32_rep(cdns_ctrl->io.virt,1985 cdns_ctrl->buf,1986 sdma_size / 4 - len_in_words);1987#ifdef CONFIG_64BIT1988 else1989 writesq(cdns_ctrl->io.virt, cdns_ctrl->buf,1990 sdma_size / 8 - len_in_words);1991#endif1992 }1993 1994 return 0;1995 }1996 1997 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {1998 status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,1999 cdns_ctrl->io.dma,2000 len, DMA_TO_DEVICE);2001 if (status == 0)2002 return 0;2003 2004 dev_warn(cdns_ctrl->dev,2005 "Slave DMA transfer failed. Try again using bounce buffer.");2006 }2007 2008 /* If DMA transfer is not possible or failed then use bounce buffer. */2009 memcpy(cdns_ctrl->buf, buf, len);2010 2011 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,2012 cdns_ctrl->io.dma,2013 sdma_size, DMA_TO_DEVICE);2014 2015 if (status)2016 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");2017 2018 return status;2019}2020 2021static int cadence_nand_force_byte_access(struct nand_chip *chip,2022 bool force_8bit)2023{2024 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);2025 2026 /*2027 * Callers of this function do not verify if the NAND is using a 16-bit2028 * an 8-bit bus for normal operations, so we need to take care of that2029 * here by leaving the configuration unchanged if the NAND does not have2030 * the NAND_BUSWIDTH_16 flag set.2031 */2032 if (!(chip->options & NAND_BUSWIDTH_16))2033 return 0;2034 2035 return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);2036}2037 2038static int cadence_nand_cmd_opcode(struct nand_chip *chip,2039 const struct nand_subop *subop)2040{2041 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);2042 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2043 const struct nand_op_instr *instr;2044 unsigned int op_id = 0;2045 u64 mini_ctrl_cmd = 0;2046 int ret;2047 2048 instr = &subop->instrs[op_id];2049 2050 if (instr->delay_ns > 0)2051 mini_ctrl_cmd |= GCMD_LAY_TWB;2052 2053 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,2054 GCMD_LAY_INSTR_CMD);2055 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,2056 instr->ctx.cmd.opcode);2057 2058 ret = cadence_nand_generic_cmd_send(cdns_ctrl,2059 cdns_chip->cs[chip->cur_cs],2060 mini_ctrl_cmd);2061 if (ret)2062 dev_err(cdns_ctrl->dev, "send cmd %x failed\n",2063 instr->ctx.cmd.opcode);2064 2065 return ret;2066}2067 2068static int cadence_nand_cmd_address(struct nand_chip *chip,2069 const struct nand_subop *subop)2070{2071 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);2072 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2073 const struct nand_op_instr *instr;2074 unsigned int op_id = 0;2075 u64 mini_ctrl_cmd = 0;2076 unsigned int offset, naddrs;2077 u64 address = 0;2078 const u8 *addrs;2079 int ret;2080 int i;2081 2082 instr = &subop->instrs[op_id];2083 2084 if (instr->delay_ns > 0)2085 mini_ctrl_cmd |= GCMD_LAY_TWB;2086 2087 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,2088 GCMD_LAY_INSTR_ADDR);2089 2090 offset = nand_subop_get_addr_start_off(subop, op_id);2091 naddrs = nand_subop_get_num_addr_cyc(subop, op_id);2092 addrs = &instr->ctx.addr.addrs[offset];2093 2094 for (i = 0; i < naddrs; i++)2095 address |= (u64)addrs[i] << (8 * i);2096 2097 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,2098 address);2099 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,2100 naddrs - 1);2101 2102 ret = cadence_nand_generic_cmd_send(cdns_ctrl,2103 cdns_chip->cs[chip->cur_cs],2104 mini_ctrl_cmd);2105 if (ret)2106 dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);2107 2108 return ret;2109}2110 2111static int cadence_nand_cmd_erase(struct nand_chip *chip,2112 const struct nand_subop *subop)2113{2114 unsigned int op_id;2115 2116 if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {2117 int i;2118 const struct nand_op_instr *instr = NULL;2119 unsigned int offset, naddrs;2120 const u8 *addrs;2121 u32 page = 0;2122 2123 instr = &subop->instrs[1];2124 offset = nand_subop_get_addr_start_off(subop, 1);2125 naddrs = nand_subop_get_num_addr_cyc(subop, 1);2126 addrs = &instr->ctx.addr.addrs[offset];2127 2128 for (i = 0; i < naddrs; i++)2129 page |= (u32)addrs[i] << (8 * i);2130 2131 return cadence_nand_erase(chip, page);2132 }2133 2134 /*2135 * If it is not an erase operation then handle operation2136 * by calling exec_op function.2137 */2138 for (op_id = 0; op_id < subop->ninstrs; op_id++) {2139 int ret;2140 const struct nand_operation nand_op = {2141 .cs = chip->cur_cs,2142 .instrs = &subop->instrs[op_id],2143 .ninstrs = 1};2144 ret = chip->controller->ops->exec_op(chip, &nand_op, false);2145 if (ret)2146 return ret;2147 }2148 2149 return 0;2150}2151 2152static int cadence_nand_cmd_data(struct nand_chip *chip,2153 const struct nand_subop *subop)2154{2155 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);2156 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2157 const struct nand_op_instr *instr;2158 unsigned int offset, op_id = 0;2159 u64 mini_ctrl_cmd = 0;2160 int len = 0;2161 int ret;2162 2163 instr = &subop->instrs[op_id];2164 2165 if (instr->delay_ns > 0)2166 mini_ctrl_cmd |= GCMD_LAY_TWB;2167 2168 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,2169 GCMD_LAY_INSTR_DATA);2170 2171 if (instr->type == NAND_OP_DATA_OUT_INSTR)2172 mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,2173 GCMD_DIR_WRITE);2174 2175 len = nand_subop_get_data_len(subop, op_id);2176 offset = nand_subop_get_data_start_off(subop, op_id);2177 mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);2178 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);2179 if (instr->ctx.data.force_8bit) {2180 ret = cadence_nand_force_byte_access(chip, true);2181 if (ret) {2182 dev_err(cdns_ctrl->dev,2183 "cannot change byte access generic data cmd failed\n");2184 return ret;2185 }2186 }2187 2188 ret = cadence_nand_generic_cmd_send(cdns_ctrl,2189 cdns_chip->cs[chip->cur_cs],2190 mini_ctrl_cmd);2191 if (ret) {2192 dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");2193 return ret;2194 }2195 2196 if (instr->type == NAND_OP_DATA_IN_INSTR) {2197 void *buf = instr->ctx.data.buf.in + offset;2198 2199 ret = cadence_nand_read_buf(cdns_ctrl, buf, len);2200 } else {2201 const void *buf = instr->ctx.data.buf.out + offset;2202 2203 ret = cadence_nand_write_buf(cdns_ctrl, buf, len);2204 }2205 2206 if (ret) {2207 dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");2208 return ret;2209 }2210 2211 if (instr->ctx.data.force_8bit) {2212 ret = cadence_nand_force_byte_access(chip, false);2213 if (ret) {2214 dev_err(cdns_ctrl->dev,2215 "cannot change byte access generic data cmd failed\n");2216 }2217 }2218 2219 return ret;2220}2221 2222static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,2223 const struct nand_subop *subop)2224{2225 int status;2226 unsigned int op_id = 0;2227 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);2228 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2229 const struct nand_op_instr *instr = &subop->instrs[op_id];2230 u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;2231 2232 status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,2233 timeout_us,2234 BIT(cdns_chip->cs[chip->cur_cs]),2235 false);2236 return status;2237}2238 2239static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(2240 NAND_OP_PARSER_PATTERN(2241 cadence_nand_cmd_erase,2242 NAND_OP_PARSER_PAT_CMD_ELEM(false),2243 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),2244 NAND_OP_PARSER_PAT_CMD_ELEM(false),2245 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),2246 NAND_OP_PARSER_PATTERN(2247 cadence_nand_cmd_opcode,2248 NAND_OP_PARSER_PAT_CMD_ELEM(false)),2249 NAND_OP_PARSER_PATTERN(2250 cadence_nand_cmd_address,2251 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),2252 NAND_OP_PARSER_PATTERN(2253 cadence_nand_cmd_data,2254 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),2255 NAND_OP_PARSER_PATTERN(2256 cadence_nand_cmd_data,2257 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),2258 NAND_OP_PARSER_PATTERN(2259 cadence_nand_cmd_waitrdy,2260 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))2261 );2262 2263static int cadence_nand_exec_op(struct nand_chip *chip,2264 const struct nand_operation *op,2265 bool check_only)2266{2267 if (!check_only) {2268 int status = cadence_nand_select_target(chip);2269 2270 if (status)2271 return status;2272 }2273 2274 return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,2275 check_only);2276}2277 2278static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,2279 struct mtd_oob_region *oobregion)2280{2281 struct nand_chip *chip = mtd_to_nand(mtd);2282 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2283 2284 if (section)2285 return -ERANGE;2286 2287 oobregion->offset = cdns_chip->bbm_len;2288 oobregion->length = cdns_chip->avail_oob_size2289 - cdns_chip->bbm_len;2290 2291 return 0;2292}2293 2294static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,2295 struct mtd_oob_region *oobregion)2296{2297 struct nand_chip *chip = mtd_to_nand(mtd);2298 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2299 2300 if (section)2301 return -ERANGE;2302 2303 oobregion->offset = cdns_chip->avail_oob_size;2304 oobregion->length = chip->ecc.total;2305 2306 return 0;2307}2308 2309static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {2310 .free = cadence_nand_ooblayout_free,2311 .ecc = cadence_nand_ooblayout_ecc,2312};2313 2314static int calc_cycl(u32 timing, u32 clock)2315{2316 if (timing == 0 || clock == 0)2317 return 0;2318 2319 if ((timing % clock) > 0)2320 return timing / clock;2321 else2322 return timing / clock - 1;2323}2324 2325/* Calculate max data valid window. */2326static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,2327 u32 board_delay_skew_min, u32 ext_mode)2328{2329 if (ext_mode == 0)2330 clk_period /= 2;2331 2332 return (trp_cnt + 1) * clk_period + trhoh_min +2333 board_delay_skew_min;2334}2335 2336/* Calculate data valid window. */2337static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,2338 u32 trea_max, u32 ext_mode)2339{2340 if (ext_mode == 0)2341 clk_period /= 2;2342 2343 return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;2344}2345 2346static int2347cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,2348 const struct nand_interface_config *conf)2349{2350 const struct nand_sdr_timings *sdr;2351 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);2352 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2353 struct cadence_nand_timings *t = &cdns_chip->timings;2354 u32 reg;2355 u32 board_delay = cdns_ctrl->board_delay;2356 u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,2357 cdns_ctrl->nf_clk_rate);2358 u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;2359 u32 tfeat_cnt, trhz_cnt, tvdly_cnt;2360 u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;2361 u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;2362 u32 if_skew = cdns_ctrl->caps1->if_skew;2363 u32 board_delay_skew_min = board_delay - if_skew;2364 u32 board_delay_skew_max = board_delay + if_skew;2365 u32 dqs_sampl_res, phony_dqs_mod;2366 u32 tdvw, tdvw_min, tdvw_max;2367 u32 ext_rd_mode, ext_wr_mode;2368 u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;2369 u32 sampling_point;2370 2371 sdr = nand_get_sdr_timings(conf);2372 if (IS_ERR(sdr))2373 return PTR_ERR(sdr);2374 2375 memset(t, 0, sizeof(*t));2376 /* Sampling point calculation. */2377 2378 if (cdns_ctrl->caps2.is_phy_type_dll)2379 phony_dqs_mod = 2;2380 else2381 phony_dqs_mod = 1;2382 2383 dqs_sampl_res = clk_period / phony_dqs_mod;2384 2385 tdvw_min = sdr->tREA_max + board_delay_skew_max;2386 /*2387 * The idea of those calculation is to get the optimum value2388 * for tRP and tRH timings. If it is NOT possible to sample data2389 * with optimal tRP/tRH settings, the parameters will be extended.2390 * If clk_period is 50ns (the lowest value) this condition is met2391 * for SDR timing modes 1, 2, 3, 4 and 5.2392 * If clk_period is 20ns the condition is met only for SDR timing2393 * mode 5.2394 */2395 if (sdr->tRC_min <= clk_period &&2396 sdr->tRP_min <= (clk_period / 2) &&2397 sdr->tREH_min <= (clk_period / 2)) {2398 /* Performance mode. */2399 ext_rd_mode = 0;2400 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,2401 sdr->tREA_max, ext_rd_mode);2402 tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,2403 board_delay_skew_min,2404 ext_rd_mode);2405 /*2406 * Check if data valid window and sampling point can be found2407 * and is not on the edge (ie. we have hold margin).2408 * If not extend the tRP timings.2409 */2410 if (tdvw > 0) {2411 if (tdvw_max <= tdvw_min ||2412 (tdvw_max % dqs_sampl_res) == 0) {2413 /*2414 * No valid sampling point so the RE pulse need2415 * to be widen widening by half clock cycle.2416 */2417 ext_rd_mode = 1;2418 }2419 } else {2420 /*2421 * There is no valid window2422 * to be able to sample data the tRP need to be widen.2423 * Very safe calculations are performed here.2424 */2425 trp_cnt = (sdr->tREA_max + board_delay_skew_max2426 + dqs_sampl_res) / clk_period;2427 ext_rd_mode = 1;2428 }2429 2430 } else {2431 /* Extended read mode. */2432 u32 trh;2433 2434 ext_rd_mode = 1;2435 trp_cnt = calc_cycl(sdr->tRP_min, clk_period);2436 trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);2437 if (sdr->tREH_min >= trh)2438 trh_cnt = calc_cycl(sdr->tREH_min, clk_period);2439 else2440 trh_cnt = calc_cycl(trh, clk_period);2441 2442 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,2443 sdr->tREA_max, ext_rd_mode);2444 /*2445 * Check if data valid window and sampling point can be found2446 * or if it is at the edge check if previous is valid2447 * - if not extend the tRP timings.2448 */2449 if (tdvw > 0) {2450 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,2451 sdr->tRHOH_min,2452 board_delay_skew_min,2453 ext_rd_mode);2454 2455 if ((((tdvw_max / dqs_sampl_res)2456 * dqs_sampl_res) <= tdvw_min) ||2457 (((tdvw_max % dqs_sampl_res) == 0) &&2458 (((tdvw_max / dqs_sampl_res - 1)2459 * dqs_sampl_res) <= tdvw_min))) {2460 /*2461 * Data valid window width is lower than2462 * sampling resolution and do not hit any2463 * sampling point to be sure the sampling point2464 * will be found the RE low pulse width will be2465 * extended by one clock cycle.2466 */2467 trp_cnt = trp_cnt + 1;2468 }2469 } else {2470 /*2471 * There is no valid window to be able to sample data.2472 * The tRP need to be widen.2473 * Very safe calculations are performed here.2474 */2475 trp_cnt = (sdr->tREA_max + board_delay_skew_max2476 + dqs_sampl_res) / clk_period;2477 }2478 }2479 2480 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,2481 sdr->tRHOH_min,2482 board_delay_skew_min, ext_rd_mode);2483 2484 if (sdr->tWC_min <= clk_period &&2485 (sdr->tWP_min + if_skew) <= (clk_period / 2) &&2486 (sdr->tWH_min + if_skew) <= (clk_period / 2)) {2487 ext_wr_mode = 0;2488 } else {2489 u32 twh;2490 2491 ext_wr_mode = 1;2492 twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);2493 if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))2494 twp_cnt = calc_cycl(sdr->tALS_min + if_skew,2495 clk_period);2496 2497 twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);2498 if (sdr->tWH_min >= twh)2499 twh = sdr->tWH_min;2500 2501 twh_cnt = calc_cycl(twh + if_skew, clk_period);2502 }2503 2504 reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);2505 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);2506 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);2507 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);2508 t->async_toggle_timings = reg;2509 dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);2510 2511 tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);2512 tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);2513 twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);2514 trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);2515 reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);2516 2517 /*2518 * If timing exceeds delay field in timing register2519 * then use maximum value.2520 */2521 if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))2522 reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);2523 else2524 reg |= TIMINGS0_TCCS;2525 2526 reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);2527 reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);2528 t->timings0 = reg;2529 dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);2530 2531 /* The following is related to single signal so skew is not needed. */2532 trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);2533 trhz_cnt = trhz_cnt + 1;2534 twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);2535 /*2536 * Because of the two stage syncflop the value must be increased by 32537 * first value is related with sync, second value is related2538 * with output if delay.2539 */2540 twb_cnt = twb_cnt + 3 + 5;2541 /*2542 * The following is related to the we edge of the random data input2543 * sequence so skew is not needed.2544 */2545 tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);2546 reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);2547 reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);2548 reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);2549 t->timings1 = reg;2550 dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);2551 2552 tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);2553 if (tfeat_cnt < twb_cnt)2554 tfeat_cnt = twb_cnt;2555 2556 tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);2557 tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);2558 2559 reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);2560 reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);2561 reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);2562 t->timings2 = reg;2563 dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);2564 2565 if (cdns_ctrl->caps2.is_phy_type_dll) {2566 reg = DLL_PHY_CTRL_DLL_RST_N;2567 if (ext_wr_mode)2568 reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;2569 if (ext_rd_mode)2570 reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;2571 2572 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);2573 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);2574 t->dll_phy_ctrl = reg;2575 dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);2576 }2577 2578 /* Sampling point calculation. */2579 if ((tdvw_max % dqs_sampl_res) > 0)2580 sampling_point = tdvw_max / dqs_sampl_res;2581 else2582 sampling_point = (tdvw_max / dqs_sampl_res - 1);2583 2584 if (sampling_point * dqs_sampl_res > tdvw_min) {2585 dll_phy_dqs_timing =2586 FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);2587 dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;2588 phony_dqs_timing = sampling_point / phony_dqs_mod;2589 2590 if ((sampling_point % 2) > 0) {2591 dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;2592 if ((tdvw_max % dqs_sampl_res) == 0)2593 /*2594 * Calculation for sampling point at the edge2595 * of data and being odd number.2596 */2597 phony_dqs_timing = (tdvw_max / dqs_sampl_res)2598 / phony_dqs_mod - 1;2599 2600 if (!cdns_ctrl->caps2.is_phy_type_dll)2601 phony_dqs_timing--;2602 2603 } else {2604 phony_dqs_timing--;2605 }2606 rd_del_sel = phony_dqs_timing + 3;2607 } else {2608 dev_warn(cdns_ctrl->dev,2609 "ERROR : cannot find valid sampling point\n");2610 }2611 2612 reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);2613 if (cdns_ctrl->caps2.is_phy_type_dll)2614 reg |= PHY_CTRL_SDR_DQS;2615 t->phy_ctrl = reg;2616 dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);2617 2618 if (cdns_ctrl->caps2.is_phy_type_dll) {2619 dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);2620 dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);2621 dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",2622 dll_phy_dqs_timing);2623 t->phy_dqs_timing = dll_phy_dqs_timing;2624 2625 reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);2626 dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",2627 reg);2628 t->phy_gate_lpbk_ctrl = reg;2629 2630 dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",2631 PHY_DLL_MASTER_CTRL_BYPASS_MODE);2632 dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);2633 }2634 2635 return 0;2636}2637 2638static int cadence_nand_attach_chip(struct nand_chip *chip)2639{2640 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);2641 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);2642 u32 ecc_size;2643 struct mtd_info *mtd = nand_to_mtd(chip);2644 int ret;2645 2646 if (chip->options & NAND_BUSWIDTH_16) {2647 ret = cadence_nand_set_access_width16(cdns_ctrl, true);2648 if (ret)2649 return ret;2650 }2651 2652 chip->bbt_options |= NAND_BBT_USE_FLASH;2653 chip->bbt_options |= NAND_BBT_NO_OOB;2654 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;2655 2656 chip->options |= NAND_NO_SUBPAGE_WRITE;2657 2658 cdns_chip->bbm_offs = chip->badblockpos;2659 cdns_chip->bbm_offs &= ~0x01;2660 /* this value should be even number */2661 cdns_chip->bbm_len = 2;2662 2663 ret = nand_ecc_choose_conf(chip,2664 &cdns_ctrl->ecc_caps,2665 mtd->oobsize - cdns_chip->bbm_len);2666 if (ret) {2667 dev_err(cdns_ctrl->dev, "ECC configuration failed\n");2668 return ret;2669 }2670 2671 dev_dbg(cdns_ctrl->dev,2672 "chosen ECC settings: step=%d, strength=%d, bytes=%d\n",2673 chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);2674 2675 /* Error correction configuration. */2676 cdns_chip->sector_size = chip->ecc.size;2677 cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;2678 ecc_size = cdns_chip->sector_count * chip->ecc.bytes;2679 2680 cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;2681 2682 if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)2683 cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;2684 2685 if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)2686 > mtd->oobsize)2687 cdns_chip->avail_oob_size -= 4;2688 2689 ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);2690 if (ret < 0)2691 return -EINVAL;2692 2693 cdns_chip->corr_str_idx = (u8)ret;2694 2695 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,2696 1000000,2697 CTRL_STATUS_CTRL_BUSY, true))2698 return -ETIMEDOUT;2699 2700 cadence_nand_set_ecc_strength(cdns_ctrl,2701 cdns_chip->corr_str_idx);2702 2703 cadence_nand_set_erase_detection(cdns_ctrl, true,2704 chip->ecc.strength);2705 2706 /* Override the default read operations. */2707 chip->ecc.read_page = cadence_nand_read_page;2708 chip->ecc.read_page_raw = cadence_nand_read_page_raw;2709 chip->ecc.write_page = cadence_nand_write_page;2710 chip->ecc.write_page_raw = cadence_nand_write_page_raw;2711 chip->ecc.read_oob = cadence_nand_read_oob;2712 chip->ecc.write_oob = cadence_nand_write_oob;2713 chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;2714 chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;2715 2716 if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)2717 cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;2718 2719 /* Is 32-bit DMA supported? */2720 ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));2721 if (ret) {2722 dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");2723 return ret;2724 }2725 2726 mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);2727 2728 return 0;2729}2730 2731static const struct nand_controller_ops cadence_nand_controller_ops = {2732 .attach_chip = cadence_nand_attach_chip,2733 .exec_op = cadence_nand_exec_op,2734 .setup_interface = cadence_nand_setup_interface,2735};2736 2737static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,2738 struct device_node *np)2739{2740 struct cdns_nand_chip *cdns_chip;2741 struct mtd_info *mtd;2742 struct nand_chip *chip;2743 int nsels, ret, i;2744 u32 cs;2745 2746 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));2747 if (nsels <= 0) {2748 dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");2749 return -EINVAL;2750 }2751 2752 /* Allocate the nand chip structure. */2753 cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +2754 (nsels * sizeof(u8)),2755 GFP_KERNEL);2756 if (!cdns_chip) {2757 dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");2758 return -ENOMEM;2759 }2760 2761 cdns_chip->nsels = nsels;2762 2763 for (i = 0; i < nsels; i++) {2764 /* Retrieve CS id. */2765 ret = of_property_read_u32_index(np, "reg", i, &cs);2766 if (ret) {2767 dev_err(cdns_ctrl->dev,2768 "could not retrieve reg property: %d\n",2769 ret);2770 return ret;2771 }2772 2773 if (cs >= cdns_ctrl->caps2.max_banks) {2774 dev_err(cdns_ctrl->dev,2775 "invalid reg value: %u (max CS = %d)\n",2776 cs, cdns_ctrl->caps2.max_banks);2777 return -EINVAL;2778 }2779 2780 if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {2781 dev_err(cdns_ctrl->dev,2782 "CS %d already assigned\n", cs);2783 return -EINVAL;2784 }2785 2786 cdns_chip->cs[i] = cs;2787 }2788 2789 chip = &cdns_chip->chip;2790 chip->controller = &cdns_ctrl->controller;2791 nand_set_flash_node(chip, np);2792 2793 mtd = nand_to_mtd(chip);2794 mtd->dev.parent = cdns_ctrl->dev;2795 2796 /*2797 * Default to HW ECC engine mode. If the nand-ecc-mode property is given2798 * in the DT node, this entry will be overwritten in nand_scan_ident().2799 */2800 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;2801 2802 ret = nand_scan(chip, cdns_chip->nsels);2803 if (ret) {2804 dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");2805 return ret;2806 }2807 2808 ret = mtd_device_register(mtd, NULL, 0);2809 if (ret) {2810 dev_err(cdns_ctrl->dev,2811 "failed to register mtd device: %d\n", ret);2812 nand_cleanup(chip);2813 return ret;2814 }2815 2816 list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);2817 2818 return 0;2819}2820 2821static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)2822{2823 struct cdns_nand_chip *entry, *temp;2824 struct nand_chip *chip;2825 int ret;2826 2827 list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {2828 chip = &entry->chip;2829 ret = mtd_device_unregister(nand_to_mtd(chip));2830 WARN_ON(ret);2831 nand_cleanup(chip);2832 list_del(&entry->node);2833 }2834}2835 2836static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)2837{2838 struct device_node *np = cdns_ctrl->dev->of_node;2839 int max_cs = cdns_ctrl->caps2.max_banks;2840 int nchips, ret;2841 2842 nchips = of_get_child_count(np);2843 2844 if (nchips > max_cs) {2845 dev_err(cdns_ctrl->dev,2846 "too many NAND chips: %d (max = %d CS)\n",2847 nchips, max_cs);2848 return -EINVAL;2849 }2850 2851 for_each_child_of_node_scoped(np, nand_np) {2852 ret = cadence_nand_chip_init(cdns_ctrl, nand_np);2853 if (ret) {2854 cadence_nand_chips_cleanup(cdns_ctrl);2855 return ret;2856 }2857 }2858 2859 return 0;2860}2861 2862static void2863cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)2864{2865 /* Disable interrupts. */2866 writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);2867}2868 2869static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)2870{2871 dma_cap_mask_t mask;2872 int ret;2873 2874 cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,2875 sizeof(*cdns_ctrl->cdma_desc),2876 &cdns_ctrl->dma_cdma_desc,2877 GFP_KERNEL);2878 if (!cdns_ctrl->dma_cdma_desc)2879 return -ENOMEM;2880 2881 cdns_ctrl->buf_size = SZ_16K;2882 cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);2883 if (!cdns_ctrl->buf) {2884 ret = -ENOMEM;2885 goto free_buf_desc;2886 }2887 2888 if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,2889 IRQF_SHARED, "cadence-nand-controller",2890 cdns_ctrl)) {2891 dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");2892 ret = -ENODEV;2893 goto free_buf;2894 }2895 2896 spin_lock_init(&cdns_ctrl->irq_lock);2897 init_completion(&cdns_ctrl->complete);2898 2899 ret = cadence_nand_hw_init(cdns_ctrl);2900 if (ret)2901 goto disable_irq;2902 2903 dma_cap_zero(mask);2904 dma_cap_set(DMA_MEMCPY, mask);2905 2906 if (cdns_ctrl->caps1->has_dma) {2907 cdns_ctrl->dmac = dma_request_channel(mask, NULL, NULL);2908 if (!cdns_ctrl->dmac) {2909 dev_err(cdns_ctrl->dev,2910 "Unable to get a DMA channel\n");2911 ret = -EBUSY;2912 goto disable_irq;2913 }2914 }2915 2916 nand_controller_init(&cdns_ctrl->controller);2917 INIT_LIST_HEAD(&cdns_ctrl->chips);2918 2919 cdns_ctrl->controller.ops = &cadence_nand_controller_ops;2920 cdns_ctrl->curr_corr_str_idx = 0xFF;2921 2922 ret = cadence_nand_chips_init(cdns_ctrl);2923 if (ret) {2924 dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",2925 ret);2926 goto dma_release_chnl;2927 }2928 2929 kfree(cdns_ctrl->buf);2930 cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);2931 if (!cdns_ctrl->buf) {2932 ret = -ENOMEM;2933 goto dma_release_chnl;2934 }2935 2936 return 0;2937 2938dma_release_chnl:2939 if (cdns_ctrl->dmac)2940 dma_release_channel(cdns_ctrl->dmac);2941 2942disable_irq:2943 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);2944 2945free_buf:2946 kfree(cdns_ctrl->buf);2947 2948free_buf_desc:2949 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),2950 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);2951 2952 return ret;2953}2954 2955/* Driver exit point. */2956static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)2957{2958 cadence_nand_chips_cleanup(cdns_ctrl);2959 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);2960 kfree(cdns_ctrl->buf);2961 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),2962 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);2963 2964 if (cdns_ctrl->dmac)2965 dma_release_channel(cdns_ctrl->dmac);2966}2967 2968struct cadence_nand_dt {2969 struct cdns_nand_ctrl cdns_ctrl;2970 struct clk *clk;2971};2972 2973static const struct cadence_nand_dt_devdata cadence_nand_default = {2974 .if_skew = 0,2975 .has_dma = 1,2976};2977 2978static const struct of_device_id cadence_nand_dt_ids[] = {2979 {2980 .compatible = "cdns,hp-nfc",2981 .data = &cadence_nand_default2982 }, {}2983};2984 2985MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);2986 2987static int cadence_nand_dt_probe(struct platform_device *ofdev)2988{2989 struct resource *res;2990 struct cadence_nand_dt *dt;2991 struct cdns_nand_ctrl *cdns_ctrl;2992 int ret;2993 const struct cadence_nand_dt_devdata *devdata;2994 u32 val;2995 2996 devdata = device_get_match_data(&ofdev->dev);2997 if (!devdata) {2998 pr_err("Failed to find the right device id.\n");2999 return -ENOMEM;3000 }3001 3002 dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);3003 if (!dt)3004 return -ENOMEM;3005 3006 cdns_ctrl = &dt->cdns_ctrl;3007 cdns_ctrl->caps1 = devdata;3008 3009 cdns_ctrl->dev = &ofdev->dev;3010 cdns_ctrl->irq = platform_get_irq(ofdev, 0);3011 if (cdns_ctrl->irq < 0)3012 return cdns_ctrl->irq;3013 3014 dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);3015 3016 cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);3017 if (IS_ERR(cdns_ctrl->reg))3018 return PTR_ERR(cdns_ctrl->reg);3019 3020 cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);3021 if (IS_ERR(cdns_ctrl->io.virt))3022 return PTR_ERR(cdns_ctrl->io.virt);3023 cdns_ctrl->io.dma = res->start;3024 3025 dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");3026 if (IS_ERR(dt->clk))3027 return PTR_ERR(dt->clk);3028 3029 cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);3030 3031 ret = of_property_read_u32(ofdev->dev.of_node,3032 "cdns,board-delay-ps", &val);3033 if (ret) {3034 val = 4830;3035 dev_info(cdns_ctrl->dev,3036 "missing cdns,board-delay-ps property, %d was set\n",3037 val);3038 }3039 cdns_ctrl->board_delay = val;3040 3041 ret = cadence_nand_init(cdns_ctrl);3042 if (ret)3043 return ret;3044 3045 platform_set_drvdata(ofdev, dt);3046 return 0;3047}3048 3049static void cadence_nand_dt_remove(struct platform_device *ofdev)3050{3051 struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);3052 3053 cadence_nand_remove(&dt->cdns_ctrl);3054}3055 3056static struct platform_driver cadence_nand_dt_driver = {3057 .probe = cadence_nand_dt_probe,3058 .remove_new = cadence_nand_dt_remove,3059 .driver = {3060 .name = "cadence-nand-controller",3061 .of_match_table = cadence_nand_dt_ids,3062 },3063};3064 3065module_platform_driver(cadence_nand_dt_driver);3066 3067MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");3068MODULE_LICENSE("GPL v2");3069MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");3070 3071