brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0+2/*3 * Freescale GPMI NAND Flash Driver4 *5 * Copyright (C) 2010-2015 Freescale Semiconductor, Inc.6 * Copyright (C) 2008 Embedded Alley Solutions, Inc.7 */8#include <linux/clk.h>9#include <linux/delay.h>10#include <linux/slab.h>11#include <linux/sched/task_stack.h>12#include <linux/interrupt.h>13#include <linux/module.h>14#include <linux/mtd/partitions.h>15#include <linux/of.h>16#include <linux/platform_device.h>17#include <linux/pm_runtime.h>18#include <linux/dma/mxs-dma.h>19#include "gpmi-nand.h"20#include "gpmi-regs.h"21#include "bch-regs.h"22 23/* Resource names for the GPMI NAND driver. */24#define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME  "gpmi-nand"25#define GPMI_NAND_BCH_REGS_ADDR_RES_NAME   "bch"26#define GPMI_NAND_BCH_INTERRUPT_RES_NAME   "bch"27 28/* Converts time to clock cycles */29#define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period)30 31#define MXS_SET_ADDR		0x432#define MXS_CLR_ADDR		0x833/*34 * Clear the bit and poll it cleared.  This is usually called with35 * a reset address and mask being either SFTRST(bit 31) or CLKGATE36 * (bit 30).37 */38static int clear_poll_bit(void __iomem *addr, u32 mask)39{40	int timeout = 0x400;41 42	/* clear the bit */43	writel(mask, addr + MXS_CLR_ADDR);44 45	/*46	 * SFTRST needs 3 GPMI clocks to settle, the reference manual47	 * recommends to wait 1us.48	 */49	udelay(1);50 51	/* poll the bit becoming clear */52	while ((readl(addr) & mask) && --timeout)53		/* nothing */;54 55	return !timeout;56}57 58#define MODULE_CLKGATE		(1 << 30)59#define MODULE_SFTRST		(1 << 31)60/*61 * The current mxs_reset_block() will do two things:62 *  [1] enable the module.63 *  [2] reset the module.64 *65 * In most of the cases, it's ok.66 * But in MX23, there is a hardware bug in the BCH block (see erratum #2847).67 * If you try to soft reset the BCH block, it becomes unusable until68 * the next hard reset. This case occurs in the NAND boot mode. When the board69 * boots by NAND, the ROM of the chip will initialize the BCH blocks itself.70 * So If the driver tries to reset the BCH again, the BCH will not work anymore.71 * You will see a DMA timeout in this case. The bug has been fixed72 * in the following chips, such as MX28.73 *74 * To avoid this bug, just add a new parameter `just_enable` for75 * the mxs_reset_block(), and rewrite it here.76 */77static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable)78{79	int ret;80	int timeout = 0x400;81 82	/* clear and poll SFTRST */83	ret = clear_poll_bit(reset_addr, MODULE_SFTRST);84	if (unlikely(ret))85		goto error;86 87	/* clear CLKGATE */88	writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR);89 90	if (!just_enable) {91		/* set SFTRST to reset the block */92		writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR);93		udelay(1);94 95		/* poll CLKGATE becoming set */96		while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout)97			/* nothing */;98		if (unlikely(!timeout))99			goto error;100	}101 102	/* clear and poll SFTRST */103	ret = clear_poll_bit(reset_addr, MODULE_SFTRST);104	if (unlikely(ret))105		goto error;106 107	/* clear and poll CLKGATE */108	ret = clear_poll_bit(reset_addr, MODULE_CLKGATE);109	if (unlikely(ret))110		goto error;111 112	return 0;113 114error:115	pr_err("%s(%p): module reset timeout\n", __func__, reset_addr);116	return -ETIMEDOUT;117}118 119static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v)120{121	struct clk *clk;122	int ret;123	int i;124 125	for (i = 0; i < GPMI_CLK_MAX; i++) {126		clk = this->resources.clock[i];127		if (!clk)128			break;129 130		if (v) {131			ret = clk_prepare_enable(clk);132			if (ret)133				goto err_clk;134		} else {135			clk_disable_unprepare(clk);136		}137	}138	return 0;139 140err_clk:141	for (; i > 0; i--)142		clk_disable_unprepare(this->resources.clock[i - 1]);143	return ret;144}145 146static int gpmi_init(struct gpmi_nand_data *this)147{148	struct resources *r = &this->resources;149	int ret;150 151	ret = pm_runtime_resume_and_get(this->dev);152	if (ret < 0)153		return ret;154 155	ret = gpmi_reset_block(r->gpmi_regs, false);156	if (ret)157		goto err_out;158 159	/*160	 * Reset BCH here, too. We got failures otherwise :(161	 * See later BCH reset for explanation of MX23 and MX28 handling162	 */163	ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));164	if (ret)165		goto err_out;166 167	/* Choose NAND mode. */168	writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR);169 170	/* Set the IRQ polarity. */171	writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY,172				r->gpmi_regs + HW_GPMI_CTRL1_SET);173 174	/* Disable Write-Protection. */175	writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET);176 177	/* Select BCH ECC. */178	writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET);179 180	/*181	 * Decouple the chip select from dma channel. We use dma0 for all182	 * the chips, force all NAND RDY_BUSY inputs to be sourced from183	 * RDY_BUSY0.184	 */185	writel(BM_GPMI_CTRL1_DECOUPLE_CS | BM_GPMI_CTRL1_GANGED_RDYBUSY,186	       r->gpmi_regs + HW_GPMI_CTRL1_SET);187 188err_out:189	pm_runtime_mark_last_busy(this->dev);190	pm_runtime_put_autosuspend(this->dev);191	return ret;192}193 194/* This function is very useful. It is called only when the bug occur. */195static void gpmi_dump_info(struct gpmi_nand_data *this)196{197	struct resources *r = &this->resources;198	struct bch_geometry *geo = &this->bch_geometry;199	u32 reg;200	int i;201 202	dev_err(this->dev, "Show GPMI registers :\n");203	for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) {204		reg = readl(r->gpmi_regs + i * 0x10);205		dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);206	}207 208	/* start to print out the BCH info */209	dev_err(this->dev, "Show BCH registers :\n");210	for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) {211		reg = readl(r->bch_regs + i * 0x10);212		dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);213	}214	dev_err(this->dev, "BCH Geometry :\n"215		"GF length              : %u\n"216		"ECC Strength           : %u\n"217		"Page Size in Bytes     : %u\n"218		"Metadata Size in Bytes : %u\n"219		"ECC0 Chunk Size in Bytes: %u\n"220		"ECCn Chunk Size in Bytes: %u\n"221		"ECC Chunk Count        : %u\n"222		"Payload Size in Bytes  : %u\n"223		"Auxiliary Size in Bytes: %u\n"224		"Auxiliary Status Offset: %u\n"225		"Block Mark Byte Offset : %u\n"226		"Block Mark Bit Offset  : %u\n",227		geo->gf_len,228		geo->ecc_strength,229		geo->page_size,230		geo->metadata_size,231		geo->ecc0_chunk_size,232		geo->eccn_chunk_size,233		geo->ecc_chunk_count,234		geo->payload_size,235		geo->auxiliary_size,236		geo->auxiliary_status_offset,237		geo->block_mark_byte_offset,238		geo->block_mark_bit_offset);239}240 241static bool gpmi_check_ecc(struct gpmi_nand_data *this)242{243	struct nand_chip *chip = &this->nand;244	struct bch_geometry *geo = &this->bch_geometry;245	struct nand_device *nand = &chip->base;246	struct nand_ecc_props *conf = &nand->ecc.ctx.conf;247 248	conf->step_size = geo->eccn_chunk_size;249	conf->strength = geo->ecc_strength;250 251	/* Do the sanity check. */252	if (GPMI_IS_MXS(this)) {253		/* The mx23/mx28 only support the GF13. */254		if (geo->gf_len == 14)255			return false;256	}257 258	if (geo->ecc_strength > this->devdata->bch_max_ecc_strength)259		return false;260 261	if (!nand_ecc_is_strong_enough(nand))262		return false;263 264	return true;265}266 267/* check if bbm locates in data chunk rather than ecc chunk */268static bool bbm_in_data_chunk(struct gpmi_nand_data *this,269			unsigned int *chunk_num)270{271	struct bch_geometry *geo = &this->bch_geometry;272	struct nand_chip *chip = &this->nand;273	struct mtd_info *mtd = nand_to_mtd(chip);274	unsigned int i, j;275 276	if (geo->ecc0_chunk_size != geo->eccn_chunk_size) {277		dev_err(this->dev,278			"The size of ecc0_chunk must equal to eccn_chunk\n");279		return false;280	}281 282	i = (mtd->writesize * 8 - geo->metadata_size * 8) /283		(geo->gf_len * geo->ecc_strength +284			geo->eccn_chunk_size * 8);285 286	j = (mtd->writesize * 8 - geo->metadata_size * 8) -287		(geo->gf_len * geo->ecc_strength +288			geo->eccn_chunk_size * 8) * i;289 290	if (j < geo->eccn_chunk_size * 8) {291		*chunk_num = i+1;292		dev_dbg(this->dev, "Set ecc to %d and bbm in chunk %d\n",293				geo->ecc_strength, *chunk_num);294		return true;295	}296 297	return false;298}299 300/*301 * If we can get the ECC information from the nand chip, we do not302 * need to calculate them ourselves.303 *304 * We may have available oob space in this case.305 */306static int set_geometry_by_ecc_info(struct gpmi_nand_data *this,307				    unsigned int ecc_strength,308				    unsigned int ecc_step)309{310	struct bch_geometry *geo = &this->bch_geometry;311	struct nand_chip *chip = &this->nand;312	struct mtd_info *mtd = nand_to_mtd(chip);313	unsigned int block_mark_bit_offset;314 315	switch (ecc_step) {316	case SZ_512:317		geo->gf_len = 13;318		break;319	case SZ_1K:320		geo->gf_len = 14;321		break;322	default:323		dev_err(this->dev,324			"unsupported nand chip. ecc bits : %d, ecc size : %d\n",325			nanddev_get_ecc_requirements(&chip->base)->strength,326			nanddev_get_ecc_requirements(&chip->base)->step_size);327		return -EINVAL;328	}329	geo->ecc0_chunk_size = ecc_step;330	geo->eccn_chunk_size = ecc_step;331	geo->ecc_strength = round_up(ecc_strength, 2);332	if (!gpmi_check_ecc(this))333		return -EINVAL;334 335	/* Keep the C >= O */336	if (geo->eccn_chunk_size < mtd->oobsize) {337		dev_err(this->dev,338			"unsupported nand chip. ecc size: %d, oob size : %d\n",339			ecc_step, mtd->oobsize);340		return -EINVAL;341	}342 343	/* The default value, see comment in the legacy_set_geometry(). */344	geo->metadata_size = 10;345 346	geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;347 348	/*349	 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below:350	 *351	 *    |                          P                            |352	 *    |<----------------------------------------------------->|353	 *    |                                                       |354	 *    |                                        (Block Mark)   |355	 *    |                      P'                      |      | |     |356	 *    |<-------------------------------------------->|  D   | |  O' |357	 *    |                                              |<---->| |<--->|358	 *    V                                              V      V V     V359	 *    +---+----------+-+----------+-+----------+-+----------+-+-----+360	 *    | M |   data   |E|   data   |E|   data   |E|   data   |E|     |361	 *    +---+----------+-+----------+-+----------+-+----------+-+-----+362	 *                                                   ^              ^363	 *                                                   |      O       |364	 *                                                   |<------------>|365	 *                                                   |              |366	 *367	 *	P : the page size for BCH module.368	 *	E : The ECC strength.369	 *	G : the length of Galois Field.370	 *	N : The chunk count of per page.371	 *	M : the metasize of per page.372	 *	C : the ecc chunk size, aka the "data" above.373	 *	P': the nand chip's page size.374	 *	O : the nand chip's oob size.375	 *	O': the free oob.376	 *377	 *	The formula for P is :378	 *379	 *	            E * G * N380	 *	       P = ------------ + P' + M381	 *                      8382	 *383	 * The position of block mark moves forward in the ECC-based view384	 * of page, and the delta is:385	 *386	 *                   E * G * (N - 1)387	 *             D = (---------------- + M)388	 *                          8389	 *390	 * Please see the comment in legacy_set_geometry().391	 * With the condition C >= O , we still can get same result.392	 * So the bit position of the physical block mark within the ECC-based393	 * view of the page is :394	 *             (P' - D) * 8395	 */396	geo->page_size = mtd->writesize + geo->metadata_size +397		(geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;398 399	geo->payload_size = mtd->writesize;400 401	geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);402	geo->auxiliary_size = ALIGN(geo->metadata_size, 4)403				+ ALIGN(geo->ecc_chunk_count, 4);404 405	if (!this->swap_block_mark)406		return 0;407 408	/* For bit swap. */409	block_mark_bit_offset = mtd->writesize * 8 -410		(geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)411				+ geo->metadata_size * 8);412 413	geo->block_mark_byte_offset = block_mark_bit_offset / 8;414	geo->block_mark_bit_offset  = block_mark_bit_offset % 8;415	return 0;416}417 418/*419 *  Calculate the ECC strength by hand:420 *	E : The ECC strength.421 *	G : the length of Galois Field.422 *	N : The chunk count of per page.423 *	O : the oobsize of the NAND chip.424 *	M : the metasize of per page.425 *426 *	The formula is :427 *		E * G * N428 *	      ------------ <= (O - M)429 *                  8430 *431 *      So, we get E by:432 *                    (O - M) * 8433 *              E <= -------------434 *                       G * N435 */436static inline int get_ecc_strength(struct gpmi_nand_data *this)437{438	struct bch_geometry *geo = &this->bch_geometry;439	struct mtd_info	*mtd = nand_to_mtd(&this->nand);440	int ecc_strength;441 442	ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8)443			/ (geo->gf_len * geo->ecc_chunk_count);444 445	/* We need the minor even number. */446	return round_down(ecc_strength, 2);447}448 449static int set_geometry_for_large_oob(struct gpmi_nand_data *this)450{451	struct bch_geometry *geo = &this->bch_geometry;452	struct nand_chip *chip = &this->nand;453	struct mtd_info *mtd = nand_to_mtd(chip);454	const struct nand_ecc_props *requirements =455		nanddev_get_ecc_requirements(&chip->base);456	unsigned int block_mark_bit_offset;457	unsigned int max_ecc;458	unsigned int bbm_chunk;459	unsigned int i;460 461	/* sanity check for the minimum ecc nand required */462	if (!(requirements->strength > 0 &&463	      requirements->step_size > 0))464		return -EINVAL;465	geo->ecc_strength = requirements->strength;466 467	/* check if platform can support this nand */468	if (!gpmi_check_ecc(this)) {469		dev_err(this->dev,470			"unsupported NAND chip, minimum ecc required %d\n",471			geo->ecc_strength);472		return -EINVAL;473	}474 475	/* calculate the maximum ecc platform can support*/476	geo->metadata_size = 10;477	geo->gf_len = 14;478	geo->ecc0_chunk_size = 1024;479	geo->eccn_chunk_size = 1024;480	geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;481	max_ecc = min(get_ecc_strength(this),482		      this->devdata->bch_max_ecc_strength);483 484	/*485	 * search a supported ecc strength that makes bbm486	 * located in data chunk487	 */488	geo->ecc_strength = max_ecc;489	while (!(geo->ecc_strength < requirements->strength)) {490		if (bbm_in_data_chunk(this, &bbm_chunk))491			goto geo_setting;492		geo->ecc_strength -= 2;493	}494 495	/* if none of them works, keep using the minimum ecc */496	/* nand required but changing ecc page layout  */497	geo->ecc_strength = requirements->strength;498	/* add extra ecc for meta data */499	geo->ecc0_chunk_size = 0;500	geo->ecc_chunk_count = (mtd->writesize / geo->eccn_chunk_size) + 1;501	geo->ecc_for_meta = 1;502	/* check if oob can afford this extra ecc chunk */503	if (mtd->oobsize * 8 < geo->metadata_size * 8 +504	    geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) {505		dev_err(this->dev, "unsupported NAND chip with new layout\n");506		return -EINVAL;507	}508 509	/* calculate in which chunk bbm located */510	bbm_chunk = (mtd->writesize * 8 - geo->metadata_size * 8 -511		     geo->gf_len * geo->ecc_strength) /512		     (geo->gf_len * geo->ecc_strength +513		     geo->eccn_chunk_size * 8) + 1;514 515geo_setting:516 517	geo->page_size = mtd->writesize + geo->metadata_size +518		(geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;519	geo->payload_size = mtd->writesize;520 521	/*522	 * The auxiliary buffer contains the metadata and the ECC status. The523	 * metadata is padded to the nearest 32-bit boundary. The ECC status524	 * contains one byte for every ECC chunk, and is also padded to the525	 * nearest 32-bit boundary.526	 */527	geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);528	geo->auxiliary_size = ALIGN(geo->metadata_size, 4)529				    + ALIGN(geo->ecc_chunk_count, 4);530 531	if (!this->swap_block_mark)532		return 0;533 534	/* calculate the number of ecc chunk behind the bbm */535	i = (mtd->writesize / geo->eccn_chunk_size) - bbm_chunk + 1;536 537	block_mark_bit_offset = mtd->writesize * 8 -538		(geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - i)539		+ geo->metadata_size * 8);540 541	geo->block_mark_byte_offset = block_mark_bit_offset / 8;542	geo->block_mark_bit_offset  = block_mark_bit_offset % 8;543 544	dev_dbg(this->dev, "BCH Geometry :\n"545		"GF length              : %u\n"546		"ECC Strength           : %u\n"547		"Page Size in Bytes     : %u\n"548		"Metadata Size in Bytes : %u\n"549		"ECC0 Chunk Size in Bytes: %u\n"550		"ECCn Chunk Size in Bytes: %u\n"551		"ECC Chunk Count        : %u\n"552		"Payload Size in Bytes  : %u\n"553		"Auxiliary Size in Bytes: %u\n"554		"Auxiliary Status Offset: %u\n"555		"Block Mark Byte Offset : %u\n"556		"Block Mark Bit Offset  : %u\n"557		"Block Mark in chunk	: %u\n"558		"Ecc for Meta data	: %u\n",559		geo->gf_len,560		geo->ecc_strength,561		geo->page_size,562		geo->metadata_size,563		geo->ecc0_chunk_size,564		geo->eccn_chunk_size,565		geo->ecc_chunk_count,566		geo->payload_size,567		geo->auxiliary_size,568		geo->auxiliary_status_offset,569		geo->block_mark_byte_offset,570		geo->block_mark_bit_offset,571		bbm_chunk,572		geo->ecc_for_meta);573 574	return 0;575}576 577static int legacy_set_geometry(struct gpmi_nand_data *this)578{579	struct bch_geometry *geo = &this->bch_geometry;580	struct mtd_info *mtd = nand_to_mtd(&this->nand);581	unsigned int metadata_size;582	unsigned int status_size;583	unsigned int block_mark_bit_offset;584 585	/*586	 * The size of the metadata can be changed, though we set it to 10587	 * bytes now. But it can't be too large, because we have to save588	 * enough space for BCH.589	 */590	geo->metadata_size = 10;591 592	/* The default for the length of Galois Field. */593	geo->gf_len = 13;594 595	/* The default for chunk size. */596	geo->ecc0_chunk_size = 512;597	geo->eccn_chunk_size = 512;598	while (geo->eccn_chunk_size < mtd->oobsize) {599		geo->ecc0_chunk_size *= 2; /* keep C >= O */600		geo->eccn_chunk_size *= 2; /* keep C >= O */601		geo->gf_len = 14;602	}603 604	geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;605 606	/* We use the same ECC strength for all chunks. */607	geo->ecc_strength = get_ecc_strength(this);608	if (!gpmi_check_ecc(this)) {609		dev_err(this->dev,610			"ecc strength: %d cannot be supported by the controller (%d)\n"611			"try to use minimum ecc strength that NAND chip required\n",612			geo->ecc_strength,613			this->devdata->bch_max_ecc_strength);614		return -EINVAL;615	}616 617	geo->page_size = mtd->writesize + geo->metadata_size +618		(geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;619	geo->payload_size = mtd->writesize;620 621	/*622	 * The auxiliary buffer contains the metadata and the ECC status. The623	 * metadata is padded to the nearest 32-bit boundary. The ECC status624	 * contains one byte for every ECC chunk, and is also padded to the625	 * nearest 32-bit boundary.626	 */627	metadata_size = ALIGN(geo->metadata_size, 4);628	status_size   = ALIGN(geo->ecc_chunk_count, 4);629 630	geo->auxiliary_size = metadata_size + status_size;631	geo->auxiliary_status_offset = metadata_size;632 633	if (!this->swap_block_mark)634		return 0;635 636	/*637	 * We need to compute the byte and bit offsets of638	 * the physical block mark within the ECC-based view of the page.639	 *640	 * NAND chip with 2K page shows below:641	 *                                             (Block Mark)642	 *                                                   |      |643	 *                                                   |  D   |644	 *                                                   |<---->|645	 *                                                   V      V646	 *    +---+----------+-+----------+-+----------+-+----------+-+647	 *    | M |   data   |E|   data   |E|   data   |E|   data   |E|648	 *    +---+----------+-+----------+-+----------+-+----------+-+649	 *650	 * The position of block mark moves forward in the ECC-based view651	 * of page, and the delta is:652	 *653	 *                   E * G * (N - 1)654	 *             D = (---------------- + M)655	 *                          8656	 *657	 * With the formula to compute the ECC strength, and the condition658	 *       : C >= O         (C is the ecc chunk size)659	 *660	 * It's easy to deduce to the following result:661	 *662	 *         E * G       (O - M)      C - M         C - M663	 *      ----------- <= ------- <=  --------  <  ---------664	 *           8            N           N          (N - 1)665	 *666	 *  So, we get:667	 *668	 *                   E * G * (N - 1)669	 *             D = (---------------- + M) < C670	 *                          8671	 *672	 *  The above inequality means the position of block mark673	 *  within the ECC-based view of the page is still in the data chunk,674	 *  and it's NOT in the ECC bits of the chunk.675	 *676	 *  Use the following to compute the bit position of the677	 *  physical block mark within the ECC-based view of the page:678	 *          (page_size - D) * 8679	 *680	 *  --Huang Shijie681	 */682	block_mark_bit_offset = mtd->writesize * 8 -683		(geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)684				+ geo->metadata_size * 8);685 686	geo->block_mark_byte_offset = block_mark_bit_offset / 8;687	geo->block_mark_bit_offset  = block_mark_bit_offset % 8;688	return 0;689}690 691static int common_nfc_set_geometry(struct gpmi_nand_data *this)692{693	struct nand_chip *chip = &this->nand;694	struct mtd_info *mtd = nand_to_mtd(&this->nand);695	const struct nand_ecc_props *requirements =696		nanddev_get_ecc_requirements(&chip->base);697	bool use_minimun_ecc;698	int err;699 700	use_minimun_ecc = of_property_read_bool(this->dev->of_node,701						"fsl,use-minimum-ecc");702 703	/* use legacy bch geometry settings by default*/704	if ((!use_minimun_ecc && mtd->oobsize < 1024) ||705	    !(requirements->strength > 0 && requirements->step_size > 0)) {706		dev_dbg(this->dev, "use legacy bch geometry\n");707		err = legacy_set_geometry(this);708		if (!err)709			return 0;710	}711 712	/* for large oob nand */713	if (mtd->oobsize > 1024) {714		dev_dbg(this->dev, "use large oob bch geometry\n");715		err = set_geometry_for_large_oob(this);716		if (!err)717			return 0;718	}719 720	/* otherwise use the minimum ecc nand chip required */721	dev_dbg(this->dev, "use minimum ecc bch geometry\n");722	err = set_geometry_by_ecc_info(this, requirements->strength,723					requirements->step_size);724	if (err)725		dev_err(this->dev, "none of the bch geometry setting works\n");726 727	return err;728}729 730/* Configures the geometry for BCH.  */731static int bch_set_geometry(struct gpmi_nand_data *this)732{733	struct resources *r = &this->resources;734	int ret;735 736	ret = common_nfc_set_geometry(this);737	if (ret)738		return ret;739 740	ret = pm_runtime_get_sync(this->dev);741	if (ret < 0) {742		pm_runtime_put_autosuspend(this->dev);743		return ret;744	}745 746	/*747	* Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this748	* chip, otherwise it will lock up. So we skip resetting BCH on the MX23.749	* and MX28.750	*/751	ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));752	if (ret)753		goto err_out;754 755	/* Set *all* chip selects to use layout 0. */756	writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT);757 758	ret = 0;759err_out:760	pm_runtime_mark_last_busy(this->dev);761	pm_runtime_put_autosuspend(this->dev);762 763	return ret;764}765 766/*767 * <1> Firstly, we should know what's the GPMI-clock means.768 *     The GPMI-clock is the internal clock in the gpmi nand controller.769 *     If you set 100MHz to gpmi nand controller, the GPMI-clock's period770 *     is 10ns. Mark the GPMI-clock's period as GPMI-clock-period.771 *772 * <2> Secondly, we should know what's the frequency on the nand chip pins.773 *     The frequency on the nand chip pins is derived from the GPMI-clock.774 *     We can get it from the following equation:775 *776 *         F = G / (DS + DH)777 *778 *         F  : the frequency on the nand chip pins.779 *         G  : the GPMI clock, such as 100MHz.780 *         DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP781 *         DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD782 *783 * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz,784 *     the nand EDO(extended Data Out) timing could be applied.785 *     The GPMI implements a feedback read strobe to sample the read data.786 *     The feedback read strobe can be delayed to support the nand EDO timing787 *     where the read strobe may deasserts before the read data is valid, and788 *     read data is valid for some time after read strobe.789 *790 *     The following figure illustrates some aspects of a NAND Flash read:791 *792 *                   |<---tREA---->|793 *                   |             |794 *                   |         |   |795 *                   |<--tRP-->|   |796 *                   |         |   |797 *                  __          ___|__________________________________798 *     RDN            \________/   |799 *                                 |800 *                                 /---------\801 *     Read Data    --------------<           >---------802 *                                 \---------/803 *                                |     |804 *                                |<-D->|805 *     FeedbackRDN  ________             ____________806 *                          \___________/807 *808 *          D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY.809 *810 *811 * <4> Now, we begin to describe how to compute the right RDN_DELAY.812 *813 *  4.1) From the aspect of the nand chip pins:814 *        Delay = (tREA + C - tRP)               {1}815 *816 *        tREA : the maximum read access time.817 *        C    : a constant to adjust the delay. default is 4000ps.818 *        tRP  : the read pulse width, which is exactly:819 *                   tRP = (GPMI-clock-period) * DATA_SETUP820 *821 *  4.2) From the aspect of the GPMI nand controller:822 *         Delay = RDN_DELAY * 0.125 * RP        {2}823 *824 *         RP   : the DLL reference period.825 *            if (GPMI-clock-period > DLL_THRETHOLD)826 *                   RP = GPMI-clock-period / 2;827 *            else828 *                   RP = GPMI-clock-period;829 *830 *            Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period831 *            is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD832 *            is 16000ps, but in mx6q, we use 12000ps.833 *834 *  4.3) since {1} equals {2}, we get:835 *836 *                     (tREA + 4000 - tRP) * 8837 *         RDN_DELAY = -----------------------     {3}838 *                           RP839 */840static int gpmi_nfc_compute_timings(struct gpmi_nand_data *this,841				    const struct nand_sdr_timings *sdr)842{843	struct gpmi_nfc_hardware_timing *hw = &this->hw;844	struct resources *r = &this->resources;845	unsigned int dll_threshold_ps = this->devdata->max_chain_delay;846	unsigned int period_ps, reference_period_ps;847	unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles;848	unsigned int tRP_ps;849	bool use_half_period;850	int sample_delay_ps, sample_delay_factor;851	unsigned int busy_timeout_cycles;852	u8 wrn_dly_sel;853	unsigned long clk_rate, min_rate;854	u64 busy_timeout_ps;855 856	if (sdr->tRC_min >= 30000) {857		/* ONFI non-EDO modes [0-3] */858		hw->clk_rate = 22000000;859		min_rate = 0;860		wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS;861	} else if (sdr->tRC_min >= 25000) {862		/* ONFI EDO mode 4 */863		hw->clk_rate = 80000000;864		min_rate = 22000000;865		wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;866	} else {867		/* ONFI EDO mode 5 */868		hw->clk_rate = 100000000;869		min_rate = 80000000;870		wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;871	}872 873	clk_rate = clk_round_rate(r->clock[0], hw->clk_rate);874	if (clk_rate <= min_rate) {875		dev_err(this->dev, "clock setting: expected %ld, got %ld\n",876			hw->clk_rate, clk_rate);877		return -ENOTSUPP;878	}879 880	hw->clk_rate = clk_rate;881	/* SDR core timings are given in picoseconds */882	period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate);883 884	addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps);885	data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps);886	data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps);887	busy_timeout_ps = max(sdr->tBERS_max, sdr->tPROG_max);888	busy_timeout_cycles = TO_CYCLES(busy_timeout_ps, period_ps);889 890	hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) |891		      BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) |892		      BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles);893	hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(DIV_ROUND_UP(busy_timeout_cycles, 4096));894 895	/*896	 * Derive NFC ideal delay from {3}:897	 *898	 *                     (tREA + 4000 - tRP) * 8899	 *         RDN_DELAY = -----------------------900	 *                                RP901	 */902	if (period_ps > dll_threshold_ps) {903		use_half_period = true;904		reference_period_ps = period_ps / 2;905	} else {906		use_half_period = false;907		reference_period_ps = period_ps;908	}909 910	tRP_ps = data_setup_cycles * period_ps;911	sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8;912	if (sample_delay_ps > 0)913		sample_delay_factor = sample_delay_ps / reference_period_ps;914	else915		sample_delay_factor = 0;916 917	hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel);918	if (sample_delay_factor)919		hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) |920			      BM_GPMI_CTRL1_DLL_ENABLE |921			      (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0);922	return 0;923}924 925static int gpmi_nfc_apply_timings(struct gpmi_nand_data *this)926{927	struct gpmi_nfc_hardware_timing *hw = &this->hw;928	struct resources *r = &this->resources;929	void __iomem *gpmi_regs = r->gpmi_regs;930	unsigned int dll_wait_time_us;931	int ret;932 933	/* Clock dividers do NOT guarantee a clean clock signal on its output934	 * during the change of the divide factor on i.MX6Q/UL/SX. On i.MX7/8,935	 * all clock dividers provide these guarantee.936	 */937	if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this))938		clk_disable_unprepare(r->clock[0]);939 940	ret = clk_set_rate(r->clock[0], hw->clk_rate);941	if (ret) {942		dev_err(this->dev, "cannot set clock rate to %lu Hz: %d\n", hw->clk_rate, ret);943		return ret;944	}945 946	if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) {947		ret = clk_prepare_enable(r->clock[0]);948		if (ret)949			return ret;950	}951 952	writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0);953	writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1);954 955	/*956	 * Clear several CTRL1 fields, DLL must be disabled when setting957	 * RDN_DELAY or HALF_PERIOD.958	 */959	writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR);960	writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET);961 962	/* Wait 64 clock cycles before using the GPMI after enabling the DLL */963	dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64;964	if (!dll_wait_time_us)965		dll_wait_time_us = 1;966 967	/* Wait for the DLL to settle. */968	udelay(dll_wait_time_us);969 970	return 0;971}972 973static int gpmi_setup_interface(struct nand_chip *chip, int chipnr,974				const struct nand_interface_config *conf)975{976	struct gpmi_nand_data *this = nand_get_controller_data(chip);977	const struct nand_sdr_timings *sdr;978	int ret;979 980	/* Retrieve required NAND timings */981	sdr = nand_get_sdr_timings(conf);982	if (IS_ERR(sdr))983		return PTR_ERR(sdr);984 985	/* Only MX28/MX6 GPMI controller can reach EDO timings */986	if (sdr->tRC_min <= 25000 && !this->devdata->support_edo_timing)987		return -ENOTSUPP;988 989	/* Stop here if this call was just a check */990	if (chipnr < 0)991		return 0;992 993	/* Do the actual derivation of the controller timings */994	ret = gpmi_nfc_compute_timings(this, sdr);995	if (ret)996		return ret;997 998	this->hw.must_apply_timings = true;999 1000	return 0;1001}1002 1003/* Clears a BCH interrupt. */1004static void gpmi_clear_bch(struct gpmi_nand_data *this)1005{1006	struct resources *r = &this->resources;1007	writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR);1008}1009 1010static struct dma_chan *get_dma_chan(struct gpmi_nand_data *this)1011{1012	/* We use the DMA channel 0 to access all the nand chips. */1013	return this->dma_chans[0];1014}1015 1016/* This will be called after the DMA operation is finished. */1017static void dma_irq_callback(void *param)1018{1019	struct gpmi_nand_data *this = param;1020	struct completion *dma_c = &this->dma_done;1021 1022	complete(dma_c);1023}1024 1025static irqreturn_t bch_irq(int irq, void *cookie)1026{1027	struct gpmi_nand_data *this = cookie;1028 1029	gpmi_clear_bch(this);1030	complete(&this->bch_done);1031	return IRQ_HANDLED;1032}1033 1034static int gpmi_raw_len_to_len(struct gpmi_nand_data *this, int raw_len)1035{1036	/*1037	 * raw_len is the length to read/write including bch data which1038	 * we are passed in exec_op. Calculate the data length from it.1039	 */1040	if (this->bch)1041		return ALIGN_DOWN(raw_len, this->bch_geometry.eccn_chunk_size);1042	else1043		return raw_len;1044}1045 1046/* Can we use the upper's buffer directly for DMA? */1047static bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf,1048			     int raw_len, struct scatterlist *sgl,1049			     enum dma_data_direction dr)1050{1051	int ret;1052	int len = gpmi_raw_len_to_len(this, raw_len);1053 1054	/* first try to map the upper buffer directly */1055	if (virt_addr_valid(buf) && !object_is_on_stack(buf)) {1056		sg_init_one(sgl, buf, len);1057		ret = dma_map_sg(this->dev, sgl, 1, dr);1058		if (ret == 0)1059			goto map_fail;1060 1061		return true;1062	}1063 1064map_fail:1065	/* We have to use our own DMA buffer. */1066	sg_init_one(sgl, this->data_buffer_dma, len);1067 1068	if (dr == DMA_TO_DEVICE && buf != this->data_buffer_dma)1069		memcpy(this->data_buffer_dma, buf, len);1070 1071	dma_map_sg(this->dev, sgl, 1, dr);1072 1073	return false;1074}1075 1076/* add our owner bbt descriptor */1077static uint8_t scan_ff_pattern[] = { 0xff };1078static struct nand_bbt_descr gpmi_bbt_descr = {1079	.options	= 0,1080	.offs		= 0,1081	.len		= 1,1082	.pattern	= scan_ff_pattern1083};1084 1085/*1086 * We may change the layout if we can get the ECC info from the datasheet,1087 * else we will use all the (page + OOB).1088 */1089static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section,1090			      struct mtd_oob_region *oobregion)1091{1092	struct nand_chip *chip = mtd_to_nand(mtd);1093	struct gpmi_nand_data *this = nand_get_controller_data(chip);1094	struct bch_geometry *geo = &this->bch_geometry;1095 1096	if (section)1097		return -ERANGE;1098 1099	oobregion->offset = 0;1100	oobregion->length = geo->page_size - mtd->writesize;1101 1102	return 0;1103}1104 1105static int gpmi_ooblayout_free(struct mtd_info *mtd, int section,1106			       struct mtd_oob_region *oobregion)1107{1108	struct nand_chip *chip = mtd_to_nand(mtd);1109	struct gpmi_nand_data *this = nand_get_controller_data(chip);1110	struct bch_geometry *geo = &this->bch_geometry;1111 1112	if (section)1113		return -ERANGE;1114 1115	/* The available oob size we have. */1116	if (geo->page_size < mtd->writesize + mtd->oobsize) {1117		oobregion->offset = geo->page_size - mtd->writesize;1118		oobregion->length = mtd->oobsize - oobregion->offset;1119	}1120 1121	return 0;1122}1123 1124static const char * const gpmi_clks_for_mx2x[] = {1125	"gpmi_io",1126};1127 1128static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = {1129	.ecc = gpmi_ooblayout_ecc,1130	.free = gpmi_ooblayout_free,1131};1132 1133static const struct gpmi_devdata gpmi_devdata_imx23 = {1134	.type = IS_MX23,1135	.bch_max_ecc_strength = 20,1136	.max_chain_delay = 16000,1137	.clks = gpmi_clks_for_mx2x,1138	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),1139};1140 1141static const struct gpmi_devdata gpmi_devdata_imx28 = {1142	.type = IS_MX28,1143	.bch_max_ecc_strength = 20,1144	.max_chain_delay = 16000,1145	.support_edo_timing = true,1146	.clks = gpmi_clks_for_mx2x,1147	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),1148};1149 1150static const char * const gpmi_clks_for_mx6[] = {1151	"gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch",1152};1153 1154static const struct gpmi_devdata gpmi_devdata_imx6q = {1155	.type = IS_MX6Q,1156	.bch_max_ecc_strength = 40,1157	.max_chain_delay = 12000,1158	.support_edo_timing = true,1159	.clks = gpmi_clks_for_mx6,1160	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),1161};1162 1163static const struct gpmi_devdata gpmi_devdata_imx6sx = {1164	.type = IS_MX6SX,1165	.bch_max_ecc_strength = 62,1166	.max_chain_delay = 12000,1167	.support_edo_timing = true,1168	.clks = gpmi_clks_for_mx6,1169	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),1170};1171 1172static const char * const gpmi_clks_for_mx7d[] = {1173	"gpmi_io", "gpmi_bch_apb",1174};1175 1176static const struct gpmi_devdata gpmi_devdata_imx7d = {1177	.type = IS_MX7D,1178	.bch_max_ecc_strength = 62,1179	.max_chain_delay = 12000,1180	.support_edo_timing = true,1181	.clks = gpmi_clks_for_mx7d,1182	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx7d),1183};1184 1185static const char *gpmi_clks_for_mx8qxp[GPMI_CLK_MAX] = {1186	"gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb",1187};1188 1189static const struct gpmi_devdata gpmi_devdata_imx8qxp = {1190	.type = IS_MX8QXP,1191	.bch_max_ecc_strength = 62,1192	.max_chain_delay = 12000,1193	.support_edo_timing = true,1194	.clks = gpmi_clks_for_mx8qxp,1195	.clks_count = ARRAY_SIZE(gpmi_clks_for_mx8qxp),1196};1197 1198static int acquire_register_block(struct gpmi_nand_data *this,1199				  const char *res_name)1200{1201	struct platform_device *pdev = this->pdev;1202	struct resources *res = &this->resources;1203	void __iomem *p;1204 1205	p = devm_platform_ioremap_resource_byname(pdev, res_name);1206	if (IS_ERR(p))1207		return PTR_ERR(p);1208 1209	if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME))1210		res->gpmi_regs = p;1211	else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME))1212		res->bch_regs = p;1213	else1214		dev_err(this->dev, "unknown resource name : %s\n", res_name);1215 1216	return 0;1217}1218 1219static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h)1220{1221	struct platform_device *pdev = this->pdev;1222	const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME;1223	int err;1224 1225	err = platform_get_irq_byname(pdev, res_name);1226	if (err < 0)1227		return err;1228 1229	err = devm_request_irq(this->dev, err, irq_h, 0, res_name, this);1230	if (err)1231		dev_err(this->dev, "error requesting BCH IRQ\n");1232 1233	return err;1234}1235 1236static void release_dma_channels(struct gpmi_nand_data *this)1237{1238	unsigned int i;1239	for (i = 0; i < DMA_CHANS; i++)1240		if (this->dma_chans[i]) {1241			dma_release_channel(this->dma_chans[i]);1242			this->dma_chans[i] = NULL;1243		}1244}1245 1246static int acquire_dma_channels(struct gpmi_nand_data *this)1247{1248	struct platform_device *pdev = this->pdev;1249	struct dma_chan *dma_chan;1250	int ret = 0;1251 1252	/* request dma channel */1253	dma_chan = dma_request_chan(&pdev->dev, "rx-tx");1254	if (IS_ERR(dma_chan)) {1255		ret = dev_err_probe(this->dev, PTR_ERR(dma_chan),1256				    "DMA channel request failed\n");1257		release_dma_channels(this);1258	} else {1259		this->dma_chans[0] = dma_chan;1260	}1261 1262	return ret;1263}1264 1265static int gpmi_get_clks(struct gpmi_nand_data *this)1266{1267	struct resources *r = &this->resources;1268	struct clk *clk;1269	int err, i;1270 1271	for (i = 0; i < this->devdata->clks_count; i++) {1272		clk = devm_clk_get(this->dev, this->devdata->clks[i]);1273		if (IS_ERR(clk)) {1274			err = PTR_ERR(clk);1275			goto err_clock;1276		}1277 1278		r->clock[i] = clk;1279	}1280 1281	return 0;1282 1283err_clock:1284	dev_dbg(this->dev, "failed in finding the clocks.\n");1285	return err;1286}1287 1288static int acquire_resources(struct gpmi_nand_data *this)1289{1290	int ret;1291 1292	ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME);1293	if (ret)1294		goto exit_regs;1295 1296	ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME);1297	if (ret)1298		goto exit_regs;1299 1300	ret = acquire_bch_irq(this, bch_irq);1301	if (ret)1302		goto exit_regs;1303 1304	ret = acquire_dma_channels(this);1305	if (ret)1306		goto exit_regs;1307 1308	ret = gpmi_get_clks(this);1309	if (ret)1310		goto exit_clock;1311	return 0;1312 1313exit_clock:1314	release_dma_channels(this);1315exit_regs:1316	return ret;1317}1318 1319static void release_resources(struct gpmi_nand_data *this)1320{1321	release_dma_channels(this);1322}1323 1324static void gpmi_free_dma_buffer(struct gpmi_nand_data *this)1325{1326	struct device *dev = this->dev;1327	struct bch_geometry *geo = &this->bch_geometry;1328 1329	if (this->auxiliary_virt && virt_addr_valid(this->auxiliary_virt))1330		dma_free_coherent(dev, geo->auxiliary_size,1331					this->auxiliary_virt,1332					this->auxiliary_phys);1333	kfree(this->data_buffer_dma);1334	kfree(this->raw_buffer);1335 1336	this->data_buffer_dma	= NULL;1337	this->raw_buffer	= NULL;1338}1339 1340/* Allocate the DMA buffers */1341static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this)1342{1343	struct bch_geometry *geo = &this->bch_geometry;1344	struct device *dev = this->dev;1345	struct mtd_info *mtd = nand_to_mtd(&this->nand);1346 1347	/*1348	 * [2] Allocate a read/write data buffer.1349	 *     The gpmi_alloc_dma_buffer can be called twice.1350	 *     We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer1351	 *     is called before the NAND identification; and we allocate a1352	 *     buffer of the real NAND page size when the gpmi_alloc_dma_buffer1353	 *     is called after.1354	 */1355	this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE,1356					GFP_DMA | GFP_KERNEL);1357	if (this->data_buffer_dma == NULL)1358		goto error_alloc;1359 1360	this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size,1361					&this->auxiliary_phys, GFP_DMA);1362	if (!this->auxiliary_virt)1363		goto error_alloc;1364 1365	this->raw_buffer = kzalloc((mtd->writesize ?: PAGE_SIZE) + mtd->oobsize, GFP_KERNEL);1366	if (!this->raw_buffer)1367		goto error_alloc;1368 1369	return 0;1370 1371error_alloc:1372	gpmi_free_dma_buffer(this);1373	return -ENOMEM;1374}1375 1376/*1377 * Handles block mark swapping.1378 * It can be called in swapping the block mark, or swapping it back,1379 * because the operations are the same.1380 */1381static void block_mark_swapping(struct gpmi_nand_data *this,1382				void *payload, void *auxiliary)1383{1384	struct bch_geometry *nfc_geo = &this->bch_geometry;1385	unsigned char *p;1386	unsigned char *a;1387	unsigned int  bit;1388	unsigned char mask;1389	unsigned char from_data;1390	unsigned char from_oob;1391 1392	if (!this->swap_block_mark)1393		return;1394 1395	/*1396	 * If control arrives here, we're swapping. Make some convenience1397	 * variables.1398	 */1399	bit = nfc_geo->block_mark_bit_offset;1400	p   = payload + nfc_geo->block_mark_byte_offset;1401	a   = auxiliary;1402 1403	/*1404	 * Get the byte from the data area that overlays the block mark. Since1405	 * the ECC engine applies its own view to the bits in the page, the1406	 * physical block mark won't (in general) appear on a byte boundary in1407	 * the data.1408	 */1409	from_data = (p[0] >> bit) | (p[1] << (8 - bit));1410 1411	/* Get the byte from the OOB. */1412	from_oob = a[0];1413 1414	/* Swap them. */1415	a[0] = from_data;1416 1417	mask = (0x1 << bit) - 1;1418	p[0] = (p[0] & mask) | (from_oob << bit);1419 1420	mask = ~0 << bit;1421	p[1] = (p[1] & mask) | (from_oob >> (8 - bit));1422}1423 1424static int gpmi_count_bitflips(struct nand_chip *chip, void *buf, int first,1425			       int last, int meta)1426{1427	struct gpmi_nand_data *this = nand_get_controller_data(chip);1428	struct bch_geometry *nfc_geo = &this->bch_geometry;1429	struct mtd_info *mtd = nand_to_mtd(chip);1430	int i;1431	unsigned char *status;1432	unsigned int max_bitflips = 0;1433 1434	/* Loop over status bytes, accumulating ECC status. */1435	status = this->auxiliary_virt + ALIGN(meta, 4);1436 1437	for (i = first; i < last; i++, status++) {1438		if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED))1439			continue;1440 1441		if (*status == STATUS_UNCORRECTABLE) {1442			int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;1443			u8 *eccbuf = this->raw_buffer;1444			int offset, bitoffset;1445			int eccbytes;1446			int flips;1447 1448			/* Read ECC bytes into our internal raw_buffer */1449			offset = nfc_geo->metadata_size * 8;1450			offset += ((8 * nfc_geo->eccn_chunk_size) + eccbits) * (i + 1);1451			offset -= eccbits;1452			bitoffset = offset % 8;1453			eccbytes = DIV_ROUND_UP(offset + eccbits, 8);1454			offset /= 8;1455			eccbytes -= offset;1456			nand_change_read_column_op(chip, offset, eccbuf,1457						   eccbytes, false);1458 1459			/*1460			 * ECC data are not byte aligned and we may have1461			 * in-band data in the first and last byte of1462			 * eccbuf. Set non-eccbits to one so that1463			 * nand_check_erased_ecc_chunk() does not count them1464			 * as bitflips.1465			 */1466			if (bitoffset)1467				eccbuf[0] |= GENMASK(bitoffset - 1, 0);1468 1469			bitoffset = (bitoffset + eccbits) % 8;1470			if (bitoffset)1471				eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset);1472 1473			/*1474			 * The ECC hardware has an uncorrectable ECC status1475			 * code in case we have bitflips in an erased page. As1476			 * nothing was written into this subpage the ECC is1477			 * obviously wrong and we can not trust it. We assume1478			 * at this point that we are reading an erased page and1479			 * try to correct the bitflips in buffer up to1480			 * ecc_strength bitflips. If this is a page with random1481			 * data, we exceed this number of bitflips and have a1482			 * ECC failure. Otherwise we use the corrected buffer.1483			 */1484			if (i == 0) {1485				/* The first block includes metadata */1486				flips = nand_check_erased_ecc_chunk(1487						buf + i * nfc_geo->eccn_chunk_size,1488						nfc_geo->eccn_chunk_size,1489						eccbuf, eccbytes,1490						this->auxiliary_virt,1491						nfc_geo->metadata_size,1492						nfc_geo->ecc_strength);1493			} else {1494				flips = nand_check_erased_ecc_chunk(1495						buf + i * nfc_geo->eccn_chunk_size,1496						nfc_geo->eccn_chunk_size,1497						eccbuf, eccbytes,1498						NULL, 0,1499						nfc_geo->ecc_strength);1500			}1501 1502			if (flips > 0) {1503				max_bitflips = max_t(unsigned int, max_bitflips,1504						     flips);1505				mtd->ecc_stats.corrected += flips;1506				continue;1507			}1508 1509			mtd->ecc_stats.failed++;1510			continue;1511		}1512 1513		mtd->ecc_stats.corrected += *status;1514		max_bitflips = max_t(unsigned int, max_bitflips, *status);1515	}1516 1517	return max_bitflips;1518}1519 1520static void gpmi_bch_layout_std(struct gpmi_nand_data *this)1521{1522	struct bch_geometry *geo = &this->bch_geometry;1523	unsigned int ecc_strength = geo->ecc_strength >> 1;1524	unsigned int gf_len = geo->gf_len;1525	unsigned int block0_size = geo->ecc0_chunk_size;1526	unsigned int blockn_size = geo->eccn_chunk_size;1527 1528	this->bch_flashlayout0 =1529		BF_BCH_FLASH0LAYOUT0_NBLOCKS(geo->ecc_chunk_count - 1) |1530		BF_BCH_FLASH0LAYOUT0_META_SIZE(geo->metadata_size) |1531		BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |1532		BF_BCH_FLASH0LAYOUT0_GF(gf_len, this) |1533		BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block0_size, this);1534 1535	this->bch_flashlayout1 =1536		BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(geo->page_size) |1537		BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |1538		BF_BCH_FLASH0LAYOUT1_GF(gf_len, this) |1539		BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(blockn_size, this);1540}1541 1542static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf,1543			      int oob_required, int page)1544{1545	struct gpmi_nand_data *this = nand_get_controller_data(chip);1546	struct mtd_info *mtd = nand_to_mtd(chip);1547	struct bch_geometry *geo = &this->bch_geometry;1548	unsigned int max_bitflips;1549	int ret;1550 1551	gpmi_bch_layout_std(this);1552	this->bch = true;1553 1554	ret = nand_read_page_op(chip, page, 0, buf, geo->page_size);1555	if (ret)1556		return ret;1557 1558	max_bitflips = gpmi_count_bitflips(chip, buf, 0,1559					   geo->ecc_chunk_count,1560					   geo->auxiliary_status_offset);1561 1562	/* handle the block mark swapping */1563	block_mark_swapping(this, buf, this->auxiliary_virt);1564 1565	if (oob_required) {1566		/*1567		 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob()1568		 * for details about our policy for delivering the OOB.1569		 *1570		 * We fill the caller's buffer with set bits, and then copy the1571		 * block mark to th caller's buffer. Note that, if block mark1572		 * swapping was necessary, it has already been done, so we can1573		 * rely on the first byte of the auxiliary buffer to contain1574		 * the block mark.1575		 */1576		memset(chip->oob_poi, ~0, mtd->oobsize);1577		chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0];1578	}1579 1580	return max_bitflips;1581}1582 1583/* Fake a virtual small page for the subpage read */1584static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs,1585				 uint32_t len, uint8_t *buf, int page)1586{1587	struct gpmi_nand_data *this = nand_get_controller_data(chip);1588	struct bch_geometry *geo = &this->bch_geometry;1589	int size = chip->ecc.size; /* ECC chunk size */1590	int meta, n, page_size;1591	unsigned int max_bitflips;1592	unsigned int ecc_strength;1593	int first, last, marker_pos;1594	int ecc_parity_size;1595	int col = 0;1596	int ret;1597 1598	/* The size of ECC parity */1599	ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;1600 1601	/* Align it with the chunk size */1602	first = offs / size;1603	last = (offs + len - 1) / size;1604 1605	if (this->swap_block_mark) {1606		/*1607		 * Find the chunk which contains the Block Marker.1608		 * If this chunk is in the range of [first, last],1609		 * we have to read out the whole page.1610		 * Why? since we had swapped the data at the position of Block1611		 * Marker to the metadata which is bound with the chunk 0.1612		 */1613		marker_pos = geo->block_mark_byte_offset / size;1614		if (last >= marker_pos && first <= marker_pos) {1615			dev_dbg(this->dev,1616				"page:%d, first:%d, last:%d, marker at:%d\n",1617				page, first, last, marker_pos);1618			return gpmi_ecc_read_page(chip, buf, 0, page);1619		}1620	}1621 1622	/*1623	 * if there is an ECC dedicate for meta:1624	 * - need to add an extra ECC size when calculating col and page_size,1625	 *   if the meta size is NOT zero.1626	 * - ecc0_chunk size need to set to the same size as other chunks,1627	 *   if the meta size is zero.1628	 */1629 1630	meta = geo->metadata_size;1631	if (first) {1632		if (geo->ecc_for_meta)1633			col = meta + ecc_parity_size1634				+ (size + ecc_parity_size) * first;1635		else1636			col = meta + (size + ecc_parity_size) * first;1637 1638		meta = 0;1639		buf = buf + first * size;1640	}1641 1642	ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;1643	n = last - first + 1;1644 1645	if (geo->ecc_for_meta && meta)1646		page_size = meta + ecc_parity_size1647			    + (size + ecc_parity_size) * n;1648	else1649		page_size = meta + (size + ecc_parity_size) * n;1650 1651	ecc_strength = geo->ecc_strength >> 1;1652 1653	this->bch_flashlayout0 = BF_BCH_FLASH0LAYOUT0_NBLOCKS(1654		(geo->ecc_for_meta ? n : n - 1)) |1655		BF_BCH_FLASH0LAYOUT0_META_SIZE(meta) |1656		BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |1657		BF_BCH_FLASH0LAYOUT0_GF(geo->gf_len, this) |1658		BF_BCH_FLASH0LAYOUT0_DATA0_SIZE((geo->ecc_for_meta ?1659		0 : geo->ecc0_chunk_size), this);1660 1661	this->bch_flashlayout1 = BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size) |1662		BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |1663		BF_BCH_FLASH0LAYOUT1_GF(geo->gf_len, this) |1664		BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(geo->eccn_chunk_size, this);1665 1666	this->bch = true;1667 1668	ret = nand_read_page_op(chip, page, col, buf, page_size);1669	if (ret)1670		return ret;1671 1672	dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n",1673		page, offs, len, col, first, n, page_size);1674 1675	max_bitflips = gpmi_count_bitflips(chip, buf, first, last, meta);1676 1677	return max_bitflips;1678}1679 1680static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf,1681			       int oob_required, int page)1682{1683	struct mtd_info *mtd = nand_to_mtd(chip);1684	struct gpmi_nand_data *this = nand_get_controller_data(chip);1685	struct bch_geometry *nfc_geo = &this->bch_geometry;1686 1687	dev_dbg(this->dev, "ecc write page.\n");1688 1689	gpmi_bch_layout_std(this);1690	this->bch = true;1691 1692	memcpy(this->auxiliary_virt, chip->oob_poi, nfc_geo->auxiliary_size);1693 1694	if (this->swap_block_mark) {1695		/*1696		 * When doing bad block marker swapping we must always copy the1697		 * input buffer as we can't modify the const buffer.1698		 */1699		memcpy(this->data_buffer_dma, buf, mtd->writesize);1700		buf = this->data_buffer_dma;1701		block_mark_swapping(this, this->data_buffer_dma,1702				    this->auxiliary_virt);1703	}1704 1705	return nand_prog_page_op(chip, page, 0, buf, nfc_geo->page_size);1706}1707 1708/*1709 * There are several places in this driver where we have to handle the OOB and1710 * block marks. This is the function where things are the most complicated, so1711 * this is where we try to explain it all. All the other places refer back to1712 * here.1713 *1714 * These are the rules, in order of decreasing importance:1715 *1716 * 1) Nothing the caller does can be allowed to imperil the block mark.1717 *1718 * 2) In read operations, the first byte of the OOB we return must reflect the1719 *    true state of the block mark, no matter where that block mark appears in1720 *    the physical page.1721 *1722 * 3) ECC-based read operations return an OOB full of set bits (since we never1723 *    allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads1724 *    return).1725 *1726 * 4) "Raw" read operations return a direct view of the physical bytes in the1727 *    page, using the conventional definition of which bytes are data and which1728 *    are OOB. This gives the caller a way to see the actual, physical bytes1729 *    in the page, without the distortions applied by our ECC engine.1730 *1731 *1732 * What we do for this specific read operation depends on two questions:1733 *1734 * 1) Are we doing a "raw" read, or an ECC-based read?1735 *1736 * 2) Are we using block mark swapping or transcription?1737 *1738 * There are four cases, illustrated by the following Karnaugh map:1739 *1740 *                    |           Raw           |         ECC-based       |1741 *       -------------+-------------------------+-------------------------+1742 *                    | Read the conventional   |                         |1743 *                    | OOB at the end of the   |                         |1744 *       Swapping     | page and return it. It  |                         |1745 *                    | contains exactly what   |                         |1746 *                    | we want.                | Read the block mark and |1747 *       -------------+-------------------------+ return it in a buffer   |1748 *                    | Read the conventional   | full of set bits.       |1749 *                    | OOB at the end of the   |                         |1750 *                    | page and also the block |                         |1751 *       Transcribing | mark in the metadata.   |                         |1752 *                    | Copy the block mark     |                         |1753 *                    | into the first byte of  |                         |1754 *                    | the OOB.                |                         |1755 *       -------------+-------------------------+-------------------------+1756 *1757 * Note that we break rule #4 in the Transcribing/Raw case because we're not1758 * giving an accurate view of the actual, physical bytes in the page (we're1759 * overwriting the block mark). That's OK because it's more important to follow1760 * rule #2.1761 *1762 * It turns out that knowing whether we want an "ECC-based" or "raw" read is not1763 * easy. When reading a page, for example, the NAND Flash MTD code calls our1764 * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an1765 * ECC-based or raw view of the page is implicit in which function it calls1766 * (there is a similar pair of ECC-based/raw functions for writing).1767 */1768static int gpmi_ecc_read_oob(struct nand_chip *chip, int page)1769{1770	struct mtd_info *mtd = nand_to_mtd(chip);1771	struct gpmi_nand_data *this = nand_get_controller_data(chip);1772	int ret;1773 1774	/* clear the OOB buffer */1775	memset(chip->oob_poi, ~0, mtd->oobsize);1776 1777	/* Read out the conventional OOB. */1778	ret = nand_read_page_op(chip, page, mtd->writesize, chip->oob_poi,1779				mtd->oobsize);1780	if (ret)1781		return ret;1782 1783	/*1784	 * Now, we want to make sure the block mark is correct. In the1785	 * non-transcribing case (!GPMI_IS_MX23()), we already have it.1786	 * Otherwise, we need to explicitly read it.1787	 */1788	if (GPMI_IS_MX23(this)) {1789		/* Read the block mark into the first byte of the OOB buffer. */1790		ret = nand_read_page_op(chip, page, 0, chip->oob_poi, 1);1791		if (ret)1792			return ret;1793	}1794 1795	return 0;1796}1797 1798static int gpmi_ecc_write_oob(struct nand_chip *chip, int page)1799{1800	struct mtd_info *mtd = nand_to_mtd(chip);1801	struct mtd_oob_region of = { };1802 1803	/* Do we have available oob area? */1804	mtd_ooblayout_free(mtd, 0, &of);1805	if (!of.length)1806		return -EPERM;1807 1808	if (!nand_is_slc(chip))1809		return -EPERM;1810 1811	return nand_prog_page_op(chip, page, mtd->writesize + of.offset,1812				 chip->oob_poi + of.offset, of.length);1813}1814 1815/*1816 * This function reads a NAND page without involving the ECC engine (no HW1817 * ECC correction).1818 * The tricky part in the GPMI/BCH controller is that it stores ECC bits1819 * inline (interleaved with payload DATA), and do not align data chunk on1820 * byte boundaries.1821 * We thus need to take care moving the payload data and ECC bits stored in the1822 * page into the provided buffers, which is why we're using nand_extract_bits().1823 *1824 * See set_geometry_by_ecc_info inline comments to have a full description1825 * of the layout used by the GPMI controller.1826 */1827static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf,1828				  int oob_required, int page)1829{1830	struct mtd_info *mtd = nand_to_mtd(chip);1831	struct gpmi_nand_data *this = nand_get_controller_data(chip);1832	struct bch_geometry *nfc_geo = &this->bch_geometry;1833	int eccsize = nfc_geo->eccn_chunk_size;1834	int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;1835	u8 *tmp_buf = this->raw_buffer;1836	size_t src_bit_off;1837	size_t oob_bit_off;1838	size_t oob_byte_off;1839	uint8_t *oob = chip->oob_poi;1840	int step;1841	int ret;1842 1843	ret = nand_read_page_op(chip, page, 0, tmp_buf,1844				mtd->writesize + mtd->oobsize);1845	if (ret)1846		return ret;1847 1848	/*1849	 * If required, swap the bad block marker and the data stored in the1850	 * metadata section, so that we don't wrongly consider a block as bad.1851	 *1852	 * See the layout description for a detailed explanation on why this1853	 * is needed.1854	 */1855	if (this->swap_block_mark)1856		swap(tmp_buf[0], tmp_buf[mtd->writesize]);1857 1858	/*1859	 * Copy the metadata section into the oob buffer (this section is1860	 * guaranteed to be aligned on a byte boundary).1861	 */1862	if (oob_required)1863		memcpy(oob, tmp_buf, nfc_geo->metadata_size);1864 1865	oob_bit_off = nfc_geo->metadata_size * 8;1866	src_bit_off = oob_bit_off;1867 1868	/* Extract interleaved payload data and ECC bits */1869	for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {1870		if (buf)1871			nand_extract_bits(buf, step * eccsize * 8, tmp_buf,1872					  src_bit_off, eccsize * 8);1873		src_bit_off += eccsize * 8;1874 1875		/* Align last ECC block to align a byte boundary */1876		if (step == nfc_geo->ecc_chunk_count - 1 &&1877		    (oob_bit_off + eccbits) % 8)1878			eccbits += 8 - ((oob_bit_off + eccbits) % 8);1879 1880		if (oob_required)1881			nand_extract_bits(oob, oob_bit_off, tmp_buf,1882					  src_bit_off, eccbits);1883 1884		src_bit_off += eccbits;1885		oob_bit_off += eccbits;1886	}1887 1888	if (oob_required) {1889		oob_byte_off = oob_bit_off / 8;1890 1891		if (oob_byte_off < mtd->oobsize)1892			memcpy(oob + oob_byte_off,1893			       tmp_buf + mtd->writesize + oob_byte_off,1894			       mtd->oobsize - oob_byte_off);1895	}1896 1897	return 0;1898}1899 1900/*1901 * This function writes a NAND page without involving the ECC engine (no HW1902 * ECC generation).1903 * The tricky part in the GPMI/BCH controller is that it stores ECC bits1904 * inline (interleaved with payload DATA), and do not align data chunk on1905 * byte boundaries.1906 * We thus need to take care moving the OOB area at the right place in the1907 * final page, which is why we're using nand_extract_bits().1908 *1909 * See set_geometry_by_ecc_info inline comments to have a full description1910 * of the layout used by the GPMI controller.1911 */1912static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf,1913				   int oob_required, int page)1914{1915	struct mtd_info *mtd = nand_to_mtd(chip);1916	struct gpmi_nand_data *this = nand_get_controller_data(chip);1917	struct bch_geometry *nfc_geo = &this->bch_geometry;1918	int eccsize = nfc_geo->eccn_chunk_size;1919	int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;1920	u8 *tmp_buf = this->raw_buffer;1921	uint8_t *oob = chip->oob_poi;1922	size_t dst_bit_off;1923	size_t oob_bit_off;1924	size_t oob_byte_off;1925	int step;1926 1927	/*1928	 * Initialize all bits to 1 in case we don't have a buffer for the1929	 * payload or oob data in order to leave unspecified bits of data1930	 * to their initial state.1931	 */1932	if (!buf || !oob_required)1933		memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize);1934 1935	/*1936	 * First copy the metadata section (stored in oob buffer) at the1937	 * beginning of the page, as imposed by the GPMI layout.1938	 */1939	memcpy(tmp_buf, oob, nfc_geo->metadata_size);1940	oob_bit_off = nfc_geo->metadata_size * 8;1941	dst_bit_off = oob_bit_off;1942 1943	/* Interleave payload data and ECC bits */1944	for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {1945		if (buf)1946			nand_extract_bits(tmp_buf, dst_bit_off, buf,1947					  step * eccsize * 8, eccsize * 8);1948		dst_bit_off += eccsize * 8;1949 1950		/* Align last ECC block to align a byte boundary */1951		if (step == nfc_geo->ecc_chunk_count - 1 &&1952		    (oob_bit_off + eccbits) % 8)1953			eccbits += 8 - ((oob_bit_off + eccbits) % 8);1954 1955		if (oob_required)1956			nand_extract_bits(tmp_buf, dst_bit_off, oob,1957					  oob_bit_off, eccbits);1958 1959		dst_bit_off += eccbits;1960		oob_bit_off += eccbits;1961	}1962 1963	oob_byte_off = oob_bit_off / 8;1964 1965	if (oob_required && oob_byte_off < mtd->oobsize)1966		memcpy(tmp_buf + mtd->writesize + oob_byte_off,1967		       oob + oob_byte_off, mtd->oobsize - oob_byte_off);1968 1969	/*1970	 * If required, swap the bad block marker and the first byte of the1971	 * metadata section, so that we don't modify the bad block marker.1972	 *1973	 * See the layout description for a detailed explanation on why this1974	 * is needed.1975	 */1976	if (this->swap_block_mark)1977		swap(tmp_buf[0], tmp_buf[mtd->writesize]);1978 1979	return nand_prog_page_op(chip, page, 0, tmp_buf,1980				 mtd->writesize + mtd->oobsize);1981}1982 1983static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page)1984{1985	return gpmi_ecc_read_page_raw(chip, NULL, 1, page);1986}1987 1988static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page)1989{1990	return gpmi_ecc_write_page_raw(chip, NULL, 1, page);1991}1992 1993static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs)1994{1995	struct mtd_info *mtd = nand_to_mtd(chip);1996	struct gpmi_nand_data *this = nand_get_controller_data(chip);1997	int ret = 0;1998	uint8_t *block_mark;1999	int column, page, chipnr;2000 2001	chipnr = (int)(ofs >> chip->chip_shift);2002	nand_select_target(chip, chipnr);2003 2004	column = !GPMI_IS_MX23(this) ? mtd->writesize : 0;2005 2006	/* Write the block mark. */2007	block_mark = this->data_buffer_dma;2008	block_mark[0] = 0; /* bad block marker */2009 2010	/* Shift to get page */2011	page = (int)(ofs >> chip->page_shift);2012 2013	ret = nand_prog_page_op(chip, page, column, block_mark, 1);2014 2015	nand_deselect_target(chip);2016 2017	return ret;2018}2019 2020static int nand_boot_set_geometry(struct gpmi_nand_data *this)2021{2022	struct boot_rom_geometry *geometry = &this->rom_geometry;2023 2024	/*2025	 * Set the boot block stride size.2026	 *2027	 * In principle, we should be reading this from the OTP bits, since2028	 * that's where the ROM is going to get it. In fact, we don't have any2029	 * way to read the OTP bits, so we go with the default and hope for the2030	 * best.2031	 */2032	geometry->stride_size_in_pages = 64;2033 2034	/*2035	 * Set the search area stride exponent.2036	 *2037	 * In principle, we should be reading this from the OTP bits, since2038	 * that's where the ROM is going to get it. In fact, we don't have any2039	 * way to read the OTP bits, so we go with the default and hope for the2040	 * best.2041	 */2042	geometry->search_area_stride_exponent = 2;2043	return 0;2044}2045 2046static const char  *fingerprint = "STMP";2047static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)2048{2049	struct boot_rom_geometry *rom_geo = &this->rom_geometry;2050	struct device *dev = this->dev;2051	struct nand_chip *chip = &this->nand;2052	unsigned int search_area_size_in_strides;2053	unsigned int stride;2054	unsigned int page;2055	u8 *buffer = nand_get_data_buf(chip);2056	int found_an_ncb_fingerprint = false;2057	int ret;2058 2059	/* Compute the number of strides in a search area. */2060	search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;2061 2062	nand_select_target(chip, 0);2063 2064	/*2065	 * Loop through the first search area, looking for the NCB fingerprint.2066	 */2067	dev_dbg(dev, "Scanning for an NCB fingerprint...\n");2068 2069	for (stride = 0; stride < search_area_size_in_strides; stride++) {2070		/* Compute the page addresses. */2071		page = stride * rom_geo->stride_size_in_pages;2072 2073		dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page);2074 2075		/*2076		 * Read the NCB fingerprint. The fingerprint is four bytes long2077		 * and starts in the 12th byte of the page.2078		 */2079		ret = nand_read_page_op(chip, page, 12, buffer,2080					strlen(fingerprint));2081		if (ret)2082			continue;2083 2084		/* Look for the fingerprint. */2085		if (!memcmp(buffer, fingerprint, strlen(fingerprint))) {2086			found_an_ncb_fingerprint = true;2087			break;2088		}2089 2090	}2091 2092	nand_deselect_target(chip);2093 2094	if (found_an_ncb_fingerprint)2095		dev_dbg(dev, "\tFound a fingerprint\n");2096	else2097		dev_dbg(dev, "\tNo fingerprint found\n");2098	return found_an_ncb_fingerprint;2099}2100 2101/* Writes a transcription stamp. */2102static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)2103{2104	struct device *dev = this->dev;2105	struct boot_rom_geometry *rom_geo = &this->rom_geometry;2106	struct nand_chip *chip = &this->nand;2107	struct mtd_info *mtd = nand_to_mtd(chip);2108	unsigned int block_size_in_pages;2109	unsigned int search_area_size_in_strides;2110	unsigned int search_area_size_in_pages;2111	unsigned int search_area_size_in_blocks;2112	unsigned int block;2113	unsigned int stride;2114	unsigned int page;2115	u8 *buffer = nand_get_data_buf(chip);2116	int status;2117 2118	/* Compute the search area geometry. */2119	block_size_in_pages = mtd->erasesize / mtd->writesize;2120	search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;2121	search_area_size_in_pages = search_area_size_in_strides *2122					rom_geo->stride_size_in_pages;2123	search_area_size_in_blocks =2124		  (search_area_size_in_pages + (block_size_in_pages - 1)) /2125				    block_size_in_pages;2126 2127	dev_dbg(dev, "Search Area Geometry :\n");2128	dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks);2129	dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides);2130	dev_dbg(dev, "\tin Pages  : %u\n", search_area_size_in_pages);2131 2132	nand_select_target(chip, 0);2133 2134	/* Loop over blocks in the first search area, erasing them. */2135	dev_dbg(dev, "Erasing the search area...\n");2136 2137	for (block = 0; block < search_area_size_in_blocks; block++) {2138		/* Erase this block. */2139		dev_dbg(dev, "\tErasing block 0x%x\n", block);2140		status = nand_erase_op(chip, block);2141		if (status)2142			dev_err(dev, "[%s] Erase failed.\n", __func__);2143	}2144 2145	/* Write the NCB fingerprint into the page buffer. */2146	memset(buffer, ~0, mtd->writesize);2147	memcpy(buffer + 12, fingerprint, strlen(fingerprint));2148 2149	/* Loop through the first search area, writing NCB fingerprints. */2150	dev_dbg(dev, "Writing NCB fingerprints...\n");2151	for (stride = 0; stride < search_area_size_in_strides; stride++) {2152		/* Compute the page addresses. */2153		page = stride * rom_geo->stride_size_in_pages;2154 2155		/* Write the first page of the current stride. */2156		dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page);2157 2158		status = chip->ecc.write_page_raw(chip, buffer, 0, page);2159		if (status)2160			dev_err(dev, "[%s] Write failed.\n", __func__);2161	}2162 2163	nand_deselect_target(chip);2164 2165	return 0;2166}2167 2168static int mx23_boot_init(struct gpmi_nand_data  *this)2169{2170	struct device *dev = this->dev;2171	struct nand_chip *chip = &this->nand;2172	struct mtd_info *mtd = nand_to_mtd(chip);2173	unsigned int block_count;2174	unsigned int block;2175	int     chipnr;2176	int     page;2177	loff_t  byte;2178	uint8_t block_mark;2179	int     ret = 0;2180 2181	/*2182	 * If control arrives here, we can't use block mark swapping, which2183	 * means we're forced to use transcription. First, scan for the2184	 * transcription stamp. If we find it, then we don't have to do2185	 * anything -- the block marks are already transcribed.2186	 */2187	if (mx23_check_transcription_stamp(this))2188		return 0;2189 2190	/*2191	 * If control arrives here, we couldn't find a transcription stamp, so2192	 * so we presume the block marks are in the conventional location.2193	 */2194	dev_dbg(dev, "Transcribing bad block marks...\n");2195 2196	/* Compute the number of blocks in the entire medium. */2197	block_count = nanddev_eraseblocks_per_target(&chip->base);2198 2199	/*2200	 * Loop over all the blocks in the medium, transcribing block marks as2201	 * we go.2202	 */2203	for (block = 0; block < block_count; block++) {2204		/*2205		 * Compute the chip, page and byte addresses for this block's2206		 * conventional mark.2207		 */2208		chipnr = block >> (chip->chip_shift - chip->phys_erase_shift);2209		page = block << (chip->phys_erase_shift - chip->page_shift);2210		byte = block <<  chip->phys_erase_shift;2211 2212		/* Send the command to read the conventional block mark. */2213		nand_select_target(chip, chipnr);2214		ret = nand_read_page_op(chip, page, mtd->writesize, &block_mark,2215					1);2216		nand_deselect_target(chip);2217 2218		if (ret)2219			continue;2220 2221		/*2222		 * Check if the block is marked bad. If so, we need to mark it2223		 * again, but this time the result will be a mark in the2224		 * location where we transcribe block marks.2225		 */2226		if (block_mark != 0xff) {2227			dev_dbg(dev, "Transcribing mark in block %u\n", block);2228			ret = chip->legacy.block_markbad(chip, byte);2229			if (ret)2230				dev_err(dev,2231					"Failed to mark block bad with ret %d\n",2232					ret);2233		}2234	}2235 2236	/* Write the stamp that indicates we've transcribed the block marks. */2237	mx23_write_transcription_stamp(this);2238	return 0;2239}2240 2241static int nand_boot_init(struct gpmi_nand_data  *this)2242{2243	nand_boot_set_geometry(this);2244 2245	/* This is ROM arch-specific initilization before the BBT scanning. */2246	if (GPMI_IS_MX23(this))2247		return mx23_boot_init(this);2248	return 0;2249}2250 2251static int gpmi_set_geometry(struct gpmi_nand_data *this)2252{2253	int ret;2254 2255	/* Free the temporary DMA memory for reading ID. */2256	gpmi_free_dma_buffer(this);2257 2258	/* Set up the NFC geometry which is used by BCH. */2259	ret = bch_set_geometry(this);2260	if (ret) {2261		dev_err(this->dev, "Error setting BCH geometry : %d\n", ret);2262		return ret;2263	}2264 2265	/* Alloc the new DMA buffers according to the pagesize and oobsize */2266	return gpmi_alloc_dma_buffer(this);2267}2268 2269static int gpmi_init_last(struct gpmi_nand_data *this)2270{2271	struct nand_chip *chip = &this->nand;2272	struct mtd_info *mtd = nand_to_mtd(chip);2273	struct nand_ecc_ctrl *ecc = &chip->ecc;2274	struct bch_geometry *bch_geo = &this->bch_geometry;2275	int ret;2276 2277	/* Set up the medium geometry */2278	ret = gpmi_set_geometry(this);2279	if (ret)2280		return ret;2281 2282	/* Init the nand_ecc_ctrl{} */2283	ecc->read_page	= gpmi_ecc_read_page;2284	ecc->write_page	= gpmi_ecc_write_page;2285	ecc->read_oob	= gpmi_ecc_read_oob;2286	ecc->write_oob	= gpmi_ecc_write_oob;2287	ecc->read_page_raw = gpmi_ecc_read_page_raw;2288	ecc->write_page_raw = gpmi_ecc_write_page_raw;2289	ecc->read_oob_raw = gpmi_ecc_read_oob_raw;2290	ecc->write_oob_raw = gpmi_ecc_write_oob_raw;2291	ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;2292	ecc->size	= bch_geo->eccn_chunk_size;2293	ecc->strength	= bch_geo->ecc_strength;2294	mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops);2295 2296	/*2297	 * We only enable the subpage read when:2298	 *  (1) the chip is imx6, and2299	 *  (2) the size of the ECC parity is byte aligned.2300	 */2301	if (GPMI_IS_MX6(this) &&2302		((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) {2303		ecc->read_subpage = gpmi_ecc_read_subpage;2304		chip->options |= NAND_SUBPAGE_READ;2305	}2306 2307	return 0;2308}2309 2310static int gpmi_nand_attach_chip(struct nand_chip *chip)2311{2312	struct gpmi_nand_data *this = nand_get_controller_data(chip);2313	int ret;2314 2315	if (chip->bbt_options & NAND_BBT_USE_FLASH) {2316		chip->bbt_options |= NAND_BBT_NO_OOB;2317 2318		if (of_property_read_bool(this->dev->of_node,2319					  "fsl,no-blockmark-swap"))2320			this->swap_block_mark = false;2321	}2322	dev_dbg(this->dev, "Blockmark swapping %sabled\n",2323		this->swap_block_mark ? "en" : "dis");2324 2325	ret = gpmi_init_last(this);2326	if (ret)2327		return ret;2328 2329	chip->options |= NAND_SKIP_BBTSCAN;2330 2331	return 0;2332}2333 2334static struct gpmi_transfer *get_next_transfer(struct gpmi_nand_data *this)2335{2336	struct gpmi_transfer *transfer = &this->transfers[this->ntransfers];2337 2338	this->ntransfers++;2339 2340	if (this->ntransfers == GPMI_MAX_TRANSFERS)2341		return NULL;2342 2343	return transfer;2344}2345 2346static struct dma_async_tx_descriptor *gpmi_chain_command(2347	struct gpmi_nand_data *this, u8 cmd, const u8 *addr, int naddr)2348{2349	struct dma_chan *channel = get_dma_chan(this);2350	struct dma_async_tx_descriptor *desc;2351	struct gpmi_transfer *transfer;2352	int chip = this->nand.cur_cs;2353	u32 pio[3];2354 2355	/* [1] send out the PIO words */2356	pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)2357		| BM_GPMI_CTRL0_WORD_LENGTH2358		| BF_GPMI_CTRL0_CS(chip, this)2359		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)2360		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE)2361		| BM_GPMI_CTRL0_ADDRESS_INCREMENT2362		| BF_GPMI_CTRL0_XFER_COUNT(naddr + 1);2363	pio[1] = 0;2364	pio[2] = 0;2365	desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),2366				      DMA_TRANS_NONE, 0);2367	if (!desc)2368		return NULL;2369 2370	transfer = get_next_transfer(this);2371	if (!transfer)2372		return NULL;2373 2374	transfer->cmdbuf[0] = cmd;2375	if (naddr)2376		memcpy(&transfer->cmdbuf[1], addr, naddr);2377 2378	sg_init_one(&transfer->sgl, transfer->cmdbuf, naddr + 1);2379	dma_map_sg(this->dev, &transfer->sgl, 1, DMA_TO_DEVICE);2380 2381	transfer->direction = DMA_TO_DEVICE;2382 2383	desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, DMA_MEM_TO_DEV,2384				       MXS_DMA_CTRL_WAIT4END);2385	return desc;2386}2387 2388static struct dma_async_tx_descriptor *gpmi_chain_wait_ready(2389	struct gpmi_nand_data *this)2390{2391	struct dma_chan *channel = get_dma_chan(this);2392	u32 pio[2];2393 2394	pio[0] =  BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY)2395		| BM_GPMI_CTRL0_WORD_LENGTH2396		| BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)2397		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)2398		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)2399		| BF_GPMI_CTRL0_XFER_COUNT(0);2400	pio[1] = 0;2401 2402	return mxs_dmaengine_prep_pio(channel, pio, 2, DMA_TRANS_NONE,2403				MXS_DMA_CTRL_WAIT4END | MXS_DMA_CTRL_WAIT4RDY);2404}2405 2406static struct dma_async_tx_descriptor *gpmi_chain_data_read(2407	struct gpmi_nand_data *this, void *buf, int raw_len, bool *direct)2408{2409	struct dma_async_tx_descriptor *desc;2410	struct dma_chan *channel = get_dma_chan(this);2411	struct gpmi_transfer *transfer;2412	u32 pio[6] = {};2413 2414	transfer = get_next_transfer(this);2415	if (!transfer)2416		return NULL;2417 2418	transfer->direction = DMA_FROM_DEVICE;2419 2420	*direct = prepare_data_dma(this, buf, raw_len, &transfer->sgl,2421				   DMA_FROM_DEVICE);2422 2423	pio[0] =  BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ)2424		| BM_GPMI_CTRL0_WORD_LENGTH2425		| BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)2426		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)2427		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)2428		| BF_GPMI_CTRL0_XFER_COUNT(raw_len);2429 2430	if (this->bch) {2431		pio[2] =  BM_GPMI_ECCCTRL_ENABLE_ECC2432			| BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE)2433			| BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE2434				| BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);2435		pio[3] = raw_len;2436		pio[4] = transfer->sgl.dma_address;2437		pio[5] = this->auxiliary_phys;2438	}2439 2440	desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),2441				      DMA_TRANS_NONE, 0);2442	if (!desc)2443		return NULL;2444 2445	if (!this->bch)2446		desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,2447					     DMA_DEV_TO_MEM,2448					     MXS_DMA_CTRL_WAIT4END);2449 2450	return desc;2451}2452 2453static struct dma_async_tx_descriptor *gpmi_chain_data_write(2454	struct gpmi_nand_data *this, const void *buf, int raw_len)2455{2456	struct dma_chan *channel = get_dma_chan(this);2457	struct dma_async_tx_descriptor *desc;2458	struct gpmi_transfer *transfer;2459	u32 pio[6] = {};2460 2461	transfer = get_next_transfer(this);2462	if (!transfer)2463		return NULL;2464 2465	transfer->direction = DMA_TO_DEVICE;2466 2467	prepare_data_dma(this, buf, raw_len, &transfer->sgl, DMA_TO_DEVICE);2468 2469	pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)2470		| BM_GPMI_CTRL0_WORD_LENGTH2471		| BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)2472		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)2473		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)2474		| BF_GPMI_CTRL0_XFER_COUNT(raw_len);2475 2476	if (this->bch) {2477		pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC2478			| BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE)2479			| BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE |2480					BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);2481		pio[3] = raw_len;2482		pio[4] = transfer->sgl.dma_address;2483		pio[5] = this->auxiliary_phys;2484	}2485 2486	desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),2487				      DMA_TRANS_NONE,2488				      (this->bch ? MXS_DMA_CTRL_WAIT4END : 0));2489	if (!desc)2490		return NULL;2491 2492	if (!this->bch)2493		desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,2494					       DMA_MEM_TO_DEV,2495					       MXS_DMA_CTRL_WAIT4END);2496 2497	return desc;2498}2499 2500static int gpmi_nfc_exec_op(struct nand_chip *chip,2501			     const struct nand_operation *op,2502			     bool check_only)2503{2504	const struct nand_op_instr *instr;2505	struct gpmi_nand_data *this = nand_get_controller_data(chip);2506	struct dma_async_tx_descriptor *desc = NULL;2507	int i, ret, buf_len = 0, nbufs = 0;2508	u8 cmd = 0;2509	void *buf_read = NULL;2510	const void *buf_write = NULL;2511	bool direct = false;2512	struct completion *dma_completion, *bch_completion;2513	unsigned long to;2514 2515	if (check_only)2516		return 0;2517 2518	this->ntransfers = 0;2519	for (i = 0; i < GPMI_MAX_TRANSFERS; i++)2520		this->transfers[i].direction = DMA_NONE;2521 2522	ret = pm_runtime_resume_and_get(this->dev);2523	if (ret < 0)2524		return ret;2525 2526	/*2527	 * This driver currently supports only one NAND chip. Plus, dies share2528	 * the same configuration. So once timings have been applied on the2529	 * controller side, they will not change anymore. When the time will2530	 * come, the check on must_apply_timings will have to be dropped.2531	 */2532	if (this->hw.must_apply_timings) {2533		this->hw.must_apply_timings = false;2534		ret = gpmi_nfc_apply_timings(this);2535		if (ret)2536			goto out_pm;2537	}2538 2539	dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs);2540 2541	for (i = 0; i < op->ninstrs; i++) {2542		instr = &op->instrs[i];2543 2544		nand_op_trace("  ", instr);2545 2546		switch (instr->type) {2547		case NAND_OP_WAITRDY_INSTR:2548			desc = gpmi_chain_wait_ready(this);2549			break;2550		case NAND_OP_CMD_INSTR:2551			cmd = instr->ctx.cmd.opcode;2552 2553			/*2554			 * When this command has an address cycle chain it2555			 * together with the address cycle2556			 */2557			if (i + 1 != op->ninstrs &&2558			    op->instrs[i + 1].type == NAND_OP_ADDR_INSTR)2559				continue;2560 2561			desc = gpmi_chain_command(this, cmd, NULL, 0);2562 2563			break;2564		case NAND_OP_ADDR_INSTR:2565			desc = gpmi_chain_command(this, cmd, instr->ctx.addr.addrs,2566						  instr->ctx.addr.naddrs);2567			break;2568		case NAND_OP_DATA_OUT_INSTR:2569			buf_write = instr->ctx.data.buf.out;2570			buf_len = instr->ctx.data.len;2571			nbufs++;2572 2573			desc = gpmi_chain_data_write(this, buf_write, buf_len);2574 2575			break;2576		case NAND_OP_DATA_IN_INSTR:2577			if (!instr->ctx.data.len)2578				break;2579			buf_read = instr->ctx.data.buf.in;2580			buf_len = instr->ctx.data.len;2581			nbufs++;2582 2583			desc = gpmi_chain_data_read(this, buf_read, buf_len,2584						   &direct);2585			break;2586		}2587 2588		if (!desc) {2589			ret = -ENXIO;2590			goto unmap;2591		}2592	}2593 2594	dev_dbg(this->dev, "%s setup done\n", __func__);2595 2596	if (nbufs > 1) {2597		dev_err(this->dev, "Multiple data instructions not supported\n");2598		ret = -EINVAL;2599		goto unmap;2600	}2601 2602	if (this->bch) {2603		writel(this->bch_flashlayout0,2604		       this->resources.bch_regs + HW_BCH_FLASH0LAYOUT0);2605		writel(this->bch_flashlayout1,2606		       this->resources.bch_regs + HW_BCH_FLASH0LAYOUT1);2607	}2608 2609	desc->callback = dma_irq_callback;2610	desc->callback_param = this;2611	dma_completion = &this->dma_done;2612	bch_completion = NULL;2613 2614	init_completion(dma_completion);2615 2616	if (this->bch && buf_read) {2617		writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,2618		       this->resources.bch_regs + HW_BCH_CTRL_SET);2619		bch_completion = &this->bch_done;2620		init_completion(bch_completion);2621	}2622 2623	dmaengine_submit(desc);2624	dma_async_issue_pending(get_dma_chan(this));2625 2626	to = wait_for_completion_timeout(dma_completion, msecs_to_jiffies(1000));2627	if (!to) {2628		dev_err(this->dev, "DMA timeout, last DMA\n");2629		gpmi_dump_info(this);2630		ret = -ETIMEDOUT;2631		goto unmap;2632	}2633 2634	if (this->bch && buf_read) {2635		to = wait_for_completion_timeout(bch_completion, msecs_to_jiffies(1000));2636		if (!to) {2637			dev_err(this->dev, "BCH timeout, last DMA\n");2638			gpmi_dump_info(this);2639			ret = -ETIMEDOUT;2640			goto unmap;2641		}2642	}2643 2644	writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,2645	       this->resources.bch_regs + HW_BCH_CTRL_CLR);2646	gpmi_clear_bch(this);2647 2648	ret = 0;2649 2650unmap:2651	for (i = 0; i < this->ntransfers; i++) {2652		struct gpmi_transfer *transfer = &this->transfers[i];2653 2654		if (transfer->direction != DMA_NONE)2655			dma_unmap_sg(this->dev, &transfer->sgl, 1,2656				     transfer->direction);2657	}2658 2659	if (!ret && buf_read && !direct)2660		memcpy(buf_read, this->data_buffer_dma,2661		       gpmi_raw_len_to_len(this, buf_len));2662 2663	this->bch = false;2664 2665out_pm:2666	pm_runtime_mark_last_busy(this->dev);2667	pm_runtime_put_autosuspend(this->dev);2668 2669	return ret;2670}2671 2672static const struct nand_controller_ops gpmi_nand_controller_ops = {2673	.attach_chip = gpmi_nand_attach_chip,2674	.setup_interface = gpmi_setup_interface,2675	.exec_op = gpmi_nfc_exec_op,2676};2677 2678static int gpmi_nand_init(struct gpmi_nand_data *this)2679{2680	struct nand_chip *chip = &this->nand;2681	struct mtd_info  *mtd = nand_to_mtd(chip);2682	int ret;2683 2684	/* init the MTD data structures */2685	mtd->name		= "gpmi-nand";2686	mtd->dev.parent		= this->dev;2687 2688	/* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */2689	nand_set_controller_data(chip, this);2690	nand_set_flash_node(chip, this->pdev->dev.of_node);2691	chip->legacy.block_markbad = gpmi_block_markbad;2692	chip->badblock_pattern	= &gpmi_bbt_descr;2693	chip->options		|= NAND_NO_SUBPAGE_WRITE;2694 2695	/* Set up swap_block_mark, must be set before the gpmi_set_geometry() */2696	this->swap_block_mark = !GPMI_IS_MX23(this);2697 2698	/*2699	 * Allocate a temporary DMA buffer for reading ID in the2700	 * nand_scan_ident().2701	 */2702	this->bch_geometry.payload_size = 1024;2703	this->bch_geometry.auxiliary_size = 128;2704	ret = gpmi_alloc_dma_buffer(this);2705	if (ret)2706		return ret;2707 2708	nand_controller_init(&this->base);2709	this->base.ops = &gpmi_nand_controller_ops;2710	chip->controller = &this->base;2711 2712	ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1);2713	if (ret)2714		goto err_out;2715 2716	ret = nand_boot_init(this);2717	if (ret)2718		goto err_nand_cleanup;2719	ret = nand_create_bbt(chip);2720	if (ret)2721		goto err_nand_cleanup;2722 2723	ret = mtd_device_register(mtd, NULL, 0);2724	if (ret)2725		goto err_nand_cleanup;2726	return 0;2727 2728err_nand_cleanup:2729	nand_cleanup(chip);2730err_out:2731	gpmi_free_dma_buffer(this);2732	return ret;2733}2734 2735static const struct of_device_id gpmi_nand_id_table[] = {2736	{ .compatible = "fsl,imx23-gpmi-nand", .data = &gpmi_devdata_imx23, },2737	{ .compatible = "fsl,imx28-gpmi-nand", .data = &gpmi_devdata_imx28, },2738	{ .compatible = "fsl,imx6q-gpmi-nand", .data = &gpmi_devdata_imx6q, },2739	{ .compatible = "fsl,imx6sx-gpmi-nand", .data = &gpmi_devdata_imx6sx, },2740	{ .compatible = "fsl,imx7d-gpmi-nand", .data = &gpmi_devdata_imx7d,},2741	{ .compatible = "fsl,imx8qxp-gpmi-nand", .data = &gpmi_devdata_imx8qxp, },2742	{}2743};2744MODULE_DEVICE_TABLE(of, gpmi_nand_id_table);2745 2746static int gpmi_nand_probe(struct platform_device *pdev)2747{2748	struct gpmi_nand_data *this;2749	int ret;2750 2751	this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL);2752	if (!this)2753		return -ENOMEM;2754 2755	this->devdata = of_device_get_match_data(&pdev->dev);2756	platform_set_drvdata(pdev, this);2757	this->pdev  = pdev;2758	this->dev   = &pdev->dev;2759 2760	ret = acquire_resources(this);2761	if (ret)2762		goto exit_acquire_resources;2763 2764	ret = __gpmi_enable_clk(this, true);2765	if (ret)2766		goto exit_acquire_resources;2767 2768	pm_runtime_set_autosuspend_delay(&pdev->dev, 500);2769	pm_runtime_use_autosuspend(&pdev->dev);2770	pm_runtime_set_active(&pdev->dev);2771	pm_runtime_enable(&pdev->dev);2772	pm_runtime_get_sync(&pdev->dev);2773 2774	ret = gpmi_init(this);2775	if (ret)2776		goto exit_nfc_init;2777 2778	ret = gpmi_nand_init(this);2779	if (ret)2780		goto exit_nfc_init;2781 2782	pm_runtime_mark_last_busy(&pdev->dev);2783	pm_runtime_put_autosuspend(&pdev->dev);2784 2785	dev_info(this->dev, "driver registered.\n");2786 2787	return 0;2788 2789exit_nfc_init:2790	pm_runtime_put(&pdev->dev);2791	pm_runtime_disable(&pdev->dev);2792	release_resources(this);2793exit_acquire_resources:2794 2795	return ret;2796}2797 2798static void gpmi_nand_remove(struct platform_device *pdev)2799{2800	struct gpmi_nand_data *this = platform_get_drvdata(pdev);2801	struct nand_chip *chip = &this->nand;2802	int ret;2803 2804	pm_runtime_put_sync(&pdev->dev);2805	pm_runtime_disable(&pdev->dev);2806 2807	ret = mtd_device_unregister(nand_to_mtd(chip));2808	WARN_ON(ret);2809	nand_cleanup(chip);2810	gpmi_free_dma_buffer(this);2811	release_resources(this);2812}2813 2814#ifdef CONFIG_PM_SLEEP2815static int gpmi_pm_suspend(struct device *dev)2816{2817	struct gpmi_nand_data *this = dev_get_drvdata(dev);2818 2819	release_dma_channels(this);2820	return 0;2821}2822 2823static int gpmi_pm_resume(struct device *dev)2824{2825	struct gpmi_nand_data *this = dev_get_drvdata(dev);2826	int ret;2827 2828	ret = acquire_dma_channels(this);2829	if (ret < 0)2830		return ret;2831 2832	/* re-init the GPMI registers */2833	ret = gpmi_init(this);2834	if (ret) {2835		dev_err(this->dev, "Error setting GPMI : %d\n", ret);2836		return ret;2837	}2838 2839	/* Set flag to get timing setup restored for next exec_op */2840	if (this->hw.clk_rate)2841		this->hw.must_apply_timings = true;2842 2843	/* re-init the BCH registers */2844	ret = bch_set_geometry(this);2845	if (ret) {2846		dev_err(this->dev, "Error setting BCH : %d\n", ret);2847		return ret;2848	}2849 2850	return 0;2851}2852#endif /* CONFIG_PM_SLEEP */2853 2854static int __maybe_unused gpmi_runtime_suspend(struct device *dev)2855{2856	struct gpmi_nand_data *this = dev_get_drvdata(dev);2857 2858	return __gpmi_enable_clk(this, false);2859}2860 2861static int __maybe_unused gpmi_runtime_resume(struct device *dev)2862{2863	struct gpmi_nand_data *this = dev_get_drvdata(dev);2864 2865	return __gpmi_enable_clk(this, true);2866}2867 2868static const struct dev_pm_ops gpmi_pm_ops = {2869	SET_SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume)2870	SET_RUNTIME_PM_OPS(gpmi_runtime_suspend, gpmi_runtime_resume, NULL)2871};2872 2873static struct platform_driver gpmi_nand_driver = {2874	.driver = {2875		.name = "gpmi-nand",2876		.pm = &gpmi_pm_ops,2877		.of_match_table = gpmi_nand_id_table,2878	},2879	.probe   = gpmi_nand_probe,2880	.remove_new = gpmi_nand_remove,2881};2882module_platform_driver(gpmi_nand_driver);2883 2884MODULE_AUTHOR("Freescale Semiconductor, Inc.");2885MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver");2886MODULE_LICENSE("GPL");2887