889 lines · c
1/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.2 *3 * Redistribution and use in source and binary forms, with or without4 * modification, are permitted provided that the following conditions are met:5 * * Redistributions of source code must retain the above copyright6 * notice, this list of conditions and the following disclaimer.7 * * Redistributions in binary form must reproduce the above copyright8 * notice, this list of conditions and the following disclaimer in the9 * documentation and/or other materials provided with the distribution.10 * * Neither the name of Freescale Semiconductor nor the11 * names of its contributors may be used to endorse or promote products12 * derived from this software without specific prior written permission.13 *14 * ALTERNATIVELY, this software may be distributed under the terms of the15 * GNU General Public License ("GPL") as published by the Free Software16 * Foundation, either version 2 of that License or (at your option) any17 * later version.18 *19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.29 */30 31#include "qman_priv.h"32 33u16 qman_ip_rev;34EXPORT_SYMBOL(qman_ip_rev);35u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1;36EXPORT_SYMBOL(qm_channel_pool1);37u16 qm_channel_caam = QMAN_CHANNEL_CAAM;38EXPORT_SYMBOL(qm_channel_caam);39 40/* Register offsets */41#define REG_QCSP_LIO_CFG(n) (0x0000 + ((n) * 0x10))42#define REG_QCSP_IO_CFG(n) (0x0004 + ((n) * 0x10))43#define REG_QCSP_DD_CFG(n) (0x000c + ((n) * 0x10))44#define REG_DD_CFG 0x020045#define REG_DCP_CFG(n) (0x0300 + ((n) * 0x10))46#define REG_DCP_DD_CFG(n) (0x0304 + ((n) * 0x10))47#define REG_DCP_DLM_AVG(n) (0x030c + ((n) * 0x10))48#define REG_PFDR_FPC 0x040049#define REG_PFDR_FP_HEAD 0x040450#define REG_PFDR_FP_TAIL 0x040851#define REG_PFDR_FP_LWIT 0x041052#define REG_PFDR_CFG 0x041453#define REG_SFDR_CFG 0x050054#define REG_SFDR_IN_USE 0x050455#define REG_WQ_CS_CFG(n) (0x0600 + ((n) * 0x04))56#define REG_WQ_DEF_ENC_WQID 0x063057#define REG_WQ_SC_DD_CFG(n) (0x640 + ((n) * 0x04))58#define REG_WQ_PC_DD_CFG(n) (0x680 + ((n) * 0x04))59#define REG_WQ_DC0_DD_CFG(n) (0x6c0 + ((n) * 0x04))60#define REG_WQ_DC1_DD_CFG(n) (0x700 + ((n) * 0x04))61#define REG_WQ_DCn_DD_CFG(n) (0x6c0 + ((n) * 0x40)) /* n=2,3 */62#define REG_CM_CFG 0x080063#define REG_ECSR 0x0a0064#define REG_ECIR 0x0a0465#define REG_EADR 0x0a0866#define REG_ECIR2 0x0a0c67#define REG_EDATA(n) (0x0a10 + ((n) * 0x04))68#define REG_SBEC(n) (0x0a80 + ((n) * 0x04))69#define REG_MCR 0x0b0070#define REG_MCP(n) (0x0b04 + ((n) * 0x04))71#define REG_MISC_CFG 0x0be072#define REG_HID_CFG 0x0bf073#define REG_IDLE_STAT 0x0bf474#define REG_IP_REV_1 0x0bf875#define REG_IP_REV_2 0x0bfc76#define REG_FQD_BARE 0x0c0077#define REG_PFDR_BARE 0x0c2078#define REG_offset_BAR 0x0004 /* relative to REG_[FQD|PFDR]_BARE */79#define REG_offset_AR 0x0010 /* relative to REG_[FQD|PFDR]_BARE */80#define REG_QCSP_BARE 0x0c8081#define REG_QCSP_BAR 0x0c8482#define REG_CI_SCHED_CFG 0x0d0083#define REG_SRCIDR 0x0d0484#define REG_LIODNR 0x0d0885#define REG_CI_RLM_AVG 0x0d1486#define REG_ERR_ISR 0x0e0087#define REG_ERR_IER 0x0e0488#define REG_REV3_QCSP_LIO_CFG(n) (0x1000 + ((n) * 0x10))89#define REG_REV3_QCSP_IO_CFG(n) (0x1004 + ((n) * 0x10))90#define REG_REV3_QCSP_DD_CFG(n) (0x100c + ((n) * 0x10))91 92/* Assists for QMAN_MCR */93#define MCR_INIT_PFDR 0x0100000094#define MCR_get_rslt(v) (u8)((v) >> 24)95#define MCR_rslt_idle(r) (!(r) || ((r) >= 0xf0))96#define MCR_rslt_ok(r) ((r) == 0xf0)97#define MCR_rslt_eaccess(r) ((r) == 0xf8)98#define MCR_rslt_inval(r) ((r) == 0xff)99 100/*101 * Corenet initiator settings. Stash request queues are 4-deep to match cores102 * ability to snarf. Stash priority is 3, other priorities are 2.103 */104#define QM_CI_SCHED_CFG_SRCCIV 4105#define QM_CI_SCHED_CFG_SRQ_W 3106#define QM_CI_SCHED_CFG_RW_W 2107#define QM_CI_SCHED_CFG_BMAN_W 2108/* write SRCCIV enable */109#define QM_CI_SCHED_CFG_SRCCIV_EN BIT(31)110 111/* Follows WQ_CS_CFG0-5 */112enum qm_wq_class {113 qm_wq_portal = 0,114 qm_wq_pool = 1,115 qm_wq_fman0 = 2,116 qm_wq_fman1 = 3,117 qm_wq_caam = 4,118 qm_wq_pme = 5,119 qm_wq_first = qm_wq_portal,120 qm_wq_last = qm_wq_pme121};122 123/* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */124enum qm_memory {125 qm_memory_fqd,126 qm_memory_pfdr127};128 129/* Used by all error interrupt registers except 'inhibit' */130#define QM_EIRQ_CIDE 0x20000000 /* Corenet Initiator Data Error */131#define QM_EIRQ_CTDE 0x10000000 /* Corenet Target Data Error */132#define QM_EIRQ_CITT 0x08000000 /* Corenet Invalid Target Transaction */133#define QM_EIRQ_PLWI 0x04000000 /* PFDR Low Watermark */134#define QM_EIRQ_MBEI 0x02000000 /* Multi-bit ECC Error */135#define QM_EIRQ_SBEI 0x01000000 /* Single-bit ECC Error */136#define QM_EIRQ_PEBI 0x00800000 /* PFDR Enqueues Blocked Interrupt */137#define QM_EIRQ_IFSI 0x00020000 /* Invalid FQ Flow Control State */138#define QM_EIRQ_ICVI 0x00010000 /* Invalid Command Verb */139#define QM_EIRQ_IDDI 0x00000800 /* Invalid Dequeue (Direct-connect) */140#define QM_EIRQ_IDFI 0x00000400 /* Invalid Dequeue FQ */141#define QM_EIRQ_IDSI 0x00000200 /* Invalid Dequeue Source */142#define QM_EIRQ_IDQI 0x00000100 /* Invalid Dequeue Queue */143#define QM_EIRQ_IECE 0x00000010 /* Invalid Enqueue Configuration */144#define QM_EIRQ_IEOI 0x00000008 /* Invalid Enqueue Overflow */145#define QM_EIRQ_IESI 0x00000004 /* Invalid Enqueue State */146#define QM_EIRQ_IECI 0x00000002 /* Invalid Enqueue Channel */147#define QM_EIRQ_IEQI 0x00000001 /* Invalid Enqueue Queue */148 149/* QMAN_ECIR valid error bit */150#define PORTAL_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \151 QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \152 QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI)153#define FQID_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \154 QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \155 QM_EIRQ_IFSI)156 157struct qm_ecir {158 u32 info; /* res[30-31], ptyp[29], pnum[24-28], fqid[0-23] */159};160 161static bool qm_ecir_is_dcp(const struct qm_ecir *p)162{163 return p->info & BIT(29);164}165 166static int qm_ecir_get_pnum(const struct qm_ecir *p)167{168 return (p->info >> 24) & 0x1f;169}170 171static int qm_ecir_get_fqid(const struct qm_ecir *p)172{173 return p->info & (BIT(24) - 1);174}175 176struct qm_ecir2 {177 u32 info; /* ptyp[31], res[10-30], pnum[0-9] */178};179 180static bool qm_ecir2_is_dcp(const struct qm_ecir2 *p)181{182 return p->info & BIT(31);183}184 185static int qm_ecir2_get_pnum(const struct qm_ecir2 *p)186{187 return p->info & (BIT(10) - 1);188}189 190struct qm_eadr {191 u32 info; /* memid[24-27], eadr[0-11] */192 /* v3: memid[24-28], eadr[0-15] */193};194 195static int qm_eadr_get_memid(const struct qm_eadr *p)196{197 return (p->info >> 24) & 0xf;198}199 200static int qm_eadr_get_eadr(const struct qm_eadr *p)201{202 return p->info & (BIT(12) - 1);203}204 205static int qm_eadr_v3_get_memid(const struct qm_eadr *p)206{207 return (p->info >> 24) & 0x1f;208}209 210static int qm_eadr_v3_get_eadr(const struct qm_eadr *p)211{212 return p->info & (BIT(16) - 1);213}214 215struct qman_hwerr_txt {216 u32 mask;217 const char *txt;218};219 220 221static const struct qman_hwerr_txt qman_hwerr_txts[] = {222 { QM_EIRQ_CIDE, "Corenet Initiator Data Error" },223 { QM_EIRQ_CTDE, "Corenet Target Data Error" },224 { QM_EIRQ_CITT, "Corenet Invalid Target Transaction" },225 { QM_EIRQ_PLWI, "PFDR Low Watermark" },226 { QM_EIRQ_MBEI, "Multi-bit ECC Error" },227 { QM_EIRQ_SBEI, "Single-bit ECC Error" },228 { QM_EIRQ_PEBI, "PFDR Enqueues Blocked Interrupt" },229 { QM_EIRQ_ICVI, "Invalid Command Verb" },230 { QM_EIRQ_IFSI, "Invalid Flow Control State" },231 { QM_EIRQ_IDDI, "Invalid Dequeue (Direct-connect)" },232 { QM_EIRQ_IDFI, "Invalid Dequeue FQ" },233 { QM_EIRQ_IDSI, "Invalid Dequeue Source" },234 { QM_EIRQ_IDQI, "Invalid Dequeue Queue" },235 { QM_EIRQ_IECE, "Invalid Enqueue Configuration" },236 { QM_EIRQ_IEOI, "Invalid Enqueue Overflow" },237 { QM_EIRQ_IESI, "Invalid Enqueue State" },238 { QM_EIRQ_IECI, "Invalid Enqueue Channel" },239 { QM_EIRQ_IEQI, "Invalid Enqueue Queue" },240};241 242struct qman_error_info_mdata {243 u16 addr_mask;244 u16 bits;245 const char *txt;246};247 248static const struct qman_error_info_mdata error_mdata[] = {249 { 0x01FF, 24, "FQD cache tag memory 0" },250 { 0x01FF, 24, "FQD cache tag memory 1" },251 { 0x01FF, 24, "FQD cache tag memory 2" },252 { 0x01FF, 24, "FQD cache tag memory 3" },253 { 0x0FFF, 512, "FQD cache memory" },254 { 0x07FF, 128, "SFDR memory" },255 { 0x01FF, 72, "WQ context memory" },256 { 0x00FF, 240, "CGR memory" },257 { 0x00FF, 302, "Internal Order Restoration List memory" },258 { 0x01FF, 256, "SW portal ring memory" },259};260 261#define QMAN_ERRS_TO_DISABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI)262 263/*264 * TODO: unimplemented registers265 *266 * Keeping a list here of QMan registers I have not yet covered;267 * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR,268 * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG,269 * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12270 */271 272/* Pointer to the start of the QMan's CCSR space */273static u32 __iomem *qm_ccsr_start;274/* A SDQCR mask comprising all the available/visible pool channels */275static u32 qm_pools_sdqcr;276static int __qman_probed;277static int __qman_requires_cleanup;278 279static inline u32 qm_ccsr_in(u32 offset)280{281 return ioread32be(qm_ccsr_start + offset/4);282}283 284static inline void qm_ccsr_out(u32 offset, u32 val)285{286 iowrite32be(val, qm_ccsr_start + offset/4);287}288 289u32 qm_get_pools_sdqcr(void)290{291 return qm_pools_sdqcr;292}293 294enum qm_dc_portal {295 qm_dc_portal_fman0 = 0,296 qm_dc_portal_fman1 = 1297};298 299static void qm_set_dc(enum qm_dc_portal portal, int ed, u8 sernd)300{301 DPAA_ASSERT(!ed || portal == qm_dc_portal_fman0 ||302 portal == qm_dc_portal_fman1);303 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)304 qm_ccsr_out(REG_DCP_CFG(portal),305 (ed ? 0x1000 : 0) | (sernd & 0x3ff));306 else307 qm_ccsr_out(REG_DCP_CFG(portal),308 (ed ? 0x100 : 0) | (sernd & 0x1f));309}310 311static void qm_set_wq_scheduling(enum qm_wq_class wq_class,312 u8 cs_elev, u8 csw2, u8 csw3, u8 csw4,313 u8 csw5, u8 csw6, u8 csw7)314{315 qm_ccsr_out(REG_WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) |316 ((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) |317 ((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) |318 ((csw6 & 0x7) << 4) | (csw7 & 0x7));319}320 321static void qm_set_hid(void)322{323 qm_ccsr_out(REG_HID_CFG, 0);324}325 326static void qm_set_corenet_initiator(void)327{328 qm_ccsr_out(REG_CI_SCHED_CFG, QM_CI_SCHED_CFG_SRCCIV_EN |329 (QM_CI_SCHED_CFG_SRCCIV << 24) |330 (QM_CI_SCHED_CFG_SRQ_W << 8) |331 (QM_CI_SCHED_CFG_RW_W << 4) |332 QM_CI_SCHED_CFG_BMAN_W);333}334 335static void qm_get_version(u16 *id, u8 *major, u8 *minor)336{337 u32 v = qm_ccsr_in(REG_IP_REV_1);338 *id = (v >> 16);339 *major = (v >> 8) & 0xff;340 *minor = v & 0xff;341}342 343#define PFDR_AR_EN BIT(31)344static int qm_set_memory(enum qm_memory memory, u64 ba, u32 size)345{346 void *ptr;347 u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE;348 u32 exp = ilog2(size);349 u32 bar, bare;350 351 /* choke if size isn't within range */352 DPAA_ASSERT((size >= 4096) && (size <= 1024*1024*1024) &&353 is_power_of_2(size));354 /* choke if 'ba' has lower-alignment than 'size' */355 DPAA_ASSERT(!(ba & (size - 1)));356 357 /* Check to see if QMan has already been initialized */358 bar = qm_ccsr_in(offset + REG_offset_BAR);359 if (bar) {360 /* Maker sure ba == what was programmed) */361 bare = qm_ccsr_in(offset);362 if (bare != upper_32_bits(ba) || bar != lower_32_bits(ba)) {363 pr_err("Attempted to reinitialize QMan with different BAR, got 0x%llx read BARE=0x%x BAR=0x%x\n",364 ba, bare, bar);365 return -ENOMEM;366 }367 __qman_requires_cleanup = 1;368 /* Return 1 to indicate memory was previously programmed */369 return 1;370 }371 /* Need to temporarily map the area to make sure it is zeroed */372 ptr = memremap(ba, size, MEMREMAP_WB);373 if (!ptr) {374 pr_crit("memremap() of QMan private memory failed\n");375 return -ENOMEM;376 }377 memset(ptr, 0, size);378 379#ifdef CONFIG_PPC380 /*381 * PPC doesn't appear to flush the cache on memunmap() but the382 * cache must be flushed since QMan does non coherent accesses383 * to this memory384 */385 flush_dcache_range((unsigned long) ptr, (unsigned long) ptr+size);386#endif387 memunmap(ptr);388 389 qm_ccsr_out(offset, upper_32_bits(ba));390 qm_ccsr_out(offset + REG_offset_BAR, lower_32_bits(ba));391 qm_ccsr_out(offset + REG_offset_AR, PFDR_AR_EN | (exp - 1));392 return 0;393}394 395static void qm_set_pfdr_threshold(u32 th, u8 k)396{397 qm_ccsr_out(REG_PFDR_FP_LWIT, th & 0xffffff);398 qm_ccsr_out(REG_PFDR_CFG, k);399}400 401static void qm_set_sfdr_threshold(u16 th)402{403 qm_ccsr_out(REG_SFDR_CFG, th & 0x3ff);404}405 406static int qm_init_pfdr(struct device *dev, u32 pfdr_start, u32 num)407{408 u8 rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR));409 410 DPAA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num);411 /* Make sure the command interface is 'idle' */412 if (!MCR_rslt_idle(rslt)) {413 dev_crit(dev, "QMAN_MCR isn't idle");414 WARN_ON(1);415 }416 417 /* Write the MCR command params then the verb */418 qm_ccsr_out(REG_MCP(0), pfdr_start);419 /*420 * TODO: remove this - it's a workaround for a model bug that is421 * corrected in more recent versions. We use the workaround until422 * everyone has upgraded.423 */424 qm_ccsr_out(REG_MCP(1), pfdr_start + num - 16);425 dma_wmb();426 qm_ccsr_out(REG_MCR, MCR_INIT_PFDR);427 /* Poll for the result */428 do {429 rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR));430 } while (!MCR_rslt_idle(rslt));431 if (MCR_rslt_ok(rslt))432 return 0;433 if (MCR_rslt_eaccess(rslt))434 return -EACCES;435 if (MCR_rslt_inval(rslt))436 return -EINVAL;437 dev_crit(dev, "Unexpected result from MCR_INIT_PFDR: %02x\n", rslt);438 return -ENODEV;439}440 441/*442 * QMan needs two global memory areas initialized at boot time:443 * 1) FQD: Frame Queue Descriptors used to manage frame queues444 * 2) PFDR: Packed Frame Queue Descriptor Records used to store frames445 * Both areas are reserved using the device tree reserved memory framework446 * and the addresses and sizes are initialized when the QMan device is probed447 */448static dma_addr_t fqd_a, pfdr_a;449static size_t fqd_sz, pfdr_sz;450 451#ifdef CONFIG_PPC452/*453 * Support for PPC Device Tree backward compatibility when compatible454 * string is set to fsl-qman-fqd and fsl-qman-pfdr455 */456static int zero_priv_mem(phys_addr_t addr, size_t sz)457{458 /* map as cacheable, non-guarded */459 void __iomem *tmpp = ioremap_cache(addr, sz);460 461 if (!tmpp)462 return -ENOMEM;463 464 memset_io(tmpp, 0, sz);465 flush_dcache_range((unsigned long)tmpp,466 (unsigned long)tmpp + sz);467 iounmap(tmpp);468 469 return 0;470}471#endif472 473unsigned int qm_get_fqid_maxcnt(void)474{475 return fqd_sz / 64;476}477 478static void log_edata_bits(struct device *dev, u32 bit_count)479{480 u32 i, j, mask = 0xffffffff;481 482 dev_warn(dev, "ErrInt, EDATA:\n");483 i = bit_count / 32;484 if (bit_count % 32) {485 i++;486 mask = ~(mask << bit_count % 32);487 }488 j = 16 - i;489 dev_warn(dev, " 0x%08x\n", qm_ccsr_in(REG_EDATA(j)) & mask);490 j++;491 for (; j < 16; j++)492 dev_warn(dev, " 0x%08x\n", qm_ccsr_in(REG_EDATA(j)));493}494 495static void log_additional_error_info(struct device *dev, u32 isr_val,496 u32 ecsr_val)497{498 struct qm_ecir ecir_val;499 struct qm_eadr eadr_val;500 int memid;501 502 ecir_val.info = qm_ccsr_in(REG_ECIR);503 /* Is portal info valid */504 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {505 struct qm_ecir2 ecir2_val;506 507 ecir2_val.info = qm_ccsr_in(REG_ECIR2);508 if (ecsr_val & PORTAL_ECSR_ERR) {509 dev_warn(dev, "ErrInt: %s id %d\n",510 qm_ecir2_is_dcp(&ecir2_val) ? "DCP" : "SWP",511 qm_ecir2_get_pnum(&ecir2_val));512 }513 if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE))514 dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n",515 qm_ecir_get_fqid(&ecir_val));516 517 if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {518 eadr_val.info = qm_ccsr_in(REG_EADR);519 memid = qm_eadr_v3_get_memid(&eadr_val);520 dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n",521 error_mdata[memid].txt,522 error_mdata[memid].addr_mask523 & qm_eadr_v3_get_eadr(&eadr_val));524 log_edata_bits(dev, error_mdata[memid].bits);525 }526 } else {527 if (ecsr_val & PORTAL_ECSR_ERR) {528 dev_warn(dev, "ErrInt: %s id %d\n",529 qm_ecir_is_dcp(&ecir_val) ? "DCP" : "SWP",530 qm_ecir_get_pnum(&ecir_val));531 }532 if (ecsr_val & FQID_ECSR_ERR)533 dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n",534 qm_ecir_get_fqid(&ecir_val));535 536 if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {537 eadr_val.info = qm_ccsr_in(REG_EADR);538 memid = qm_eadr_get_memid(&eadr_val);539 dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n",540 error_mdata[memid].txt,541 error_mdata[memid].addr_mask542 & qm_eadr_get_eadr(&eadr_val));543 log_edata_bits(dev, error_mdata[memid].bits);544 }545 }546}547 548static irqreturn_t qman_isr(int irq, void *ptr)549{550 u32 isr_val, ier_val, ecsr_val, isr_mask, i;551 struct device *dev = ptr;552 553 ier_val = qm_ccsr_in(REG_ERR_IER);554 isr_val = qm_ccsr_in(REG_ERR_ISR);555 ecsr_val = qm_ccsr_in(REG_ECSR);556 isr_mask = isr_val & ier_val;557 558 if (!isr_mask)559 return IRQ_NONE;560 561 for (i = 0; i < ARRAY_SIZE(qman_hwerr_txts); i++) {562 if (qman_hwerr_txts[i].mask & isr_mask) {563 dev_err_ratelimited(dev, "ErrInt: %s\n",564 qman_hwerr_txts[i].txt);565 if (qman_hwerr_txts[i].mask & ecsr_val) {566 log_additional_error_info(dev, isr_mask,567 ecsr_val);568 /* Re-arm error capture registers */569 qm_ccsr_out(REG_ECSR, ecsr_val);570 }571 if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_DISABLE) {572 dev_dbg(dev, "Disabling error 0x%x\n",573 qman_hwerr_txts[i].mask);574 ier_val &= ~qman_hwerr_txts[i].mask;575 qm_ccsr_out(REG_ERR_IER, ier_val);576 }577 }578 }579 qm_ccsr_out(REG_ERR_ISR, isr_val);580 581 return IRQ_HANDLED;582}583 584static int qman_init_ccsr(struct device *dev)585{586 int i, err;587 588 /* FQD memory */589 err = qm_set_memory(qm_memory_fqd, fqd_a, fqd_sz);590 if (err < 0)591 return err;592 /* PFDR memory */593 err = qm_set_memory(qm_memory_pfdr, pfdr_a, pfdr_sz);594 if (err < 0)595 return err;596 /* Only initialize PFDRs if the QMan was not initialized before */597 if (err == 0) {598 err = qm_init_pfdr(dev, 8, pfdr_sz / 64 - 8);599 if (err)600 return err;601 }602 /* thresholds */603 qm_set_pfdr_threshold(512, 64);604 qm_set_sfdr_threshold(128);605 /* clear stale PEBI bit from interrupt status register */606 qm_ccsr_out(REG_ERR_ISR, QM_EIRQ_PEBI);607 /* corenet initiator settings */608 qm_set_corenet_initiator();609 /* HID settings */610 qm_set_hid();611 /* Set scheduling weights to defaults */612 for (i = qm_wq_first; i <= qm_wq_last; i++)613 qm_set_wq_scheduling(i, 0, 0, 0, 0, 0, 0, 0);614 /* We are not prepared to accept ERNs for hardware enqueues */615 qm_set_dc(qm_dc_portal_fman0, 1, 0);616 qm_set_dc(qm_dc_portal_fman1, 1, 0);617 return 0;618}619 620#define LIO_CFG_LIODN_MASK 0x0fff0000621void __qman_liodn_fixup(u16 channel)622{623 static int done;624 static u32 liodn_offset;625 u32 before, after;626 int idx = channel - QM_CHANNEL_SWPORTAL0;627 628 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)629 before = qm_ccsr_in(REG_REV3_QCSP_LIO_CFG(idx));630 else631 before = qm_ccsr_in(REG_QCSP_LIO_CFG(idx));632 if (!done) {633 liodn_offset = before & LIO_CFG_LIODN_MASK;634 done = 1;635 return;636 }637 after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset;638 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)639 qm_ccsr_out(REG_REV3_QCSP_LIO_CFG(idx), after);640 else641 qm_ccsr_out(REG_QCSP_LIO_CFG(idx), after);642}643 644#define IO_CFG_SDEST_MASK 0x00ff0000645void qman_set_sdest(u16 channel, unsigned int cpu_idx)646{647 int idx = channel - QM_CHANNEL_SWPORTAL0;648 u32 before, after;649 650 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {651 before = qm_ccsr_in(REG_REV3_QCSP_IO_CFG(idx));652 /* Each pair of vcpu share the same SRQ(SDEST) */653 cpu_idx /= 2;654 after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);655 qm_ccsr_out(REG_REV3_QCSP_IO_CFG(idx), after);656 } else {657 before = qm_ccsr_in(REG_QCSP_IO_CFG(idx));658 after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);659 qm_ccsr_out(REG_QCSP_IO_CFG(idx), after);660 }661}662 663static int qman_resource_init(struct device *dev)664{665 int pool_chan_num, cgrid_num;666 int ret, i;667 668 switch (qman_ip_rev >> 8) {669 case 1:670 pool_chan_num = 15;671 cgrid_num = 256;672 break;673 case 2:674 pool_chan_num = 3;675 cgrid_num = 64;676 break;677 case 3:678 pool_chan_num = 15;679 cgrid_num = 256;680 break;681 default:682 return -ENODEV;683 }684 685 ret = gen_pool_add(qm_qpalloc, qm_channel_pool1 | DPAA_GENALLOC_OFF,686 pool_chan_num, -1);687 if (ret) {688 dev_err(dev, "Failed to seed pool channels (%d)\n", ret);689 return ret;690 }691 692 ret = gen_pool_add(qm_cgralloc, DPAA_GENALLOC_OFF, cgrid_num, -1);693 if (ret) {694 dev_err(dev, "Failed to seed CGRID range (%d)\n", ret);695 return ret;696 }697 698 /* parse pool channels into the SDQCR mask */699 for (i = 0; i < cgrid_num; i++)700 qm_pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(i);701 702 ret = gen_pool_add(qm_fqalloc, QM_FQID_RANGE_START | DPAA_GENALLOC_OFF,703 qm_get_fqid_maxcnt() - QM_FQID_RANGE_START, -1);704 if (ret) {705 dev_err(dev, "Failed to seed FQID range (%d)\n", ret);706 return ret;707 }708 709 return 0;710}711 712int qman_is_probed(void)713{714 return __qman_probed;715}716EXPORT_SYMBOL_GPL(qman_is_probed);717 718int qman_requires_cleanup(void)719{720 return __qman_requires_cleanup;721}722 723void qman_done_cleanup(void)724{725 qman_enable_irqs();726 __qman_requires_cleanup = 0;727}728 729 730static int fsl_qman_probe(struct platform_device *pdev)731{732 struct device *dev = &pdev->dev;733 struct device_node *node = dev->of_node;734 struct resource *res;735 int ret, err_irq;736 u16 id;737 u8 major, minor;738 739 __qman_probed = -1;740 741 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);742 if (!res) {743 dev_err(dev, "Can't get %pOF property 'IORESOURCE_MEM'\n",744 node);745 return -ENXIO;746 }747 qm_ccsr_start = devm_ioremap(dev, res->start, resource_size(res));748 if (!qm_ccsr_start)749 return -ENXIO;750 751 qm_get_version(&id, &major, &minor);752 if (major == 1 && minor == 0) {753 dev_err(dev, "Rev1.0 on P4080 rev1 is not supported!\n");754 return -ENODEV;755 } else if (major == 1 && minor == 1)756 qman_ip_rev = QMAN_REV11;757 else if (major == 1 && minor == 2)758 qman_ip_rev = QMAN_REV12;759 else if (major == 2 && minor == 0)760 qman_ip_rev = QMAN_REV20;761 else if (major == 3 && minor == 0)762 qman_ip_rev = QMAN_REV30;763 else if (major == 3 && minor == 1)764 qman_ip_rev = QMAN_REV31;765 else if (major == 3 && minor == 2)766 qman_ip_rev = QMAN_REV32;767 else {768 dev_err(dev, "Unknown QMan version\n");769 return -ENODEV;770 }771 772 if ((qman_ip_rev & 0xff00) >= QMAN_REV30) {773 qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3;774 qm_channel_caam = QMAN_CHANNEL_CAAM_REV3;775 }776 777 /*778 * Order of memory regions is assumed as FQD followed by PFDR779 * in order to ensure allocations from the correct regions the780 * driver initializes then allocates each piece in order781 */782 ret = qbman_init_private_mem(dev, 0, "fsl,qman-fqd", &fqd_a, &fqd_sz);783 if (ret) {784 dev_err(dev, "qbman_init_private_mem() for FQD failed 0x%x\n",785 ret);786 return -ENODEV;787 }788#ifdef CONFIG_PPC789 /*790 * For PPC backward DT compatibility791 * FQD memory MUST be zero'd by software792 */793 zero_priv_mem(fqd_a, fqd_sz);794#endif795 dev_dbg(dev, "Allocated FQD 0x%llx 0x%zx\n", fqd_a, fqd_sz);796 797 /* Setup PFDR memory */798 ret = qbman_init_private_mem(dev, 1, "fsl,qman-pfdr", &pfdr_a, &pfdr_sz);799 if (ret) {800 dev_err(dev, "qbman_init_private_mem() for PFDR failed 0x%x\n",801 ret);802 return -ENODEV;803 }804 dev_dbg(dev, "Allocated PFDR 0x%llx 0x%zx\n", pfdr_a, pfdr_sz);805 806 ret = qman_init_ccsr(dev);807 if (ret) {808 dev_err(dev, "CCSR setup failed\n");809 return ret;810 }811 812 err_irq = platform_get_irq(pdev, 0);813 if (err_irq <= 0) {814 dev_info(dev, "Can't get %pOF property 'interrupts'\n",815 node);816 return -ENODEV;817 }818 ret = devm_request_irq(dev, err_irq, qman_isr, IRQF_SHARED, "qman-err",819 dev);820 if (ret) {821 dev_err(dev, "devm_request_irq() failed %d for '%pOF'\n",822 ret, node);823 return ret;824 }825 826 /*827 * Write-to-clear any stale bits, (eg. starvation being asserted prior828 * to resource allocation during driver init).829 */830 qm_ccsr_out(REG_ERR_ISR, 0xffffffff);831 /* Enable Error Interrupts */832 qm_ccsr_out(REG_ERR_IER, 0xffffffff);833 834 qm_fqalloc = devm_gen_pool_create(dev, 0, -1, "qman-fqalloc");835 if (IS_ERR(qm_fqalloc)) {836 ret = PTR_ERR(qm_fqalloc);837 dev_err(dev, "qman-fqalloc pool init failed (%d)\n", ret);838 return ret;839 }840 841 qm_qpalloc = devm_gen_pool_create(dev, 0, -1, "qman-qpalloc");842 if (IS_ERR(qm_qpalloc)) {843 ret = PTR_ERR(qm_qpalloc);844 dev_err(dev, "qman-qpalloc pool init failed (%d)\n", ret);845 return ret;846 }847 848 qm_cgralloc = devm_gen_pool_create(dev, 0, -1, "qman-cgralloc");849 if (IS_ERR(qm_cgralloc)) {850 ret = PTR_ERR(qm_cgralloc);851 dev_err(dev, "qman-cgralloc pool init failed (%d)\n", ret);852 return ret;853 }854 855 ret = qman_resource_init(dev);856 if (ret)857 return ret;858 859 ret = qman_alloc_fq_table(qm_get_fqid_maxcnt());860 if (ret)861 return ret;862 863 ret = qman_wq_alloc();864 if (ret)865 return ret;866 867 __qman_probed = 1;868 869 return 0;870}871 872static const struct of_device_id fsl_qman_ids[] = {873 {874 .compatible = "fsl,qman",875 },876 {}877};878 879static struct platform_driver fsl_qman_driver = {880 .driver = {881 .name = KBUILD_MODNAME,882 .of_match_table = fsl_qman_ids,883 .suppress_bind_attrs = true,884 },885 .probe = fsl_qman_probe,886};887 888builtin_platform_driver(fsl_qman_driver);889