1685 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2// Copyright (C) 2017 Broadcom3 4/*5 * Broadcom FlexRM Mailbox Driver6 *7 * Each Broadcom FlexSparx4 offload engine is implemented as an8 * extension to Broadcom FlexRM ring manager. The FlexRM ring9 * manager provides a set of rings which can be used to submit10 * work to a FlexSparx4 offload engine.11 *12 * This driver creates a mailbox controller using a set of FlexRM13 * rings where each mailbox channel represents a separate FlexRM ring.14 */15 16#include <asm/barrier.h>17#include <asm/byteorder.h>18#include <linux/atomic.h>19#include <linux/bitmap.h>20#include <linux/debugfs.h>21#include <linux/delay.h>22#include <linux/device.h>23#include <linux/dma-mapping.h>24#include <linux/dmapool.h>25#include <linux/err.h>26#include <linux/interrupt.h>27#include <linux/kernel.h>28#include <linux/mailbox_controller.h>29#include <linux/mailbox_client.h>30#include <linux/mailbox/brcm-message.h>31#include <linux/module.h>32#include <linux/msi.h>33#include <linux/of_address.h>34#include <linux/of_irq.h>35#include <linux/platform_device.h>36#include <linux/spinlock.h>37 38/* ====== FlexRM register defines ===== */39 40/* FlexRM configuration */41#define RING_REGS_SIZE 0x1000042#define RING_DESC_SIZE 843#define RING_DESC_INDEX(offset) \44 ((offset) / RING_DESC_SIZE)45#define RING_DESC_OFFSET(index) \46 ((index) * RING_DESC_SIZE)47#define RING_MAX_REQ_COUNT 102448#define RING_BD_ALIGN_ORDER 1249#define RING_BD_ALIGN_CHECK(addr) \50 (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))51#define RING_BD_TOGGLE_INVALID(offset) \52 (((offset) >> RING_BD_ALIGN_ORDER) & 0x1)53#define RING_BD_TOGGLE_VALID(offset) \54 (!RING_BD_TOGGLE_INVALID(offset))55#define RING_BD_DESC_PER_REQ 3256#define RING_BD_DESC_COUNT \57 (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)58#define RING_BD_SIZE \59 (RING_BD_DESC_COUNT * RING_DESC_SIZE)60#define RING_CMPL_ALIGN_ORDER 1361#define RING_CMPL_DESC_COUNT RING_MAX_REQ_COUNT62#define RING_CMPL_SIZE \63 (RING_CMPL_DESC_COUNT * RING_DESC_SIZE)64#define RING_VER_MAGIC 0x7630303165 66/* Per-Ring register offsets */67#define RING_VER 0x00068#define RING_BD_START_ADDR 0x00469#define RING_BD_READ_PTR 0x00870#define RING_BD_WRITE_PTR 0x00c71#define RING_BD_READ_PTR_DDR_LS 0x01072#define RING_BD_READ_PTR_DDR_MS 0x01473#define RING_CMPL_START_ADDR 0x01874#define RING_CMPL_WRITE_PTR 0x01c75#define RING_NUM_REQ_RECV_LS 0x02076#define RING_NUM_REQ_RECV_MS 0x02477#define RING_NUM_REQ_TRANS_LS 0x02878#define RING_NUM_REQ_TRANS_MS 0x02c79#define RING_NUM_REQ_OUTSTAND 0x03080#define RING_CONTROL 0x03481#define RING_FLUSH_DONE 0x03882#define RING_MSI_ADDR_LS 0x03c83#define RING_MSI_ADDR_MS 0x04084#define RING_MSI_CONTROL 0x04885#define RING_BD_READ_PTR_DDR_CONTROL 0x04c86#define RING_MSI_DATA_VALUE 0x06487 88/* Register RING_BD_START_ADDR fields */89#define BD_LAST_UPDATE_HW_SHIFT 2890#define BD_LAST_UPDATE_HW_MASK 0x191#define BD_START_ADDR_VALUE(pa) \92 ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))93#define BD_START_ADDR_DECODE(val) \94 ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)95 96/* Register RING_CMPL_START_ADDR fields */97#define CMPL_START_ADDR_VALUE(pa) \98 ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))99 100/* Register RING_CONTROL fields */101#define CONTROL_MASK_DISABLE_CONTROL 12102#define CONTROL_FLUSH_SHIFT 5103#define CONTROL_ACTIVE_SHIFT 4104#define CONTROL_RATE_ADAPT_MASK 0xf105#define CONTROL_RATE_DYNAMIC 0x0106#define CONTROL_RATE_FAST 0x8107#define CONTROL_RATE_MEDIUM 0x9108#define CONTROL_RATE_SLOW 0xa109#define CONTROL_RATE_IDLE 0xb110 111/* Register RING_FLUSH_DONE fields */112#define FLUSH_DONE_MASK 0x1113 114/* Register RING_MSI_CONTROL fields */115#define MSI_TIMER_VAL_SHIFT 16116#define MSI_TIMER_VAL_MASK 0xffff117#define MSI_ENABLE_SHIFT 15118#define MSI_ENABLE_MASK 0x1119#define MSI_COUNT_SHIFT 0120#define MSI_COUNT_MASK 0x3ff121 122/* Register RING_BD_READ_PTR_DDR_CONTROL fields */123#define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16124#define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff125#define BD_READ_PTR_DDR_ENABLE_SHIFT 15126#define BD_READ_PTR_DDR_ENABLE_MASK 0x1127 128/* ====== FlexRM ring descriptor defines ===== */129 130/* Completion descriptor format */131#define CMPL_OPAQUE_SHIFT 0132#define CMPL_OPAQUE_MASK 0xffff133#define CMPL_ENGINE_STATUS_SHIFT 16134#define CMPL_ENGINE_STATUS_MASK 0xffff135#define CMPL_DME_STATUS_SHIFT 32136#define CMPL_DME_STATUS_MASK 0xffff137#define CMPL_RM_STATUS_SHIFT 48138#define CMPL_RM_STATUS_MASK 0xffff139 140/* Completion DME status code */141#define DME_STATUS_MEM_COR_ERR BIT(0)142#define DME_STATUS_MEM_UCOR_ERR BIT(1)143#define DME_STATUS_FIFO_UNDERFLOW BIT(2)144#define DME_STATUS_FIFO_OVERFLOW BIT(3)145#define DME_STATUS_RRESP_ERR BIT(4)146#define DME_STATUS_BRESP_ERR BIT(5)147#define DME_STATUS_ERROR_MASK (DME_STATUS_MEM_COR_ERR | \148 DME_STATUS_MEM_UCOR_ERR | \149 DME_STATUS_FIFO_UNDERFLOW | \150 DME_STATUS_FIFO_OVERFLOW | \151 DME_STATUS_RRESP_ERR | \152 DME_STATUS_BRESP_ERR)153 154/* Completion RM status code */155#define RM_STATUS_CODE_SHIFT 0156#define RM_STATUS_CODE_MASK 0x3ff157#define RM_STATUS_CODE_GOOD 0x0158#define RM_STATUS_CODE_AE_TIMEOUT 0x3ff159 160/* General descriptor format */161#define DESC_TYPE_SHIFT 60162#define DESC_TYPE_MASK 0xf163#define DESC_PAYLOAD_SHIFT 0164#define DESC_PAYLOAD_MASK 0x0fffffffffffffff165 166/* Null descriptor format */167#define NULL_TYPE 0168#define NULL_TOGGLE_SHIFT 58169#define NULL_TOGGLE_MASK 0x1170 171/* Header descriptor format */172#define HEADER_TYPE 1173#define HEADER_TOGGLE_SHIFT 58174#define HEADER_TOGGLE_MASK 0x1175#define HEADER_ENDPKT_SHIFT 57176#define HEADER_ENDPKT_MASK 0x1177#define HEADER_STARTPKT_SHIFT 56178#define HEADER_STARTPKT_MASK 0x1179#define HEADER_BDCOUNT_SHIFT 36180#define HEADER_BDCOUNT_MASK 0x1f181#define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK182#define HEADER_FLAGS_SHIFT 16183#define HEADER_FLAGS_MASK 0xffff184#define HEADER_OPAQUE_SHIFT 0185#define HEADER_OPAQUE_MASK 0xffff186 187/* Source (SRC) descriptor format */188#define SRC_TYPE 2189#define SRC_LENGTH_SHIFT 44190#define SRC_LENGTH_MASK 0xffff191#define SRC_ADDR_SHIFT 0192#define SRC_ADDR_MASK 0x00000fffffffffff193 194/* Destination (DST) descriptor format */195#define DST_TYPE 3196#define DST_LENGTH_SHIFT 44197#define DST_LENGTH_MASK 0xffff198#define DST_ADDR_SHIFT 0199#define DST_ADDR_MASK 0x00000fffffffffff200 201/* Immediate (IMM) descriptor format */202#define IMM_TYPE 4203#define IMM_DATA_SHIFT 0204#define IMM_DATA_MASK 0x0fffffffffffffff205 206/* Next pointer (NPTR) descriptor format */207#define NPTR_TYPE 5208#define NPTR_TOGGLE_SHIFT 58209#define NPTR_TOGGLE_MASK 0x1210#define NPTR_ADDR_SHIFT 0211#define NPTR_ADDR_MASK 0x00000fffffffffff212 213/* Mega source (MSRC) descriptor format */214#define MSRC_TYPE 6215#define MSRC_LENGTH_SHIFT 44216#define MSRC_LENGTH_MASK 0xffff217#define MSRC_ADDR_SHIFT 0218#define MSRC_ADDR_MASK 0x00000fffffffffff219 220/* Mega destination (MDST) descriptor format */221#define MDST_TYPE 7222#define MDST_LENGTH_SHIFT 44223#define MDST_LENGTH_MASK 0xffff224#define MDST_ADDR_SHIFT 0225#define MDST_ADDR_MASK 0x00000fffffffffff226 227/* Source with tlast (SRCT) descriptor format */228#define SRCT_TYPE 8229#define SRCT_LENGTH_SHIFT 44230#define SRCT_LENGTH_MASK 0xffff231#define SRCT_ADDR_SHIFT 0232#define SRCT_ADDR_MASK 0x00000fffffffffff233 234/* Destination with tlast (DSTT) descriptor format */235#define DSTT_TYPE 9236#define DSTT_LENGTH_SHIFT 44237#define DSTT_LENGTH_MASK 0xffff238#define DSTT_ADDR_SHIFT 0239#define DSTT_ADDR_MASK 0x00000fffffffffff240 241/* Immediate with tlast (IMMT) descriptor format */242#define IMMT_TYPE 10243#define IMMT_DATA_SHIFT 0244#define IMMT_DATA_MASK 0x0fffffffffffffff245 246/* Descriptor helper macros */247#define DESC_DEC(_d, _s, _m) (((_d) >> (_s)) & (_m))248#define DESC_ENC(_d, _v, _s, _m) \249 do { \250 (_d) &= ~((u64)(_m) << (_s)); \251 (_d) |= (((u64)(_v) & (_m)) << (_s)); \252 } while (0)253 254/* ====== FlexRM data structures ===== */255 256struct flexrm_ring {257 /* Unprotected members */258 int num;259 struct flexrm_mbox *mbox;260 void __iomem *regs;261 bool irq_requested;262 unsigned int irq;263 cpumask_t irq_aff_hint;264 unsigned int msi_timer_val;265 unsigned int msi_count_threshold;266 struct brcm_message *requests[RING_MAX_REQ_COUNT];267 void *bd_base;268 dma_addr_t bd_dma_base;269 u32 bd_write_offset;270 void *cmpl_base;271 dma_addr_t cmpl_dma_base;272 /* Atomic stats */273 atomic_t msg_send_count;274 atomic_t msg_cmpl_count;275 /* Protected members */276 spinlock_t lock;277 DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);278 u32 cmpl_read_offset;279};280 281struct flexrm_mbox {282 struct device *dev;283 void __iomem *regs;284 u32 num_rings;285 struct flexrm_ring *rings;286 struct dma_pool *bd_pool;287 struct dma_pool *cmpl_pool;288 struct dentry *root;289 struct mbox_controller controller;290};291 292/* ====== FlexRM ring descriptor helper routines ===== */293 294static u64 flexrm_read_desc(void *desc_ptr)295{296 return le64_to_cpu(*((u64 *)desc_ptr));297}298 299static void flexrm_write_desc(void *desc_ptr, u64 desc)300{301 *((u64 *)desc_ptr) = cpu_to_le64(desc);302}303 304static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)305{306 return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);307}308 309static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)310{311 u32 status;312 313 status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,314 CMPL_DME_STATUS_MASK);315 if (status & DME_STATUS_ERROR_MASK)316 return -EIO;317 318 status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,319 CMPL_RM_STATUS_MASK);320 status &= RM_STATUS_CODE_MASK;321 if (status == RM_STATUS_CODE_AE_TIMEOUT)322 return -ETIMEDOUT;323 324 return 0;325}326 327static bool flexrm_is_next_table_desc(void *desc_ptr)328{329 u64 desc = flexrm_read_desc(desc_ptr);330 u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);331 332 return (type == NPTR_TYPE) ? true : false;333}334 335static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)336{337 u64 desc = 0;338 339 DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);340 DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);341 DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);342 343 return desc;344}345 346static u64 flexrm_null_desc(u32 toggle)347{348 u64 desc = 0;349 350 DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);351 DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);352 353 return desc;354}355 356static u32 flexrm_estimate_header_desc_count(u32 nhcnt)357{358 u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;359 360 if (!(nhcnt % HEADER_BDCOUNT_MAX))361 hcnt += 1;362 363 return hcnt;364}365 366static void flexrm_flip_header_toggle(void *desc_ptr)367{368 u64 desc = flexrm_read_desc(desc_ptr);369 370 if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))371 desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);372 else373 desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);374 375 flexrm_write_desc(desc_ptr, desc);376}377 378static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,379 u32 bdcount, u32 flags, u32 opaque)380{381 u64 desc = 0;382 383 DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);384 DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);385 DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);386 DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);387 DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);388 DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);389 DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);390 391 return desc;392}393 394static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,395 u64 desc, void **desc_ptr, u32 *toggle,396 void *start_desc, void *end_desc)397{398 u64 d;399 u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;400 401 /* Sanity check */402 if (nhcnt <= nhpos)403 return;404 405 /*406 * Each request or packet start with a HEADER descriptor followed407 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,408 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors409 * following a HEADER descriptor is represented by BDCOUNT field410 * of HEADER descriptor. The max value of BDCOUNT field is 31 which411 * means we can only have 31 non-HEADER descriptors following one412 * HEADER descriptor.413 *414 * In general use, number of non-HEADER descriptors can easily go415 * beyond 31. To tackle this situation, we have packet (or request)416 * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.417 *418 * To use packet extension, the first HEADER descriptor of request419 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate420 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last421 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the422 * TOGGLE bit of the first HEADER will be set to invalid state to423 * ensure that FlexRM does not start fetching descriptors till all424 * descriptors are enqueued. The user of this function will flip425 * the TOGGLE bit of first HEADER after all descriptors are426 * enqueued.427 */428 429 if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {430 /* Prepare the header descriptor */431 nhavail = (nhcnt - nhpos);432 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);433 _startpkt = (nhpos == 0) ? 0x1 : 0x0;434 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;435 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?436 nhavail : HEADER_BDCOUNT_MAX;437 if (nhavail <= HEADER_BDCOUNT_MAX)438 _bdcount = nhavail;439 else440 _bdcount = HEADER_BDCOUNT_MAX;441 d = flexrm_header_desc(_toggle, _startpkt, _endpkt,442 _bdcount, 0x0, reqid);443 444 /* Write header descriptor */445 flexrm_write_desc(*desc_ptr, d);446 447 /* Point to next descriptor */448 *desc_ptr += sizeof(desc);449 if (*desc_ptr == end_desc)450 *desc_ptr = start_desc;451 452 /* Skip next pointer descriptors */453 while (flexrm_is_next_table_desc(*desc_ptr)) {454 *toggle = (*toggle) ? 0 : 1;455 *desc_ptr += sizeof(desc);456 if (*desc_ptr == end_desc)457 *desc_ptr = start_desc;458 }459 }460 461 /* Write desired descriptor */462 flexrm_write_desc(*desc_ptr, desc);463 464 /* Point to next descriptor */465 *desc_ptr += sizeof(desc);466 if (*desc_ptr == end_desc)467 *desc_ptr = start_desc;468 469 /* Skip next pointer descriptors */470 while (flexrm_is_next_table_desc(*desc_ptr)) {471 *toggle = (*toggle) ? 0 : 1;472 *desc_ptr += sizeof(desc);473 if (*desc_ptr == end_desc)474 *desc_ptr = start_desc;475 }476}477 478static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)479{480 u64 desc = 0;481 482 DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);483 DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);484 DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);485 486 return desc;487}488 489static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)490{491 u64 desc = 0;492 493 DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);494 DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);495 DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);496 497 return desc;498}499 500static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)501{502 u64 desc = 0;503 504 DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);505 DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);506 DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);507 508 return desc;509}510 511static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)512{513 u64 desc = 0;514 515 DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);516 DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);517 DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);518 519 return desc;520}521 522static u64 flexrm_imm_desc(u64 data)523{524 u64 desc = 0;525 526 DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);527 DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);528 529 return desc;530}531 532static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)533{534 u64 desc = 0;535 536 DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);537 DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);538 DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);539 540 return desc;541}542 543static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)544{545 u64 desc = 0;546 547 DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);548 DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);549 DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);550 551 return desc;552}553 554static u64 flexrm_immt_desc(u64 data)555{556 u64 desc = 0;557 558 DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);559 DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);560 561 return desc;562}563 564static bool flexrm_spu_sanity_check(struct brcm_message *msg)565{566 struct scatterlist *sg;567 568 if (!msg->spu.src || !msg->spu.dst)569 return false;570 for (sg = msg->spu.src; sg; sg = sg_next(sg)) {571 if (sg->length & 0xf) {572 if (sg->length > SRC_LENGTH_MASK)573 return false;574 } else {575 if (sg->length > (MSRC_LENGTH_MASK * 16))576 return false;577 }578 }579 for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {580 if (sg->length & 0xf) {581 if (sg->length > DST_LENGTH_MASK)582 return false;583 } else {584 if (sg->length > (MDST_LENGTH_MASK * 16))585 return false;586 }587 }588 589 return true;590}591 592static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)593{594 u32 cnt = 0;595 unsigned int dst_target = 0;596 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;597 598 while (src_sg || dst_sg) {599 if (src_sg) {600 cnt++;601 dst_target = src_sg->length;602 src_sg = sg_next(src_sg);603 } else604 dst_target = UINT_MAX;605 606 while (dst_target && dst_sg) {607 cnt++;608 if (dst_sg->length < dst_target)609 dst_target -= dst_sg->length;610 else611 dst_target = 0;612 dst_sg = sg_next(dst_sg);613 }614 }615 616 return cnt;617}618 619static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)620{621 int rc;622 623 rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),624 DMA_TO_DEVICE);625 if (!rc)626 return -EIO;627 628 rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),629 DMA_FROM_DEVICE);630 if (!rc) {631 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),632 DMA_TO_DEVICE);633 return -EIO;634 }635 636 return 0;637}638 639static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)640{641 dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),642 DMA_FROM_DEVICE);643 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),644 DMA_TO_DEVICE);645}646 647static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,648 u32 reqid, void *desc_ptr, u32 toggle,649 void *start_desc, void *end_desc)650{651 u64 d;652 u32 nhpos = 0;653 void *orig_desc_ptr = desc_ptr;654 unsigned int dst_target = 0;655 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;656 657 while (src_sg || dst_sg) {658 if (src_sg) {659 if (sg_dma_len(src_sg) & 0xf)660 d = flexrm_src_desc(sg_dma_address(src_sg),661 sg_dma_len(src_sg));662 else663 d = flexrm_msrc_desc(sg_dma_address(src_sg),664 sg_dma_len(src_sg)/16);665 flexrm_enqueue_desc(nhpos, nhcnt, reqid,666 d, &desc_ptr, &toggle,667 start_desc, end_desc);668 nhpos++;669 dst_target = sg_dma_len(src_sg);670 src_sg = sg_next(src_sg);671 } else672 dst_target = UINT_MAX;673 674 while (dst_target && dst_sg) {675 if (sg_dma_len(dst_sg) & 0xf)676 d = flexrm_dst_desc(sg_dma_address(dst_sg),677 sg_dma_len(dst_sg));678 else679 d = flexrm_mdst_desc(sg_dma_address(dst_sg),680 sg_dma_len(dst_sg)/16);681 flexrm_enqueue_desc(nhpos, nhcnt, reqid,682 d, &desc_ptr, &toggle,683 start_desc, end_desc);684 nhpos++;685 if (sg_dma_len(dst_sg) < dst_target)686 dst_target -= sg_dma_len(dst_sg);687 else688 dst_target = 0;689 dst_sg = sg_next(dst_sg);690 }691 }692 693 /* Null descriptor with invalid toggle bit */694 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));695 696 /* Ensure that descriptors have been written to memory */697 wmb();698 699 /* Flip toggle bit in header */700 flexrm_flip_header_toggle(orig_desc_ptr);701 702 return desc_ptr;703}704 705static bool flexrm_sba_sanity_check(struct brcm_message *msg)706{707 u32 i;708 709 if (!msg->sba.cmds || !msg->sba.cmds_count)710 return false;711 712 for (i = 0; i < msg->sba.cmds_count; i++) {713 if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||714 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&715 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))716 return false;717 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&718 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))719 return false;720 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&721 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))722 return false;723 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&724 (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))725 return false;726 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&727 (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))728 return false;729 }730 731 return true;732}733 734static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)735{736 u32 i, cnt;737 738 cnt = 0;739 for (i = 0; i < msg->sba.cmds_count; i++) {740 cnt++;741 742 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||743 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))744 cnt++;745 746 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)747 cnt++;748 749 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)750 cnt++;751 }752 753 return cnt;754}755 756static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,757 u32 reqid, void *desc_ptr, u32 toggle,758 void *start_desc, void *end_desc)759{760 u64 d;761 u32 i, nhpos = 0;762 struct brcm_sba_command *c;763 void *orig_desc_ptr = desc_ptr;764 765 /* Convert SBA commands into descriptors */766 for (i = 0; i < msg->sba.cmds_count; i++) {767 c = &msg->sba.cmds[i];768 769 if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&770 (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {771 /* Destination response descriptor */772 d = flexrm_dst_desc(c->resp, c->resp_len);773 flexrm_enqueue_desc(nhpos, nhcnt, reqid,774 d, &desc_ptr, &toggle,775 start_desc, end_desc);776 nhpos++;777 } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {778 /* Destination response with tlast descriptor */779 d = flexrm_dstt_desc(c->resp, c->resp_len);780 flexrm_enqueue_desc(nhpos, nhcnt, reqid,781 d, &desc_ptr, &toggle,782 start_desc, end_desc);783 nhpos++;784 }785 786 if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {787 /* Destination with tlast descriptor */788 d = flexrm_dstt_desc(c->data, c->data_len);789 flexrm_enqueue_desc(nhpos, nhcnt, reqid,790 d, &desc_ptr, &toggle,791 start_desc, end_desc);792 nhpos++;793 }794 795 if (c->flags & BRCM_SBA_CMD_TYPE_B) {796 /* Command as immediate descriptor */797 d = flexrm_imm_desc(c->cmd);798 flexrm_enqueue_desc(nhpos, nhcnt, reqid,799 d, &desc_ptr, &toggle,800 start_desc, end_desc);801 nhpos++;802 } else {803 /* Command as immediate descriptor with tlast */804 d = flexrm_immt_desc(c->cmd);805 flexrm_enqueue_desc(nhpos, nhcnt, reqid,806 d, &desc_ptr, &toggle,807 start_desc, end_desc);808 nhpos++;809 }810 811 if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||812 (c->flags & BRCM_SBA_CMD_TYPE_C)) {813 /* Source with tlast descriptor */814 d = flexrm_srct_desc(c->data, c->data_len);815 flexrm_enqueue_desc(nhpos, nhcnt, reqid,816 d, &desc_ptr, &toggle,817 start_desc, end_desc);818 nhpos++;819 }820 }821 822 /* Null descriptor with invalid toggle bit */823 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));824 825 /* Ensure that descriptors have been written to memory */826 wmb();827 828 /* Flip toggle bit in header */829 flexrm_flip_header_toggle(orig_desc_ptr);830 831 return desc_ptr;832}833 834static bool flexrm_sanity_check(struct brcm_message *msg)835{836 if (!msg)837 return false;838 839 switch (msg->type) {840 case BRCM_MESSAGE_SPU:841 return flexrm_spu_sanity_check(msg);842 case BRCM_MESSAGE_SBA:843 return flexrm_sba_sanity_check(msg);844 default:845 return false;846 };847}848 849static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)850{851 if (!msg)852 return 0;853 854 switch (msg->type) {855 case BRCM_MESSAGE_SPU:856 return flexrm_spu_estimate_nonheader_desc_count(msg);857 case BRCM_MESSAGE_SBA:858 return flexrm_sba_estimate_nonheader_desc_count(msg);859 default:860 return 0;861 };862}863 864static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)865{866 if (!dev || !msg)867 return -EINVAL;868 869 switch (msg->type) {870 case BRCM_MESSAGE_SPU:871 return flexrm_spu_dma_map(dev, msg);872 default:873 break;874 }875 876 return 0;877}878 879static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)880{881 if (!dev || !msg)882 return;883 884 switch (msg->type) {885 case BRCM_MESSAGE_SPU:886 flexrm_spu_dma_unmap(dev, msg);887 break;888 default:889 break;890 }891}892 893static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,894 u32 reqid, void *desc_ptr, u32 toggle,895 void *start_desc, void *end_desc)896{897 if (!msg || !desc_ptr || !start_desc || !end_desc)898 return ERR_PTR(-ENOTSUPP);899 900 if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))901 return ERR_PTR(-ERANGE);902 903 switch (msg->type) {904 case BRCM_MESSAGE_SPU:905 return flexrm_spu_write_descs(msg, nhcnt, reqid,906 desc_ptr, toggle,907 start_desc, end_desc);908 case BRCM_MESSAGE_SBA:909 return flexrm_sba_write_descs(msg, nhcnt, reqid,910 desc_ptr, toggle,911 start_desc, end_desc);912 default:913 return ERR_PTR(-ENOTSUPP);914 };915}916 917/* ====== FlexRM driver helper routines ===== */918 919static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,920 struct seq_file *file)921{922 int i;923 const char *state;924 struct flexrm_ring *ring;925 926 seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",927 "Ring#", "State", "BD_Addr", "BD_Size",928 "Cmpl_Addr", "Cmpl_Size");929 930 for (i = 0; i < mbox->num_rings; i++) {931 ring = &mbox->rings[i];932 if (readl(ring->regs + RING_CONTROL) &933 BIT(CONTROL_ACTIVE_SHIFT))934 state = "active";935 else936 state = "inactive";937 seq_printf(file,938 "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",939 ring->num, state,940 (unsigned long long)ring->bd_dma_base,941 (u32)RING_BD_SIZE,942 (unsigned long long)ring->cmpl_dma_base,943 (u32)RING_CMPL_SIZE);944 }945}946 947static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,948 struct seq_file *file)949{950 int i;951 u32 val, bd_read_offset;952 struct flexrm_ring *ring;953 954 seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",955 "Ring#", "BD_Read", "BD_Write",956 "Cmpl_Read", "Submitted", "Completed");957 958 for (i = 0; i < mbox->num_rings; i++) {959 ring = &mbox->rings[i];960 bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);961 val = readl_relaxed(ring->regs + RING_BD_START_ADDR);962 bd_read_offset *= RING_DESC_SIZE;963 bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -964 ring->bd_dma_base);965 seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",966 ring->num,967 (u32)bd_read_offset,968 (u32)ring->bd_write_offset,969 (u32)ring->cmpl_read_offset,970 (u32)atomic_read(&ring->msg_send_count),971 (u32)atomic_read(&ring->msg_cmpl_count));972 }973}974 975static int flexrm_new_request(struct flexrm_ring *ring,976 struct brcm_message *batch_msg,977 struct brcm_message *msg)978{979 void *next;980 unsigned long flags;981 u32 val, count, nhcnt;982 u32 read_offset, write_offset;983 bool exit_cleanup = false;984 int ret = 0, reqid;985 986 /* Do sanity check on message */987 if (!flexrm_sanity_check(msg))988 return -EIO;989 msg->error = 0;990 991 /* If no requests possible then save data pointer and goto done. */992 spin_lock_irqsave(&ring->lock, flags);993 reqid = bitmap_find_free_region(ring->requests_bmap,994 RING_MAX_REQ_COUNT, 0);995 spin_unlock_irqrestore(&ring->lock, flags);996 if (reqid < 0)997 return -ENOSPC;998 ring->requests[reqid] = msg;999 1000 /* Do DMA mappings for the message */1001 ret = flexrm_dma_map(ring->mbox->dev, msg);1002 if (ret < 0) {1003 ring->requests[reqid] = NULL;1004 spin_lock_irqsave(&ring->lock, flags);1005 bitmap_release_region(ring->requests_bmap, reqid, 0);1006 spin_unlock_irqrestore(&ring->lock, flags);1007 return ret;1008 }1009 1010 /* Determine current HW BD read offset */1011 read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);1012 val = readl_relaxed(ring->regs + RING_BD_START_ADDR);1013 read_offset *= RING_DESC_SIZE;1014 read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);1015 1016 /*1017 * Number required descriptors = number of non-header descriptors +1018 * number of header descriptors +1019 * 1x null descriptor1020 */1021 nhcnt = flexrm_estimate_nonheader_desc_count(msg);1022 count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;1023 1024 /* Check for available descriptor space. */1025 write_offset = ring->bd_write_offset;1026 while (count) {1027 if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))1028 count--;1029 write_offset += RING_DESC_SIZE;1030 if (write_offset == RING_BD_SIZE)1031 write_offset = 0x0;1032 if (write_offset == read_offset)1033 break;1034 }1035 if (count) {1036 ret = -ENOSPC;1037 exit_cleanup = true;1038 goto exit;1039 }1040 1041 /* Write descriptors to ring */1042 next = flexrm_write_descs(msg, nhcnt, reqid,1043 ring->bd_base + ring->bd_write_offset,1044 RING_BD_TOGGLE_VALID(ring->bd_write_offset),1045 ring->bd_base, ring->bd_base + RING_BD_SIZE);1046 if (IS_ERR(next)) {1047 ret = PTR_ERR(next);1048 exit_cleanup = true;1049 goto exit;1050 }1051 1052 /* Save ring BD write offset */1053 ring->bd_write_offset = (unsigned long)(next - ring->bd_base);1054 1055 /* Increment number of messages sent */1056 atomic_inc_return(&ring->msg_send_count);1057 1058exit:1059 /* Update error status in message */1060 msg->error = ret;1061 1062 /* Cleanup if we failed */1063 if (exit_cleanup) {1064 flexrm_dma_unmap(ring->mbox->dev, msg);1065 ring->requests[reqid] = NULL;1066 spin_lock_irqsave(&ring->lock, flags);1067 bitmap_release_region(ring->requests_bmap, reqid, 0);1068 spin_unlock_irqrestore(&ring->lock, flags);1069 }1070 1071 return ret;1072}1073 1074static int flexrm_process_completions(struct flexrm_ring *ring)1075{1076 u64 desc;1077 int err, count = 0;1078 unsigned long flags;1079 struct brcm_message *msg = NULL;1080 u32 reqid, cmpl_read_offset, cmpl_write_offset;1081 struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];1082 1083 spin_lock_irqsave(&ring->lock, flags);1084 1085 /*1086 * Get current completion read and write offset1087 *1088 * Note: We should read completion write pointer at least once1089 * after we get a MSI interrupt because HW maintains internal1090 * MSI status which will allow next MSI interrupt only after1091 * completion write pointer is read.1092 */1093 cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);1094 cmpl_write_offset *= RING_DESC_SIZE;1095 cmpl_read_offset = ring->cmpl_read_offset;1096 ring->cmpl_read_offset = cmpl_write_offset;1097 1098 spin_unlock_irqrestore(&ring->lock, flags);1099 1100 /* For each completed request notify mailbox clients */1101 reqid = 0;1102 while (cmpl_read_offset != cmpl_write_offset) {1103 /* Dequeue next completion descriptor */1104 desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));1105 1106 /* Next read offset */1107 cmpl_read_offset += RING_DESC_SIZE;1108 if (cmpl_read_offset == RING_CMPL_SIZE)1109 cmpl_read_offset = 0;1110 1111 /* Decode error from completion descriptor */1112 err = flexrm_cmpl_desc_to_error(desc);1113 if (err < 0) {1114 dev_warn(ring->mbox->dev,1115 "ring%d got completion desc=0x%lx with error %d\n",1116 ring->num, (unsigned long)desc, err);1117 }1118 1119 /* Determine request id from completion descriptor */1120 reqid = flexrm_cmpl_desc_to_reqid(desc);1121 1122 /* Determine message pointer based on reqid */1123 msg = ring->requests[reqid];1124 if (!msg) {1125 dev_warn(ring->mbox->dev,1126 "ring%d null msg pointer for completion desc=0x%lx\n",1127 ring->num, (unsigned long)desc);1128 continue;1129 }1130 1131 /* Release reqid for recycling */1132 ring->requests[reqid] = NULL;1133 spin_lock_irqsave(&ring->lock, flags);1134 bitmap_release_region(ring->requests_bmap, reqid, 0);1135 spin_unlock_irqrestore(&ring->lock, flags);1136 1137 /* Unmap DMA mappings */1138 flexrm_dma_unmap(ring->mbox->dev, msg);1139 1140 /* Give-back message to mailbox client */1141 msg->error = err;1142 mbox_chan_received_data(chan, msg);1143 1144 /* Increment number of completions processed */1145 atomic_inc_return(&ring->msg_cmpl_count);1146 count++;1147 }1148 1149 return count;1150}1151 1152/* ====== FlexRM Debugfs callbacks ====== */1153 1154static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)1155{1156 struct flexrm_mbox *mbox = dev_get_drvdata(file->private);1157 1158 /* Write config in file */1159 flexrm_write_config_in_seqfile(mbox, file);1160 1161 return 0;1162}1163 1164static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)1165{1166 struct flexrm_mbox *mbox = dev_get_drvdata(file->private);1167 1168 /* Write stats in file */1169 flexrm_write_stats_in_seqfile(mbox, file);1170 1171 return 0;1172}1173 1174/* ====== FlexRM interrupt handler ===== */1175 1176static irqreturn_t flexrm_irq_event(int irq, void *dev_id)1177{1178 /* We only have MSI for completions so just wakeup IRQ thread */1179 /* Ring related errors will be informed via completion descriptors */1180 1181 return IRQ_WAKE_THREAD;1182}1183 1184static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)1185{1186 flexrm_process_completions(dev_id);1187 1188 return IRQ_HANDLED;1189}1190 1191/* ====== FlexRM mailbox callbacks ===== */1192 1193static int flexrm_send_data(struct mbox_chan *chan, void *data)1194{1195 int i, rc;1196 struct flexrm_ring *ring = chan->con_priv;1197 struct brcm_message *msg = data;1198 1199 if (msg->type == BRCM_MESSAGE_BATCH) {1200 for (i = msg->batch.msgs_queued;1201 i < msg->batch.msgs_count; i++) {1202 rc = flexrm_new_request(ring, msg,1203 &msg->batch.msgs[i]);1204 if (rc) {1205 msg->error = rc;1206 return rc;1207 }1208 msg->batch.msgs_queued++;1209 }1210 return 0;1211 }1212 1213 return flexrm_new_request(ring, NULL, data);1214}1215 1216static bool flexrm_peek_data(struct mbox_chan *chan)1217{1218 int cnt = flexrm_process_completions(chan->con_priv);1219 1220 return (cnt > 0) ? true : false;1221}1222 1223static int flexrm_startup(struct mbox_chan *chan)1224{1225 u64 d;1226 u32 val, off;1227 int ret = 0;1228 dma_addr_t next_addr;1229 struct flexrm_ring *ring = chan->con_priv;1230 1231 /* Allocate BD memory */1232 ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,1233 GFP_KERNEL, &ring->bd_dma_base);1234 if (!ring->bd_base) {1235 dev_err(ring->mbox->dev,1236 "can't allocate BD memory for ring%d\n",1237 ring->num);1238 ret = -ENOMEM;1239 goto fail;1240 }1241 1242 /* Configure next table pointer entries in BD memory */1243 for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {1244 next_addr = off + RING_DESC_SIZE;1245 if (next_addr == RING_BD_SIZE)1246 next_addr = 0;1247 next_addr += ring->bd_dma_base;1248 if (RING_BD_ALIGN_CHECK(next_addr))1249 d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),1250 next_addr);1251 else1252 d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));1253 flexrm_write_desc(ring->bd_base + off, d);1254 }1255 1256 /* Allocate completion memory */1257 ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,1258 GFP_KERNEL, &ring->cmpl_dma_base);1259 if (!ring->cmpl_base) {1260 dev_err(ring->mbox->dev,1261 "can't allocate completion memory for ring%d\n",1262 ring->num);1263 ret = -ENOMEM;1264 goto fail_free_bd_memory;1265 }1266 1267 /* Request IRQ */1268 if (ring->irq == UINT_MAX) {1269 dev_err(ring->mbox->dev,1270 "ring%d IRQ not available\n", ring->num);1271 ret = -ENODEV;1272 goto fail_free_cmpl_memory;1273 }1274 ret = request_threaded_irq(ring->irq,1275 flexrm_irq_event,1276 flexrm_irq_thread,1277 0, dev_name(ring->mbox->dev), ring);1278 if (ret) {1279 dev_err(ring->mbox->dev,1280 "failed to request ring%d IRQ\n", ring->num);1281 goto fail_free_cmpl_memory;1282 }1283 ring->irq_requested = true;1284 1285 /* Set IRQ affinity hint */1286 ring->irq_aff_hint = CPU_MASK_NONE;1287 val = ring->mbox->num_rings;1288 val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;1289 cpumask_set_cpu((ring->num / val) % num_online_cpus(),1290 &ring->irq_aff_hint);1291 ret = irq_update_affinity_hint(ring->irq, &ring->irq_aff_hint);1292 if (ret) {1293 dev_err(ring->mbox->dev,1294 "failed to set IRQ affinity hint for ring%d\n",1295 ring->num);1296 goto fail_free_irq;1297 }1298 1299 /* Disable/inactivate ring */1300 writel_relaxed(0x0, ring->regs + RING_CONTROL);1301 1302 /* Program BD start address */1303 val = BD_START_ADDR_VALUE(ring->bd_dma_base);1304 writel_relaxed(val, ring->regs + RING_BD_START_ADDR);1305 1306 /* BD write pointer will be same as HW write pointer */1307 ring->bd_write_offset =1308 readl_relaxed(ring->regs + RING_BD_WRITE_PTR);1309 ring->bd_write_offset *= RING_DESC_SIZE;1310 1311 /* Program completion start address */1312 val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);1313 writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);1314 1315 /* Completion read pointer will be same as HW write pointer */1316 ring->cmpl_read_offset =1317 readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);1318 ring->cmpl_read_offset *= RING_DESC_SIZE;1319 1320 /* Read ring Tx, Rx, and Outstanding counts to clear */1321 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);1322 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);1323 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);1324 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);1325 readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);1326 1327 /* Configure RING_MSI_CONTROL */1328 val = 0;1329 val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);1330 val |= BIT(MSI_ENABLE_SHIFT);1331 val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;1332 writel_relaxed(val, ring->regs + RING_MSI_CONTROL);1333 1334 /* Enable/activate ring */1335 val = BIT(CONTROL_ACTIVE_SHIFT);1336 writel_relaxed(val, ring->regs + RING_CONTROL);1337 1338 /* Reset stats to zero */1339 atomic_set(&ring->msg_send_count, 0);1340 atomic_set(&ring->msg_cmpl_count, 0);1341 1342 return 0;1343 1344fail_free_irq:1345 free_irq(ring->irq, ring);1346 ring->irq_requested = false;1347fail_free_cmpl_memory:1348 dma_pool_free(ring->mbox->cmpl_pool,1349 ring->cmpl_base, ring->cmpl_dma_base);1350 ring->cmpl_base = NULL;1351fail_free_bd_memory:1352 dma_pool_free(ring->mbox->bd_pool,1353 ring->bd_base, ring->bd_dma_base);1354 ring->bd_base = NULL;1355fail:1356 return ret;1357}1358 1359static void flexrm_shutdown(struct mbox_chan *chan)1360{1361 u32 reqid;1362 unsigned int timeout;1363 struct brcm_message *msg;1364 struct flexrm_ring *ring = chan->con_priv;1365 1366 /* Disable/inactivate ring */1367 writel_relaxed(0x0, ring->regs + RING_CONTROL);1368 1369 /* Set ring flush state */1370 timeout = 1000; /* timeout of 1s */1371 writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),1372 ring->regs + RING_CONTROL);1373 do {1374 if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &1375 FLUSH_DONE_MASK)1376 break;1377 mdelay(1);1378 } while (--timeout);1379 if (!timeout)1380 dev_err(ring->mbox->dev,1381 "setting ring%d flush state timedout\n", ring->num);1382 1383 /* Clear ring flush state */1384 timeout = 1000; /* timeout of 1s */1385 writel_relaxed(0x0, ring->regs + RING_CONTROL);1386 do {1387 if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &1388 FLUSH_DONE_MASK))1389 break;1390 mdelay(1);1391 } while (--timeout);1392 if (!timeout)1393 dev_err(ring->mbox->dev,1394 "clearing ring%d flush state timedout\n", ring->num);1395 1396 /* Abort all in-flight requests */1397 for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {1398 msg = ring->requests[reqid];1399 if (!msg)1400 continue;1401 1402 /* Release reqid for recycling */1403 ring->requests[reqid] = NULL;1404 1405 /* Unmap DMA mappings */1406 flexrm_dma_unmap(ring->mbox->dev, msg);1407 1408 /* Give-back message to mailbox client */1409 msg->error = -EIO;1410 mbox_chan_received_data(chan, msg);1411 }1412 1413 /* Clear requests bitmap */1414 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);1415 1416 /* Release IRQ */1417 if (ring->irq_requested) {1418 irq_update_affinity_hint(ring->irq, NULL);1419 free_irq(ring->irq, ring);1420 ring->irq_requested = false;1421 }1422 1423 /* Free-up completion descriptor ring */1424 if (ring->cmpl_base) {1425 dma_pool_free(ring->mbox->cmpl_pool,1426 ring->cmpl_base, ring->cmpl_dma_base);1427 ring->cmpl_base = NULL;1428 }1429 1430 /* Free-up BD descriptor ring */1431 if (ring->bd_base) {1432 dma_pool_free(ring->mbox->bd_pool,1433 ring->bd_base, ring->bd_dma_base);1434 ring->bd_base = NULL;1435 }1436}1437 1438static const struct mbox_chan_ops flexrm_mbox_chan_ops = {1439 .send_data = flexrm_send_data,1440 .startup = flexrm_startup,1441 .shutdown = flexrm_shutdown,1442 .peek_data = flexrm_peek_data,1443};1444 1445static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,1446 const struct of_phandle_args *pa)1447{1448 struct mbox_chan *chan;1449 struct flexrm_ring *ring;1450 1451 if (pa->args_count < 3)1452 return ERR_PTR(-EINVAL);1453 1454 if (pa->args[0] >= cntlr->num_chans)1455 return ERR_PTR(-ENOENT);1456 1457 if (pa->args[1] > MSI_COUNT_MASK)1458 return ERR_PTR(-EINVAL);1459 1460 if (pa->args[2] > MSI_TIMER_VAL_MASK)1461 return ERR_PTR(-EINVAL);1462 1463 chan = &cntlr->chans[pa->args[0]];1464 ring = chan->con_priv;1465 ring->msi_count_threshold = pa->args[1];1466 ring->msi_timer_val = pa->args[2];1467 1468 return chan;1469}1470 1471/* ====== FlexRM platform driver ===== */1472 1473static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)1474{1475 struct device *dev = msi_desc_to_dev(desc);1476 struct flexrm_mbox *mbox = dev_get_drvdata(dev);1477 struct flexrm_ring *ring = &mbox->rings[desc->msi_index];1478 1479 /* Configure per-Ring MSI registers */1480 writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);1481 writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);1482 writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);1483}1484 1485static int flexrm_mbox_probe(struct platform_device *pdev)1486{1487 int index, ret = 0;1488 void __iomem *regs;1489 void __iomem *regs_end;1490 struct resource *iomem;1491 struct flexrm_ring *ring;1492 struct flexrm_mbox *mbox;1493 struct device *dev = &pdev->dev;1494 1495 /* Allocate driver mailbox struct */1496 mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);1497 if (!mbox) {1498 ret = -ENOMEM;1499 goto fail;1500 }1501 mbox->dev = dev;1502 platform_set_drvdata(pdev, mbox);1503 1504 /* Get resource for registers and map registers of all rings */1505 mbox->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &iomem);1506 if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {1507 ret = -ENODEV;1508 goto fail;1509 } else if (IS_ERR(mbox->regs)) {1510 ret = PTR_ERR(mbox->regs);1511 goto fail;1512 }1513 regs_end = mbox->regs + resource_size(iomem);1514 1515 /* Scan and count available rings */1516 mbox->num_rings = 0;1517 for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {1518 if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)1519 mbox->num_rings++;1520 }1521 if (!mbox->num_rings) {1522 ret = -ENODEV;1523 goto fail;1524 }1525 1526 /* Allocate driver ring structs */1527 ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);1528 if (!ring) {1529 ret = -ENOMEM;1530 goto fail;1531 }1532 mbox->rings = ring;1533 1534 /* Initialize members of driver ring structs */1535 regs = mbox->regs;1536 for (index = 0; index < mbox->num_rings; index++) {1537 ring = &mbox->rings[index];1538 ring->num = index;1539 ring->mbox = mbox;1540 while ((regs < regs_end) &&1541 (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))1542 regs += RING_REGS_SIZE;1543 if (regs_end <= regs) {1544 ret = -ENODEV;1545 goto fail;1546 }1547 ring->regs = regs;1548 regs += RING_REGS_SIZE;1549 ring->irq = UINT_MAX;1550 ring->irq_requested = false;1551 ring->msi_timer_val = MSI_TIMER_VAL_MASK;1552 ring->msi_count_threshold = 0x1;1553 memset(ring->requests, 0, sizeof(ring->requests));1554 ring->bd_base = NULL;1555 ring->bd_dma_base = 0;1556 ring->cmpl_base = NULL;1557 ring->cmpl_dma_base = 0;1558 atomic_set(&ring->msg_send_count, 0);1559 atomic_set(&ring->msg_cmpl_count, 0);1560 spin_lock_init(&ring->lock);1561 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);1562 ring->cmpl_read_offset = 0;1563 }1564 1565 /* FlexRM is capable of 40-bit physical addresses only */1566 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));1567 if (ret) {1568 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));1569 if (ret)1570 goto fail;1571 }1572 1573 /* Create DMA pool for ring BD memory */1574 mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,1575 1 << RING_BD_ALIGN_ORDER, 0);1576 if (!mbox->bd_pool) {1577 ret = -ENOMEM;1578 goto fail;1579 }1580 1581 /* Create DMA pool for ring completion memory */1582 mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,1583 1 << RING_CMPL_ALIGN_ORDER, 0);1584 if (!mbox->cmpl_pool) {1585 ret = -ENOMEM;1586 goto fail_destroy_bd_pool;1587 }1588 1589 /* Allocate platform MSIs for each ring */1590 ret = platform_device_msi_init_and_alloc_irqs(dev, mbox->num_rings,1591 flexrm_mbox_msi_write);1592 if (ret)1593 goto fail_destroy_cmpl_pool;1594 1595 /* Save alloced IRQ numbers for each ring */1596 for (index = 0; index < mbox->num_rings; index++)1597 mbox->rings[index].irq = msi_get_virq(dev, index);1598 1599 /* Check availability of debugfs */1600 if (!debugfs_initialized())1601 goto skip_debugfs;1602 1603 /* Create debugfs root entry */1604 mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);1605 1606 /* Create debugfs config entry */1607 debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root,1608 flexrm_debugfs_conf_show);1609 1610 /* Create debugfs stats entry */1611 debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root,1612 flexrm_debugfs_stats_show);1613 1614skip_debugfs:1615 1616 /* Initialize mailbox controller */1617 mbox->controller.txdone_irq = false;1618 mbox->controller.txdone_poll = false;1619 mbox->controller.ops = &flexrm_mbox_chan_ops;1620 mbox->controller.dev = dev;1621 mbox->controller.num_chans = mbox->num_rings;1622 mbox->controller.of_xlate = flexrm_mbox_of_xlate;1623 mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,1624 sizeof(*mbox->controller.chans), GFP_KERNEL);1625 if (!mbox->controller.chans) {1626 ret = -ENOMEM;1627 goto fail_free_debugfs_root;1628 }1629 for (index = 0; index < mbox->num_rings; index++)1630 mbox->controller.chans[index].con_priv = &mbox->rings[index];1631 1632 /* Register mailbox controller */1633 ret = devm_mbox_controller_register(dev, &mbox->controller);1634 if (ret)1635 goto fail_free_debugfs_root;1636 1637 dev_info(dev, "registered flexrm mailbox with %d channels\n",1638 mbox->controller.num_chans);1639 1640 return 0;1641 1642fail_free_debugfs_root:1643 debugfs_remove_recursive(mbox->root);1644 platform_device_msi_free_irqs_all(dev);1645fail_destroy_cmpl_pool:1646 dma_pool_destroy(mbox->cmpl_pool);1647fail_destroy_bd_pool:1648 dma_pool_destroy(mbox->bd_pool);1649fail:1650 return ret;1651}1652 1653static void flexrm_mbox_remove(struct platform_device *pdev)1654{1655 struct device *dev = &pdev->dev;1656 struct flexrm_mbox *mbox = platform_get_drvdata(pdev);1657 1658 debugfs_remove_recursive(mbox->root);1659 1660 platform_device_msi_free_irqs_all(dev);1661 1662 dma_pool_destroy(mbox->cmpl_pool);1663 dma_pool_destroy(mbox->bd_pool);1664}1665 1666static const struct of_device_id flexrm_mbox_of_match[] = {1667 { .compatible = "brcm,iproc-flexrm-mbox", },1668 {},1669};1670MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);1671 1672static struct platform_driver flexrm_mbox_driver = {1673 .driver = {1674 .name = "brcm-flexrm-mbox",1675 .of_match_table = flexrm_mbox_of_match,1676 },1677 .probe = flexrm_mbox_probe,1678 .remove_new = flexrm_mbox_remove,1679};1680module_platform_driver(flexrm_mbox_driver);1681 1682MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");1683MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");1684MODULE_LICENSE("GPL v2");1685