1798 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * PiSP Back End driver.4 * Copyright (c) 2021-2024 Raspberry Pi Limited.5 *6 */7#include <linux/clk.h>8#include <linux/interrupt.h>9#include <linux/io.h>10#include <linux/kernel.h>11#include <linux/lockdep.h>12#include <linux/module.h>13#include <linux/platform_device.h>14#include <linux/pm_runtime.h>15#include <media/v4l2-device.h>16#include <media/v4l2-ioctl.h>17#include <media/videobuf2-dma-contig.h>18#include <media/videobuf2-vmalloc.h>19 20#include <uapi/linux/media/raspberrypi/pisp_be_config.h>21 22#include "pisp_be_formats.h"23 24/* Maximum number of config buffers possible */25#define PISP_BE_NUM_CONFIG_BUFFERS VB2_MAX_FRAME26 27#define PISPBE_NAME "pispbe"28 29/* Some ISP-BE registers */30#define PISP_BE_VERSION_REG 0x031#define PISP_BE_CONTROL_REG 0x432#define PISP_BE_CONTROL_COPY_CONFIG BIT(1)33#define PISP_BE_CONTROL_QUEUE_JOB BIT(0)34#define PISP_BE_CONTROL_NUM_TILES(n) ((n) << 16)35#define PISP_BE_TILE_ADDR_LO_REG 0x836#define PISP_BE_TILE_ADDR_HI_REG 0xc37#define PISP_BE_STATUS_REG 0x1038#define PISP_BE_STATUS_QUEUED BIT(0)39#define PISP_BE_BATCH_STATUS_REG 0x1440#define PISP_BE_INTERRUPT_EN_REG 0x1841#define PISP_BE_INTERRUPT_STATUS_REG 0x1c42#define PISP_BE_AXI_REG 0x2043#define PISP_BE_CONFIG_BASE_REG 0x4044#define PISP_BE_IO_ADDR_LOW(n) (PISP_BE_CONFIG_BASE_REG + 8 * (n))45#define PISP_BE_IO_ADDR_HIGH(n) (PISP_BE_IO_ADDR_LOW((n)) + 4)46#define PISP_BE_GLOBAL_BAYER_ENABLE 0xb047#define PISP_BE_GLOBAL_RGB_ENABLE 0xb448#define N_HW_ADDRESSES 1349#define N_HW_ENABLES 250 51#define PISP_BE_VERSION_2712 0x0225270052#define PISP_BE_VERSION_MINOR_BITS 0xf53 54/*55 * This maps our nodes onto the inputs/outputs of the actual PiSP Back End.56 * Be wary of the word "OUTPUT" which is used ambiguously here. In a V4L257 * context it means an input to the hardware (source image or metadata).58 * Elsewhere it means an output from the hardware.59 */60enum pispbe_node_ids {61 MAIN_INPUT_NODE,62 TDN_INPUT_NODE,63 STITCH_INPUT_NODE,64 OUTPUT0_NODE,65 OUTPUT1_NODE,66 TDN_OUTPUT_NODE,67 STITCH_OUTPUT_NODE,68 CONFIG_NODE,69 PISPBE_NUM_NODES70};71 72struct pispbe_node_description {73 const char *ent_name;74 enum v4l2_buf_type buf_type;75 unsigned int caps;76};77 78static const struct pispbe_node_description node_desc[PISPBE_NUM_NODES] = {79 /* MAIN_INPUT_NODE */80 {81 .ent_name = PISPBE_NAME "-input",82 .buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,83 .caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,84 },85 /* TDN_INPUT_NODE */86 {87 .ent_name = PISPBE_NAME "-tdn_input",88 .buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,89 .caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,90 },91 /* STITCH_INPUT_NODE */92 {93 .ent_name = PISPBE_NAME "-stitch_input",94 .buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,95 .caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,96 },97 /* OUTPUT0_NODE */98 {99 .ent_name = PISPBE_NAME "-output0",100 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,101 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,102 },103 /* OUTPUT1_NODE */104 {105 .ent_name = PISPBE_NAME "-output1",106 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,107 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,108 },109 /* TDN_OUTPUT_NODE */110 {111 .ent_name = PISPBE_NAME "-tdn_output",112 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,113 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,114 },115 /* STITCH_OUTPUT_NODE */116 {117 .ent_name = PISPBE_NAME "-stitch_output",118 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,119 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,120 },121 /* CONFIG_NODE */122 {123 .ent_name = PISPBE_NAME "-config",124 .buf_type = V4L2_BUF_TYPE_META_OUTPUT,125 .caps = V4L2_CAP_META_OUTPUT,126 }127};128 129#define NODE_DESC_IS_OUTPUT(desc) ( \130 ((desc)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \131 ((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \132 ((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE))133 134#define NODE_IS_META(node) ( \135 ((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT))136#define NODE_IS_OUTPUT(node) ( \137 ((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \138 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \139 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE))140#define NODE_IS_CAPTURE(node) ( \141 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) || \142 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE))143#define NODE_IS_MPLANE(node) ( \144 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) || \145 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE))146 147/*148 * Structure to describe a single node /dev/video<N> which represents a single149 * input or output queue to the PiSP Back End device.150 */151struct pispbe_node {152 unsigned int id;153 int vfl_dir;154 enum v4l2_buf_type buf_type;155 struct video_device vfd;156 struct media_pad pad;157 struct media_intf_devnode *intf_devnode;158 struct media_link *intf_link;159 struct pispbe_dev *pispbe;160 /* Video device lock */161 struct mutex node_lock;162 /* vb2_queue lock */163 struct mutex queue_lock;164 /* Protect pispbe_node->ready_queue and pispbe_buffer->ready_list */165 spinlock_t ready_lock;166 struct list_head ready_queue;167 struct vb2_queue queue;168 struct v4l2_format format;169 const struct pisp_be_format *pisp_format;170};171 172/* For logging only, use the entity name with "pispbe" and separator removed */173#define NODE_NAME(node) \174 (node_desc[(node)->id].ent_name + sizeof(PISPBE_NAME))175 176/* Records details of the jobs currently running or queued on the h/w. */177struct pispbe_job {178 bool valid;179 /*180 * An array of buffer pointers - remember it's source buffers first,181 * then captures, then metadata last.182 */183 struct pispbe_buffer *buf[PISPBE_NUM_NODES];184};185 186struct pispbe_hw_enables {187 u32 bayer_enables;188 u32 rgb_enables;189};190 191/* Records a job configuration and memory addresses. */192struct pispbe_job_descriptor {193 dma_addr_t hw_dma_addrs[N_HW_ADDRESSES];194 struct pisp_be_tiles_config *config;195 struct pispbe_hw_enables hw_enables;196 dma_addr_t tiles;197};198 199/*200 * Structure representing the entire PiSP Back End device, comprising several201 * nodes which share platform resources and a mutex for the actual HW.202 */203struct pispbe_dev {204 struct device *dev;205 struct pispbe_dev *pispbe;206 struct pisp_be_tiles_config *config;207 void __iomem *be_reg_base;208 struct clk *clk;209 struct v4l2_device v4l2_dev;210 struct v4l2_subdev sd;211 struct media_device mdev;212 struct media_pad pad[PISPBE_NUM_NODES]; /* output pads first */213 struct pispbe_node node[PISPBE_NUM_NODES];214 dma_addr_t config_dma_addr;215 unsigned int sequence;216 u32 streaming_map;217 struct pispbe_job queued_job, running_job;218 spinlock_t hw_lock; /* protects "hw_busy" flag and streaming_map */219 bool hw_busy; /* non-zero if a job is queued or is being started */220 int irq;221 u32 hw_version;222 u8 done, started;223};224 225static u32 pispbe_rd(struct pispbe_dev *pispbe, unsigned int offset)226{227 return readl(pispbe->be_reg_base + offset);228}229 230static void pispbe_wr(struct pispbe_dev *pispbe, unsigned int offset, u32 val)231{232 writel(val, pispbe->be_reg_base + offset);233}234 235/*236 * Queue a job to the h/w. If the h/w is idle it will begin immediately.237 * Caller must ensure it is "safe to queue", i.e. we don't already have a238 * queued, unstarted job.239 */240static void pispbe_queue_job(struct pispbe_dev *pispbe,241 struct pispbe_job_descriptor *job)242{243 unsigned int begin, end;244 245 if (pispbe_rd(pispbe, PISP_BE_STATUS_REG) & PISP_BE_STATUS_QUEUED)246 dev_err(pispbe->dev, "ERROR: not safe to queue new job!\n");247 248 /*249 * Write configuration to hardware. DMA addresses and enable flags250 * are passed separately, because the driver needs to sanitize them,251 * and we don't want to modify (or be vulnerable to modifications of)252 * the mmap'd buffer.253 */254 for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) {255 pispbe_wr(pispbe, PISP_BE_IO_ADDR_LOW(u),256 lower_32_bits(job->hw_dma_addrs[u]));257 pispbe_wr(pispbe, PISP_BE_IO_ADDR_HIGH(u),258 upper_32_bits(job->hw_dma_addrs[u]));259 }260 pispbe_wr(pispbe, PISP_BE_GLOBAL_BAYER_ENABLE,261 job->hw_enables.bayer_enables);262 pispbe_wr(pispbe, PISP_BE_GLOBAL_RGB_ENABLE,263 job->hw_enables.rgb_enables);264 265 /* Everything else is as supplied by the user. */266 begin = offsetof(struct pisp_be_config, global.bayer_order) /267 sizeof(u32);268 end = sizeof(struct pisp_be_config) / sizeof(u32);269 for (unsigned int u = begin; u < end; u++)270 pispbe_wr(pispbe, PISP_BE_CONFIG_BASE_REG + sizeof(u32) * u,271 ((u32 *)job->config)[u]);272 273 /* Read back the addresses -- an error here could be fatal */274 for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) {275 unsigned int offset = PISP_BE_IO_ADDR_LOW(u);276 u64 along = pispbe_rd(pispbe, offset);277 278 along += ((u64)pispbe_rd(pispbe, offset + 4)) << 32;279 if (along != (u64)(job->hw_dma_addrs[u])) {280 dev_dbg(pispbe->dev,281 "ISP BE config error: check if ISP RAMs enabled?\n");282 return;283 }284 }285 286 /*287 * Write tile pointer to hardware. The IOMMU should prevent288 * out-of-bounds offsets reaching non-ISP buffers.289 */290 pispbe_wr(pispbe, PISP_BE_TILE_ADDR_LO_REG, lower_32_bits(job->tiles));291 pispbe_wr(pispbe, PISP_BE_TILE_ADDR_HI_REG, upper_32_bits(job->tiles));292 293 /* Enqueue the job */294 pispbe_wr(pispbe, PISP_BE_CONTROL_REG,295 PISP_BE_CONTROL_COPY_CONFIG | PISP_BE_CONTROL_QUEUE_JOB |296 PISP_BE_CONTROL_NUM_TILES(job->config->num_tiles));297}298 299struct pispbe_buffer {300 struct vb2_v4l2_buffer vb;301 struct list_head ready_list;302 unsigned int config_index;303};304 305static int pispbe_get_planes_addr(dma_addr_t addr[3], struct pispbe_buffer *buf,306 struct pispbe_node *node)307{308 unsigned int num_planes = node->format.fmt.pix_mp.num_planes;309 unsigned int plane_factor = 0;310 unsigned int size;311 unsigned int p;312 313 if (!buf || !node->pisp_format)314 return 0;315 316 /*317 * Determine the base plane size. This will not be the same318 * as node->format.fmt.pix_mp.plane_fmt[0].sizeimage for a single319 * plane buffer in an mplane format.320 */321 size = node->format.fmt.pix_mp.plane_fmt[0].bytesperline *322 node->format.fmt.pix_mp.height;323 324 for (p = 0; p < num_planes && p < PISPBE_MAX_PLANES; p++) {325 addr[p] = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, p);326 plane_factor += node->pisp_format->plane_factor[p];327 }328 329 for (; p < PISPBE_MAX_PLANES && node->pisp_format->plane_factor[p]; p++) {330 /*331 * Calculate the address offset of this plane as needed332 * by the hardware. This is specifically for non-mplane333 * buffer formats, where there are 3 image planes, e.g.334 * for the V4L2_PIX_FMT_YUV420 format.335 */336 addr[p] = addr[0] + ((size * plane_factor) >> 3);337 plane_factor += node->pisp_format->plane_factor[p];338 }339 340 return num_planes;341}342 343static dma_addr_t pispbe_get_addr(struct pispbe_buffer *buf)344{345 if (buf)346 return vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);347 348 return 0;349}350 351static void pispbe_xlate_addrs(struct pispbe_dev *pispbe,352 struct pispbe_job_descriptor *job,353 struct pispbe_buffer *buf[PISPBE_NUM_NODES])354{355 struct pispbe_hw_enables *hw_en = &job->hw_enables;356 struct pisp_be_tiles_config *config = job->config;357 dma_addr_t *addrs = job->hw_dma_addrs;358 int ret;359 360 /* Take a copy of the "enable" bitmaps so we can modify them. */361 hw_en->bayer_enables = config->config.global.bayer_enables;362 hw_en->rgb_enables = config->config.global.rgb_enables;363 364 /*365 * Main input first. There are 3 address pointers, corresponding to up366 * to 3 planes.367 */368 ret = pispbe_get_planes_addr(addrs, buf[MAIN_INPUT_NODE],369 &pispbe->node[MAIN_INPUT_NODE]);370 if (ret <= 0) {371 /*372 * This shouldn't happen; pispbe_schedule_internal should insist373 * on an input.374 */375 dev_warn(pispbe->dev, "ISP-BE missing input\n");376 hw_en->bayer_enables = 0;377 hw_en->rgb_enables = 0;378 return;379 }380 381 /*382 * Now TDN/Stitch inputs and outputs. These are single-plane and only383 * used with Bayer input. Input enables must match the requirements384 * of the processing stages, otherwise the hardware can lock up!385 */386 if (hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) {387 addrs[3] = pispbe_get_addr(buf[TDN_INPUT_NODE]);388 if (addrs[3] == 0 ||389 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN_INPUT) ||390 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN) ||391 (config->config.tdn.reset & 1)) {392 hw_en->bayer_enables &=393 ~(PISP_BE_BAYER_ENABLE_TDN_INPUT |394 PISP_BE_BAYER_ENABLE_TDN_DECOMPRESS);395 if (!(config->config.tdn.reset & 1))396 hw_en->bayer_enables &=397 ~PISP_BE_BAYER_ENABLE_TDN;398 }399 400 addrs[4] = pispbe_get_addr(buf[STITCH_INPUT_NODE]);401 if (addrs[4] == 0 ||402 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_INPUT) ||403 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH)) {404 hw_en->bayer_enables &=405 ~(PISP_BE_BAYER_ENABLE_STITCH_INPUT |406 PISP_BE_BAYER_ENABLE_STITCH_DECOMPRESS |407 PISP_BE_BAYER_ENABLE_STITCH);408 }409 410 addrs[5] = pispbe_get_addr(buf[TDN_OUTPUT_NODE]);411 if (addrs[5] == 0)412 hw_en->bayer_enables &=413 ~(PISP_BE_BAYER_ENABLE_TDN_COMPRESS |414 PISP_BE_BAYER_ENABLE_TDN_OUTPUT);415 416 addrs[6] = pispbe_get_addr(buf[STITCH_OUTPUT_NODE]);417 if (addrs[6] == 0)418 hw_en->bayer_enables &=419 ~(PISP_BE_BAYER_ENABLE_STITCH_COMPRESS |420 PISP_BE_BAYER_ENABLE_STITCH_OUTPUT);421 } else {422 /* No Bayer input? Disable entire Bayer pipe (else lockup) */423 hw_en->bayer_enables = 0;424 }425 426 /* Main image output channels. */427 for (unsigned int i = 0; i < PISP_BACK_END_NUM_OUTPUTS; i++) {428 ret = pispbe_get_planes_addr(addrs + 7 + 3 * i,429 buf[OUTPUT0_NODE + i],430 &pispbe->node[OUTPUT0_NODE + i]);431 if (ret <= 0)432 hw_en->rgb_enables &= ~(PISP_BE_RGB_ENABLE_OUTPUT0 << i);433 }434}435 436/*437 * Prepare a job description to be submitted to the HW.438 *439 * To schedule a job, we need all streaming nodes (apart from Output0,440 * Output1, Tdn and Stitch) to have a buffer ready, which must441 * include at least a config buffer and a main input image.442 *443 * For Output0, Output1, Tdn and Stitch, a buffer only needs to be444 * available if the blocks are enabled in the config.445 *446 * Needs to be called with hw_lock held.447 *448 * Returns 0 if a job has been successfully prepared, < 0 otherwise.449 */450static int pispbe_prepare_job(struct pispbe_dev *pispbe,451 struct pispbe_job_descriptor *job)452{453 struct pispbe_buffer *buf[PISPBE_NUM_NODES] = {};454 unsigned int config_index;455 struct pispbe_node *node;456 unsigned long flags;457 458 lockdep_assert_held(&pispbe->hw_lock);459 460 memset(job, 0, sizeof(struct pispbe_job_descriptor));461 462 if (((BIT(CONFIG_NODE) | BIT(MAIN_INPUT_NODE)) &463 pispbe->streaming_map) !=464 (BIT(CONFIG_NODE) | BIT(MAIN_INPUT_NODE)))465 return -ENODEV;466 467 node = &pispbe->node[CONFIG_NODE];468 spin_lock_irqsave(&node->ready_lock, flags);469 buf[CONFIG_NODE] = list_first_entry_or_null(&node->ready_queue,470 struct pispbe_buffer,471 ready_list);472 if (buf[CONFIG_NODE]) {473 list_del(&buf[CONFIG_NODE]->ready_list);474 pispbe->queued_job.buf[CONFIG_NODE] = buf[CONFIG_NODE];475 }476 spin_unlock_irqrestore(&node->ready_lock, flags);477 478 /* Exit early if no config buffer has been queued. */479 if (!buf[CONFIG_NODE])480 return -ENODEV;481 482 config_index = buf[CONFIG_NODE]->vb.vb2_buf.index;483 job->config = &pispbe->config[config_index];484 job->tiles = pispbe->config_dma_addr +485 config_index * sizeof(struct pisp_be_tiles_config) +486 offsetof(struct pisp_be_tiles_config, tiles);487 488 /* remember: srcimages, captures then metadata */489 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {490 unsigned int bayer_en =491 job->config->config.global.bayer_enables;492 unsigned int rgb_en =493 job->config->config.global.rgb_enables;494 bool ignore_buffers = false;495 496 /* Config node is handled outside the loop above. */497 if (i == CONFIG_NODE)498 continue;499 500 buf[i] = NULL;501 if (!(pispbe->streaming_map & BIT(i)))502 continue;503 504 if ((!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT0) &&505 i == OUTPUT0_NODE) ||506 (!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT1) &&507 i == OUTPUT1_NODE) ||508 (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_INPUT) &&509 i == TDN_INPUT_NODE) ||510 (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) &&511 i == TDN_OUTPUT_NODE) ||512 (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_INPUT) &&513 i == STITCH_INPUT_NODE) ||514 (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) &&515 i == STITCH_OUTPUT_NODE)) {516 /*517 * Ignore Output0/Output1/Tdn/Stitch buffer check if the518 * global enables aren't set for these blocks. If a519 * buffer has been provided, we dequeue it back to the520 * user with the other in-use buffers.521 */522 ignore_buffers = true;523 }524 525 node = &pispbe->node[i];526 527 /* Pull a buffer from each V4L2 queue to form the queued job */528 spin_lock_irqsave(&node->ready_lock, flags);529 buf[i] = list_first_entry_or_null(&node->ready_queue,530 struct pispbe_buffer,531 ready_list);532 if (buf[i]) {533 list_del(&buf[i]->ready_list);534 pispbe->queued_job.buf[i] = buf[i];535 }536 spin_unlock_irqrestore(&node->ready_lock, flags);537 538 if (!buf[i] && !ignore_buffers)539 goto err_return_buffers;540 }541 542 pispbe->queued_job.valid = true;543 544 /* Convert buffers to DMA addresses for the hardware */545 pispbe_xlate_addrs(pispbe, job, buf);546 547 return 0;548 549err_return_buffers:550 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {551 struct pispbe_node *n = &pispbe->node[i];552 553 if (!buf[i])554 continue;555 556 /* Return the buffer to the ready_list queue */557 spin_lock_irqsave(&n->ready_lock, flags);558 list_add(&buf[i]->ready_list, &n->ready_queue);559 spin_unlock_irqrestore(&n->ready_lock, flags);560 }561 562 memset(&pispbe->queued_job, 0, sizeof(pispbe->queued_job));563 564 return -ENODEV;565}566 567static void pispbe_schedule(struct pispbe_dev *pispbe, bool clear_hw_busy)568{569 struct pispbe_job_descriptor job;570 unsigned long flags;571 int ret;572 573 spin_lock_irqsave(&pispbe->hw_lock, flags);574 575 if (clear_hw_busy)576 pispbe->hw_busy = false;577 578 if (pispbe->hw_busy)579 goto unlock_and_return;580 581 ret = pispbe_prepare_job(pispbe, &job);582 if (ret)583 goto unlock_and_return;584 585 /*586 * We can kick the job off without the hw_lock, as this can587 * never run again until hw_busy is cleared, which will happen588 * only when the following job has been queued and an interrupt589 * is rised.590 */591 pispbe->hw_busy = true;592 spin_unlock_irqrestore(&pispbe->hw_lock, flags);593 594 if (job.config->num_tiles <= 0 ||595 job.config->num_tiles > PISP_BACK_END_NUM_TILES ||596 !((job.hw_enables.bayer_enables | job.hw_enables.rgb_enables) &597 PISP_BE_BAYER_ENABLE_INPUT)) {598 /*599 * Bad job. We can't let it proceed as it could lock up600 * the hardware, or worse!601 *602 * For now, just force num_tiles to 0, which causes the603 * H/W to do something bizarre but survivable. It604 * increments (started,done) counters by more than 1,605 * but we seem to survive...606 */607 dev_dbg(pispbe->dev, "Bad job: invalid number of tiles: %u\n",608 job.config->num_tiles);609 job.config->num_tiles = 0;610 }611 612 pispbe_queue_job(pispbe, &job);613 614 return;615 616unlock_and_return:617 /* No job has been queued, just release the lock and return. */618 spin_unlock_irqrestore(&pispbe->hw_lock, flags);619}620 621static void pispbe_isr_jobdone(struct pispbe_dev *pispbe,622 struct pispbe_job *job)623{624 struct pispbe_buffer **buf = job->buf;625 u64 ts = ktime_get_ns();626 627 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {628 if (buf[i]) {629 buf[i]->vb.vb2_buf.timestamp = ts;630 buf[i]->vb.sequence = pispbe->sequence;631 vb2_buffer_done(&buf[i]->vb.vb2_buf,632 VB2_BUF_STATE_DONE);633 }634 }635 636 pispbe->sequence++;637}638 639static irqreturn_t pispbe_isr(int irq, void *dev)640{641 struct pispbe_dev *pispbe = (struct pispbe_dev *)dev;642 bool can_queue_another = false;643 u8 started, done;644 u32 u;645 646 u = pispbe_rd(pispbe, PISP_BE_INTERRUPT_STATUS_REG);647 if (u == 0)648 return IRQ_NONE;649 650 pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, u);651 u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG);652 done = (uint8_t)u;653 started = (uint8_t)(u >> 8);654 655 /*656 * Be aware that done can go up by 2 and started by 1 when: a job that657 * we previously saw "start" now finishes, and we then queued a new job658 * which we see both start and finish "simultaneously".659 */660 if (pispbe->running_job.valid && pispbe->done != done) {661 pispbe_isr_jobdone(pispbe, &pispbe->running_job);662 memset(&pispbe->running_job, 0, sizeof(pispbe->running_job));663 pispbe->done++;664 }665 666 if (pispbe->started != started) {667 pispbe->started++;668 can_queue_another = 1;669 670 if (pispbe->done != done && pispbe->queued_job.valid) {671 pispbe_isr_jobdone(pispbe, &pispbe->queued_job);672 pispbe->done++;673 } else {674 pispbe->running_job = pispbe->queued_job;675 }676 677 memset(&pispbe->queued_job, 0, sizeof(pispbe->queued_job));678 }679 680 if (pispbe->done != done || pispbe->started != started) {681 dev_dbg(pispbe->dev,682 "Job counters not matching: done = %u, expected %u - started = %u, expected %u\n",683 pispbe->done, done, pispbe->started, started);684 pispbe->started = started;685 pispbe->done = done;686 }687 688 /* check if there's more to do before going to sleep */689 pispbe_schedule(pispbe, can_queue_another);690 691 return IRQ_HANDLED;692}693 694static int pisp_be_validate_config(struct pispbe_dev *pispbe,695 struct pisp_be_tiles_config *config)696{697 u32 bayer_enables = config->config.global.bayer_enables;698 u32 rgb_enables = config->config.global.rgb_enables;699 struct device *dev = pispbe->dev;700 struct v4l2_format *fmt;701 unsigned int bpl, size;702 703 if (!(bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) ==704 !(rgb_enables & PISP_BE_RGB_ENABLE_INPUT)) {705 dev_dbg(dev, "%s: Not one input enabled\n", __func__);706 return -EIO;707 }708 709 /* Ensure output config strides and buffer sizes match the V4L2 formats. */710 fmt = &pispbe->node[TDN_OUTPUT_NODE].format;711 if (bayer_enables & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) {712 bpl = config->config.tdn_output_format.stride;713 size = bpl * config->config.tdn_output_format.height;714 715 if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) {716 dev_dbg(dev, "%s: bpl mismatch on tdn_output\n",717 __func__);718 return -EINVAL;719 }720 721 if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) {722 dev_dbg(dev, "%s: size mismatch on tdn_output\n",723 __func__);724 return -EINVAL;725 }726 }727 728 fmt = &pispbe->node[STITCH_OUTPUT_NODE].format;729 if (bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) {730 bpl = config->config.stitch_output_format.stride;731 size = bpl * config->config.stitch_output_format.height;732 733 if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) {734 dev_dbg(dev, "%s: bpl mismatch on stitch_output\n",735 __func__);736 return -EINVAL;737 }738 739 if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) {740 dev_dbg(dev, "%s: size mismatch on stitch_output\n",741 __func__);742 return -EINVAL;743 }744 }745 746 for (unsigned int j = 0; j < PISP_BACK_END_NUM_OUTPUTS; j++) {747 if (!(rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT(j)))748 continue;749 750 if (config->config.output_format[j].image.format &751 PISP_IMAGE_FORMAT_WALLPAPER_ROLL)752 continue; /* TODO: Size checks for wallpaper formats */753 754 fmt = &pispbe->node[OUTPUT0_NODE + j].format;755 for (unsigned int i = 0; i < fmt->fmt.pix_mp.num_planes; i++) {756 bpl = !i ? config->config.output_format[j].image.stride757 : config->config.output_format[j].image.stride2;758 size = bpl * config->config.output_format[j].image.height;759 760 if (config->config.output_format[j].image.format &761 PISP_IMAGE_FORMAT_SAMPLING_420)762 size >>= 1;763 764 if (fmt->fmt.pix_mp.plane_fmt[i].bytesperline < bpl) {765 dev_dbg(dev, "%s: bpl mismatch on output %d\n",766 __func__, j);767 return -EINVAL;768 }769 770 if (fmt->fmt.pix_mp.plane_fmt[i].sizeimage < size) {771 dev_dbg(dev, "%s: size mismatch on output\n",772 __func__);773 return -EINVAL;774 }775 }776 }777 778 return 0;779}780 781static int pispbe_node_queue_setup(struct vb2_queue *q, unsigned int *nbuffers,782 unsigned int *nplanes, unsigned int sizes[],783 struct device *alloc_devs[])784{785 struct pispbe_node *node = vb2_get_drv_priv(q);786 struct pispbe_dev *pispbe = node->pispbe;787 unsigned int num_planes = NODE_IS_MPLANE(node) ?788 node->format.fmt.pix_mp.num_planes : 1;789 790 if (*nplanes) {791 if (*nplanes != num_planes)792 return -EINVAL;793 794 for (unsigned int i = 0; i < *nplanes; i++) {795 unsigned int size = NODE_IS_MPLANE(node) ?796 node->format.fmt.pix_mp.plane_fmt[i].sizeimage :797 node->format.fmt.meta.buffersize;798 799 if (sizes[i] < size)800 return -EINVAL;801 }802 803 return 0;804 }805 806 *nplanes = num_planes;807 for (unsigned int i = 0; i < *nplanes; i++) {808 unsigned int size = NODE_IS_MPLANE(node) ?809 node->format.fmt.pix_mp.plane_fmt[i].sizeimage :810 node->format.fmt.meta.buffersize;811 sizes[i] = size;812 }813 814 dev_dbg(pispbe->dev,815 "Image (or metadata) size %u, nbuffers %u for node %s\n",816 sizes[0], *nbuffers, NODE_NAME(node));817 818 return 0;819}820 821static int pispbe_node_buffer_prepare(struct vb2_buffer *vb)822{823 struct pispbe_node *node = vb2_get_drv_priv(vb->vb2_queue);824 struct pispbe_dev *pispbe = node->pispbe;825 unsigned int num_planes = NODE_IS_MPLANE(node) ?826 node->format.fmt.pix_mp.num_planes : 1;827 828 for (unsigned int i = 0; i < num_planes; i++) {829 unsigned long size = NODE_IS_MPLANE(node) ?830 node->format.fmt.pix_mp.plane_fmt[i].sizeimage :831 node->format.fmt.meta.buffersize;832 833 if (vb2_plane_size(vb, i) < size) {834 dev_dbg(pispbe->dev,835 "data will not fit into plane %d (%lu < %lu)\n",836 i, vb2_plane_size(vb, i), size);837 return -EINVAL;838 }839 840 vb2_set_plane_payload(vb, i, size);841 }842 843 if (node->id == CONFIG_NODE) {844 void *dst = &node->pispbe->config[vb->index];845 void *src = vb2_plane_vaddr(vb, 0);846 847 memcpy(dst, src, sizeof(struct pisp_be_tiles_config));848 849 return pisp_be_validate_config(pispbe, dst);850 }851 852 return 0;853}854 855static void pispbe_node_buffer_queue(struct vb2_buffer *buf)856{857 struct vb2_v4l2_buffer *vbuf =858 container_of(buf, struct vb2_v4l2_buffer, vb2_buf);859 struct pispbe_buffer *buffer =860 container_of(vbuf, struct pispbe_buffer, vb);861 struct pispbe_node *node = vb2_get_drv_priv(buf->vb2_queue);862 struct pispbe_dev *pispbe = node->pispbe;863 unsigned long flags;864 865 dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node));866 spin_lock_irqsave(&node->ready_lock, flags);867 list_add_tail(&buffer->ready_list, &node->ready_queue);868 spin_unlock_irqrestore(&node->ready_lock, flags);869 870 /*871 * Every time we add a buffer, check if there's now some work for the hw872 * to do.873 */874 pispbe_schedule(pispbe, false);875}876 877static int pispbe_node_start_streaming(struct vb2_queue *q, unsigned int count)878{879 struct pispbe_node *node = vb2_get_drv_priv(q);880 struct pispbe_dev *pispbe = node->pispbe;881 struct pispbe_buffer *buf, *tmp;882 unsigned long flags;883 int ret;884 885 ret = pm_runtime_resume_and_get(pispbe->dev);886 if (ret < 0)887 goto err_return_buffers;888 889 spin_lock_irqsave(&pispbe->hw_lock, flags);890 node->pispbe->streaming_map |= BIT(node->id);891 node->pispbe->sequence = 0;892 spin_unlock_irqrestore(&pispbe->hw_lock, flags);893 894 dev_dbg(pispbe->dev, "%s: for node %s (count %u)\n",895 __func__, NODE_NAME(node), count);896 dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n",897 node->pispbe->streaming_map);898 899 /* Maybe we're ready to run. */900 pispbe_schedule(pispbe, false);901 902 return 0;903 904err_return_buffers:905 spin_lock_irqsave(&pispbe->hw_lock, flags);906 list_for_each_entry_safe(buf, tmp, &node->ready_queue, ready_list) {907 list_del(&buf->ready_list);908 vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);909 }910 spin_unlock_irqrestore(&pispbe->hw_lock, flags);911 912 return ret;913}914 915static void pispbe_node_stop_streaming(struct vb2_queue *q)916{917 struct pispbe_node *node = vb2_get_drv_priv(q);918 struct pispbe_dev *pispbe = node->pispbe;919 struct pispbe_buffer *buf;920 unsigned long flags;921 922 /*923 * Now this is a bit awkward. In a simple M2M device we could just wait924 * for all queued jobs to complete, but here there's a risk that a925 * partial set of buffers was queued and cannot be run. For now, just926 * cancel all buffers stuck in the "ready queue", then wait for any927 * running job.928 *929 * This may return buffers out of order.930 */931 dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node));932 spin_lock_irqsave(&pispbe->hw_lock, flags);933 do {934 unsigned long flags1;935 936 spin_lock_irqsave(&node->ready_lock, flags1);937 buf = list_first_entry_or_null(&node->ready_queue,938 struct pispbe_buffer,939 ready_list);940 if (buf) {941 list_del(&buf->ready_list);942 vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);943 }944 spin_unlock_irqrestore(&node->ready_lock, flags1);945 } while (buf);946 spin_unlock_irqrestore(&pispbe->hw_lock, flags);947 948 vb2_wait_for_all_buffers(&node->queue);949 950 spin_lock_irqsave(&pispbe->hw_lock, flags);951 pispbe->streaming_map &= ~BIT(node->id);952 spin_unlock_irqrestore(&pispbe->hw_lock, flags);953 954 pm_runtime_mark_last_busy(pispbe->dev);955 pm_runtime_put_autosuspend(pispbe->dev);956 957 dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n",958 pispbe->streaming_map);959}960 961static const struct vb2_ops pispbe_node_queue_ops = {962 .queue_setup = pispbe_node_queue_setup,963 .buf_prepare = pispbe_node_buffer_prepare,964 .buf_queue = pispbe_node_buffer_queue,965 .start_streaming = pispbe_node_start_streaming,966 .stop_streaming = pispbe_node_stop_streaming,967};968 969static const struct v4l2_file_operations pispbe_fops = {970 .owner = THIS_MODULE,971 .open = v4l2_fh_open,972 .release = vb2_fop_release,973 .poll = vb2_fop_poll,974 .unlocked_ioctl = video_ioctl2,975 .mmap = vb2_fop_mmap976};977 978static int pispbe_node_querycap(struct file *file, void *priv,979 struct v4l2_capability *cap)980{981 struct pispbe_node *node = video_drvdata(file);982 struct pispbe_dev *pispbe = node->pispbe;983 984 strscpy(cap->driver, PISPBE_NAME, sizeof(cap->driver));985 strscpy(cap->card, PISPBE_NAME, sizeof(cap->card));986 987 dev_dbg(pispbe->dev, "Caps for node %s: %x and %x (dev %x)\n",988 NODE_NAME(node), cap->capabilities, cap->device_caps,989 node->vfd.device_caps);990 991 return 0;992}993 994static int pispbe_node_g_fmt_vid_cap(struct file *file, void *priv,995 struct v4l2_format *f)996{997 struct pispbe_node *node = video_drvdata(file);998 struct pispbe_dev *pispbe = node->pispbe;999 1000 if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {1001 dev_dbg(pispbe->dev,1002 "Cannot get capture fmt for output node %s\n",1003 NODE_NAME(node));1004 return -EINVAL;1005 }1006 1007 *f = node->format;1008 dev_dbg(pispbe->dev, "Get capture format for node %s\n",1009 NODE_NAME(node));1010 1011 return 0;1012}1013 1014static int pispbe_node_g_fmt_vid_out(struct file *file, void *priv,1015 struct v4l2_format *f)1016{1017 struct pispbe_node *node = video_drvdata(file);1018 struct pispbe_dev *pispbe = node->pispbe;1019 1020 if (NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {1021 dev_dbg(pispbe->dev,1022 "Cannot get capture fmt for output node %s\n",1023 NODE_NAME(node));1024 return -EINVAL;1025 }1026 1027 *f = node->format;1028 dev_dbg(pispbe->dev, "Get output format for node %s\n",1029 NODE_NAME(node));1030 1031 return 0;1032}1033 1034static int pispbe_node_g_fmt_meta_out(struct file *file, void *priv,1035 struct v4l2_format *f)1036{1037 struct pispbe_node *node = video_drvdata(file);1038 struct pispbe_dev *pispbe = node->pispbe;1039 1040 if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) {1041 dev_dbg(pispbe->dev,1042 "Cannot get capture fmt for meta output node %s\n",1043 NODE_NAME(node));1044 return -EINVAL;1045 }1046 1047 *f = node->format;1048 dev_dbg(pispbe->dev, "Get output format for meta node %s\n",1049 NODE_NAME(node));1050 1051 return 0;1052}1053 1054static const struct pisp_be_format *pispbe_find_fmt(unsigned int fourcc)1055{1056 for (unsigned int i = 0; i < ARRAY_SIZE(supported_formats); i++) {1057 if (supported_formats[i].fourcc == fourcc)1058 return &supported_formats[i];1059 }1060 1061 return NULL;1062}1063 1064static void pispbe_set_plane_params(struct v4l2_format *f,1065 const struct pisp_be_format *fmt)1066{1067 unsigned int nplanes = f->fmt.pix_mp.num_planes;1068 unsigned int total_plane_factor = 0;1069 1070 for (unsigned int i = 0; i < PISPBE_MAX_PLANES; i++)1071 total_plane_factor += fmt->plane_factor[i];1072 1073 for (unsigned int i = 0; i < nplanes; i++) {1074 struct v4l2_plane_pix_format *p = &f->fmt.pix_mp.plane_fmt[i];1075 unsigned int bpl, plane_size;1076 1077 bpl = (f->fmt.pix_mp.width * fmt->bit_depth) >> 3;1078 bpl = ALIGN(max(p->bytesperline, bpl), fmt->align);1079 1080 plane_size = bpl * f->fmt.pix_mp.height *1081 (nplanes > 1 ? fmt->plane_factor[i] : total_plane_factor);1082 /*1083 * The shift is to divide out the plane_factor fixed point1084 * scaling of 8.1085 */1086 plane_size = max(p->sizeimage, plane_size >> 3);1087 1088 p->bytesperline = bpl;1089 p->sizeimage = plane_size;1090 }1091}1092 1093static void pispbe_try_format(struct v4l2_format *f, struct pispbe_node *node)1094{1095 struct pispbe_dev *pispbe = node->pispbe;1096 u32 pixfmt = f->fmt.pix_mp.pixelformat;1097 const struct pisp_be_format *fmt;1098 bool is_rgb;1099 1100 dev_dbg(pispbe->dev,1101 "%s: [%s] req %ux%u %p4cc, planes %d\n",1102 __func__, NODE_NAME(node), f->fmt.pix_mp.width,1103 f->fmt.pix_mp.height, &pixfmt,1104 f->fmt.pix_mp.num_planes);1105 1106 fmt = pispbe_find_fmt(pixfmt);1107 if (!fmt) {1108 dev_dbg(pispbe->dev,1109 "%s: [%s] Format not found, defaulting to YUV420\n",1110 __func__, NODE_NAME(node));1111 fmt = pispbe_find_fmt(V4L2_PIX_FMT_YUV420);1112 }1113 1114 f->fmt.pix_mp.pixelformat = fmt->fourcc;1115 f->fmt.pix_mp.num_planes = fmt->num_planes;1116 f->fmt.pix_mp.field = V4L2_FIELD_NONE;1117 f->fmt.pix_mp.width = max(min(f->fmt.pix_mp.width, 65536u),1118 PISP_BACK_END_MIN_TILE_WIDTH);1119 f->fmt.pix_mp.height = max(min(f->fmt.pix_mp.height, 65536u),1120 PISP_BACK_END_MIN_TILE_HEIGHT);1121 1122 /*1123 * Fill in the actual colour space when the requested one was1124 * not supported. This also catches the case when the "default"1125 * colour space was requested (as that's never in the mask).1126 */1127 if (!(V4L2_COLORSPACE_MASK(f->fmt.pix_mp.colorspace) &1128 fmt->colorspace_mask))1129 f->fmt.pix_mp.colorspace = fmt->colorspace_default;1130 1131 /* In all cases, we only support the defaults for these: */1132 f->fmt.pix_mp.ycbcr_enc =1133 V4L2_MAP_YCBCR_ENC_DEFAULT(f->fmt.pix_mp.colorspace);1134 f->fmt.pix_mp.xfer_func =1135 V4L2_MAP_XFER_FUNC_DEFAULT(f->fmt.pix_mp.colorspace);1136 1137 is_rgb = f->fmt.pix_mp.colorspace == V4L2_COLORSPACE_SRGB;1138 f->fmt.pix_mp.quantization =1139 V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb, f->fmt.pix_mp.colorspace,1140 f->fmt.pix_mp.ycbcr_enc);1141 1142 /* Set plane size and bytes/line for each plane. */1143 pispbe_set_plane_params(f, fmt);1144 1145 for (unsigned int i = 0; i < f->fmt.pix_mp.num_planes; i++) {1146 dev_dbg(pispbe->dev,1147 "%s: [%s] calc plane %d, %ux%u, depth %u, bpl %u size %u\n",1148 __func__, NODE_NAME(node), i, f->fmt.pix_mp.width,1149 f->fmt.pix_mp.height, fmt->bit_depth,1150 f->fmt.pix_mp.plane_fmt[i].bytesperline,1151 f->fmt.pix_mp.plane_fmt[i].sizeimage);1152 }1153}1154 1155static int pispbe_node_try_fmt_vid_cap(struct file *file, void *priv,1156 struct v4l2_format *f)1157{1158 struct pispbe_node *node = video_drvdata(file);1159 struct pispbe_dev *pispbe = node->pispbe;1160 1161 if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {1162 dev_dbg(pispbe->dev,1163 "Cannot set capture fmt for output node %s\n",1164 NODE_NAME(node));1165 return -EINVAL;1166 }1167 1168 pispbe_try_format(f, node);1169 1170 return 0;1171}1172 1173static int pispbe_node_try_fmt_vid_out(struct file *file, void *priv,1174 struct v4l2_format *f)1175{1176 struct pispbe_node *node = video_drvdata(file);1177 struct pispbe_dev *pispbe = node->pispbe;1178 1179 if (!NODE_IS_OUTPUT(node) || NODE_IS_META(node)) {1180 dev_dbg(pispbe->dev,1181 "Cannot set capture fmt for output node %s\n",1182 NODE_NAME(node));1183 return -EINVAL;1184 }1185 1186 pispbe_try_format(f, node);1187 1188 return 0;1189}1190 1191static int pispbe_node_try_fmt_meta_out(struct file *file, void *priv,1192 struct v4l2_format *f)1193{1194 struct pispbe_node *node = video_drvdata(file);1195 struct pispbe_dev *pispbe = node->pispbe;1196 1197 if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) {1198 dev_dbg(pispbe->dev,1199 "Cannot set capture fmt for meta output node %s\n",1200 NODE_NAME(node));1201 return -EINVAL;1202 }1203 1204 f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG;1205 f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config);1206 1207 return 0;1208}1209 1210static int pispbe_node_s_fmt_vid_cap(struct file *file, void *priv,1211 struct v4l2_format *f)1212{1213 struct pispbe_node *node = video_drvdata(file);1214 struct pispbe_dev *pispbe = node->pispbe;1215 int ret;1216 1217 ret = pispbe_node_try_fmt_vid_cap(file, priv, f);1218 if (ret < 0)1219 return ret;1220 1221 if (vb2_is_busy(&node->queue))1222 return -EBUSY;1223 1224 node->format = *f;1225 node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat);1226 1227 dev_dbg(pispbe->dev, "Set capture format for node %s to %p4cc\n",1228 NODE_NAME(node), &f->fmt.pix_mp.pixelformat);1229 1230 return 0;1231}1232 1233static int pispbe_node_s_fmt_vid_out(struct file *file, void *priv,1234 struct v4l2_format *f)1235{1236 struct pispbe_node *node = video_drvdata(file);1237 struct pispbe_dev *pispbe = node->pispbe;1238 int ret;1239 1240 ret = pispbe_node_try_fmt_vid_out(file, priv, f);1241 if (ret < 0)1242 return ret;1243 1244 if (vb2_is_busy(&node->queue))1245 return -EBUSY;1246 1247 node->format = *f;1248 node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat);1249 1250 dev_dbg(pispbe->dev, "Set output format for node %s to %p4cc\n",1251 NODE_NAME(node), &f->fmt.pix_mp.pixelformat);1252 1253 return 0;1254}1255 1256static int pispbe_node_s_fmt_meta_out(struct file *file, void *priv,1257 struct v4l2_format *f)1258{1259 struct pispbe_node *node = video_drvdata(file);1260 struct pispbe_dev *pispbe = node->pispbe;1261 int ret;1262 1263 ret = pispbe_node_try_fmt_meta_out(file, priv, f);1264 if (ret < 0)1265 return ret;1266 1267 if (vb2_is_busy(&node->queue))1268 return -EBUSY;1269 1270 node->format = *f;1271 node->pisp_format = &meta_out_supported_formats[0];1272 1273 dev_dbg(pispbe->dev, "Set output format for meta node %s to %p4cc\n",1274 NODE_NAME(node), &f->fmt.meta.dataformat);1275 1276 return 0;1277}1278 1279static int pispbe_node_enum_fmt(struct file *file, void *priv,1280 struct v4l2_fmtdesc *f)1281{1282 struct pispbe_node *node = video_drvdata(file);1283 1284 if (f->type != node->queue.type)1285 return -EINVAL;1286 1287 if (NODE_IS_META(node)) {1288 if (f->index)1289 return -EINVAL;1290 1291 f->pixelformat = V4L2_META_FMT_RPI_BE_CFG;1292 f->flags = 0;1293 return 0;1294 }1295 1296 if (f->index >= ARRAY_SIZE(supported_formats))1297 return -EINVAL;1298 1299 f->pixelformat = supported_formats[f->index].fourcc;1300 f->flags = 0;1301 1302 return 0;1303}1304 1305static int pispbe_enum_framesizes(struct file *file, void *priv,1306 struct v4l2_frmsizeenum *fsize)1307{1308 struct pispbe_node *node = video_drvdata(file);1309 struct pispbe_dev *pispbe = node->pispbe;1310 1311 if (NODE_IS_META(node) || fsize->index)1312 return -EINVAL;1313 1314 if (!pispbe_find_fmt(fsize->pixel_format)) {1315 dev_dbg(pispbe->dev, "Invalid pixel code: %x\n",1316 fsize->pixel_format);1317 return -EINVAL;1318 }1319 1320 fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;1321 fsize->stepwise.min_width = 32;1322 fsize->stepwise.max_width = 65535;1323 fsize->stepwise.step_width = 2;1324 1325 fsize->stepwise.min_height = 32;1326 fsize->stepwise.max_height = 65535;1327 fsize->stepwise.step_height = 2;1328 1329 return 0;1330}1331 1332static const struct v4l2_ioctl_ops pispbe_node_ioctl_ops = {1333 .vidioc_querycap = pispbe_node_querycap,1334 .vidioc_g_fmt_vid_cap_mplane = pispbe_node_g_fmt_vid_cap,1335 .vidioc_g_fmt_vid_out_mplane = pispbe_node_g_fmt_vid_out,1336 .vidioc_g_fmt_meta_out = pispbe_node_g_fmt_meta_out,1337 .vidioc_try_fmt_vid_cap_mplane = pispbe_node_try_fmt_vid_cap,1338 .vidioc_try_fmt_vid_out_mplane = pispbe_node_try_fmt_vid_out,1339 .vidioc_try_fmt_meta_out = pispbe_node_try_fmt_meta_out,1340 .vidioc_s_fmt_vid_cap_mplane = pispbe_node_s_fmt_vid_cap,1341 .vidioc_s_fmt_vid_out_mplane = pispbe_node_s_fmt_vid_out,1342 .vidioc_s_fmt_meta_out = pispbe_node_s_fmt_meta_out,1343 .vidioc_enum_fmt_vid_cap = pispbe_node_enum_fmt,1344 .vidioc_enum_fmt_vid_out = pispbe_node_enum_fmt,1345 .vidioc_enum_fmt_meta_out = pispbe_node_enum_fmt,1346 .vidioc_enum_framesizes = pispbe_enum_framesizes,1347 .vidioc_create_bufs = vb2_ioctl_create_bufs,1348 .vidioc_prepare_buf = vb2_ioctl_prepare_buf,1349 .vidioc_querybuf = vb2_ioctl_querybuf,1350 .vidioc_qbuf = vb2_ioctl_qbuf,1351 .vidioc_dqbuf = vb2_ioctl_dqbuf,1352 .vidioc_expbuf = vb2_ioctl_expbuf,1353 .vidioc_reqbufs = vb2_ioctl_reqbufs,1354 .vidioc_streamon = vb2_ioctl_streamon,1355 .vidioc_streamoff = vb2_ioctl_streamoff,1356};1357 1358static const struct video_device pispbe_videodev = {1359 .name = PISPBE_NAME,1360 .vfl_dir = VFL_DIR_M2M, /* gets overwritten */1361 .fops = &pispbe_fops,1362 .ioctl_ops = &pispbe_node_ioctl_ops,1363 .minor = -1,1364 .release = video_device_release_empty,1365};1366 1367static void pispbe_node_def_fmt(struct pispbe_node *node)1368{1369 if (NODE_IS_META(node) && NODE_IS_OUTPUT(node)) {1370 /* Config node */1371 struct v4l2_format *f = &node->format;1372 1373 f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG;1374 f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config);1375 f->type = node->buf_type;1376 } else {1377 struct v4l2_format f = {1378 .fmt.pix_mp.pixelformat = V4L2_PIX_FMT_YUV420,1379 .fmt.pix_mp.width = 1920,1380 .fmt.pix_mp.height = 1080,1381 .type = node->buf_type,1382 };1383 pispbe_try_format(&f, node);1384 node->format = f;1385 }1386 1387 node->pisp_format = pispbe_find_fmt(node->format.fmt.pix_mp.pixelformat);1388}1389 1390/*1391 * Initialise a struct pispbe_node and register it as /dev/video<N>1392 * to represent one of the PiSP Back End's input or output streams.1393 */1394static int pispbe_init_node(struct pispbe_dev *pispbe, unsigned int id)1395{1396 bool output = NODE_DESC_IS_OUTPUT(&node_desc[id]);1397 struct pispbe_node *node = &pispbe->node[id];1398 struct media_entity *entity = &node->vfd.entity;1399 struct video_device *vdev = &node->vfd;1400 struct vb2_queue *q = &node->queue;1401 int ret;1402 1403 node->id = id;1404 node->pispbe = pispbe;1405 node->buf_type = node_desc[id].buf_type;1406 1407 mutex_init(&node->node_lock);1408 mutex_init(&node->queue_lock);1409 INIT_LIST_HEAD(&node->ready_queue);1410 spin_lock_init(&node->ready_lock);1411 1412 node->format.type = node->buf_type;1413 pispbe_node_def_fmt(node);1414 1415 q->type = node->buf_type;1416 q->io_modes = VB2_MMAP | VB2_DMABUF;1417 q->mem_ops = &vb2_dma_contig_memops;1418 q->drv_priv = node;1419 q->ops = &pispbe_node_queue_ops;1420 q->buf_struct_size = sizeof(struct pispbe_buffer);1421 q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;1422 q->dev = pispbe->dev;1423 /* get V4L2 to handle node->queue locking */1424 q->lock = &node->queue_lock;1425 1426 ret = vb2_queue_init(q);1427 if (ret < 0) {1428 dev_err(pispbe->dev, "vb2_queue_init failed\n");1429 goto err_mutex_destroy;1430 }1431 1432 *vdev = pispbe_videodev; /* default initialization */1433 strscpy(vdev->name, node_desc[id].ent_name, sizeof(vdev->name));1434 vdev->v4l2_dev = &pispbe->v4l2_dev;1435 vdev->vfl_dir = output ? VFL_DIR_TX : VFL_DIR_RX;1436 /* get V4L2 to serialise our ioctls */1437 vdev->lock = &node->node_lock;1438 vdev->queue = &node->queue;1439 vdev->device_caps = V4L2_CAP_STREAMING | node_desc[id].caps;1440 1441 node->pad.flags = output ? MEDIA_PAD_FL_SOURCE : MEDIA_PAD_FL_SINK;1442 ret = media_entity_pads_init(entity, 1, &node->pad);1443 if (ret) {1444 dev_err(pispbe->dev,1445 "Failed to register media pads for %s device node\n",1446 NODE_NAME(node));1447 goto err_unregister_queue;1448 }1449 1450 ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);1451 if (ret) {1452 dev_err(pispbe->dev,1453 "Failed to register video %s device node\n",1454 NODE_NAME(node));1455 goto err_unregister_queue;1456 }1457 video_set_drvdata(vdev, node);1458 1459 if (output)1460 ret = media_create_pad_link(entity, 0, &pispbe->sd.entity,1461 id, MEDIA_LNK_FL_IMMUTABLE |1462 MEDIA_LNK_FL_ENABLED);1463 else1464 ret = media_create_pad_link(&pispbe->sd.entity, id, entity,1465 0, MEDIA_LNK_FL_IMMUTABLE |1466 MEDIA_LNK_FL_ENABLED);1467 if (ret)1468 goto err_unregister_video_dev;1469 1470 dev_dbg(pispbe->dev, "%s device node registered as /dev/video%d\n",1471 NODE_NAME(node), node->vfd.num);1472 1473 return 0;1474 1475err_unregister_video_dev:1476 video_unregister_device(&node->vfd);1477err_unregister_queue:1478 vb2_queue_release(&node->queue);1479err_mutex_destroy:1480 mutex_destroy(&node->node_lock);1481 mutex_destroy(&node->queue_lock);1482 return ret;1483}1484 1485static const struct v4l2_subdev_pad_ops pispbe_pad_ops = {1486 .link_validate = v4l2_subdev_link_validate_default,1487};1488 1489static const struct v4l2_subdev_ops pispbe_sd_ops = {1490 .pad = &pispbe_pad_ops,1491};1492 1493static int pispbe_init_subdev(struct pispbe_dev *pispbe)1494{1495 struct v4l2_subdev *sd = &pispbe->sd;1496 int ret;1497 1498 v4l2_subdev_init(sd, &pispbe_sd_ops);1499 sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;1500 sd->owner = THIS_MODULE;1501 sd->dev = pispbe->dev;1502 strscpy(sd->name, PISPBE_NAME, sizeof(sd->name));1503 1504 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++)1505 pispbe->pad[i].flags =1506 NODE_DESC_IS_OUTPUT(&node_desc[i]) ?1507 MEDIA_PAD_FL_SINK : MEDIA_PAD_FL_SOURCE;1508 1509 ret = media_entity_pads_init(&sd->entity, PISPBE_NUM_NODES,1510 pispbe->pad);1511 if (ret)1512 goto error;1513 1514 ret = v4l2_device_register_subdev(&pispbe->v4l2_dev, sd);1515 if (ret)1516 goto error;1517 1518 return 0;1519 1520error:1521 media_entity_cleanup(&sd->entity);1522 return ret;1523}1524 1525static int pispbe_init_devices(struct pispbe_dev *pispbe)1526{1527 struct v4l2_device *v4l2_dev;1528 struct media_device *mdev;1529 unsigned int num_regist;1530 int ret;1531 1532 /* Register v4l2_device and media_device */1533 mdev = &pispbe->mdev;1534 mdev->hw_revision = pispbe->hw_version;1535 mdev->dev = pispbe->dev;1536 strscpy(mdev->model, PISPBE_NAME, sizeof(mdev->model));1537 media_device_init(mdev);1538 1539 v4l2_dev = &pispbe->v4l2_dev;1540 v4l2_dev->mdev = &pispbe->mdev;1541 strscpy(v4l2_dev->name, PISPBE_NAME, sizeof(v4l2_dev->name));1542 1543 ret = v4l2_device_register(pispbe->dev, v4l2_dev);1544 if (ret)1545 goto err_media_dev_cleanup;1546 1547 /* Register the PISPBE subdevice. */1548 ret = pispbe_init_subdev(pispbe);1549 if (ret)1550 goto err_unregister_v4l2;1551 1552 /* Create device video nodes */1553 for (num_regist = 0; num_regist < PISPBE_NUM_NODES; num_regist++) {1554 ret = pispbe_init_node(pispbe, num_regist);1555 if (ret)1556 goto err_unregister_nodes;1557 }1558 1559 ret = media_device_register(mdev);1560 if (ret)1561 goto err_unregister_nodes;1562 1563 pispbe->config =1564 dma_alloc_coherent(pispbe->dev,1565 sizeof(struct pisp_be_tiles_config) *1566 PISP_BE_NUM_CONFIG_BUFFERS,1567 &pispbe->config_dma_addr, GFP_KERNEL);1568 if (!pispbe->config) {1569 dev_err(pispbe->dev, "Unable to allocate cached config buffers.\n");1570 ret = -ENOMEM;1571 goto err_unregister_mdev;1572 }1573 1574 return 0;1575 1576err_unregister_mdev:1577 media_device_unregister(mdev);1578err_unregister_nodes:1579 while (num_regist-- > 0) {1580 video_unregister_device(&pispbe->node[num_regist].vfd);1581 vb2_queue_release(&pispbe->node[num_regist].queue);1582 }1583 v4l2_device_unregister_subdev(&pispbe->sd);1584 media_entity_cleanup(&pispbe->sd.entity);1585err_unregister_v4l2:1586 v4l2_device_unregister(v4l2_dev);1587err_media_dev_cleanup:1588 media_device_cleanup(mdev);1589 return ret;1590}1591 1592static void pispbe_destroy_devices(struct pispbe_dev *pispbe)1593{1594 if (pispbe->config) {1595 dma_free_coherent(pispbe->dev,1596 sizeof(struct pisp_be_tiles_config) *1597 PISP_BE_NUM_CONFIG_BUFFERS,1598 pispbe->config,1599 pispbe->config_dma_addr);1600 }1601 1602 dev_dbg(pispbe->dev, "Unregister from media controller\n");1603 1604 v4l2_device_unregister_subdev(&pispbe->sd);1605 media_entity_cleanup(&pispbe->sd.entity);1606 media_device_unregister(&pispbe->mdev);1607 1608 for (int i = PISPBE_NUM_NODES - 1; i >= 0; i--) {1609 video_unregister_device(&pispbe->node[i].vfd);1610 vb2_queue_release(&pispbe->node[i].queue);1611 mutex_destroy(&pispbe->node[i].node_lock);1612 mutex_destroy(&pispbe->node[i].queue_lock);1613 }1614 1615 media_device_cleanup(&pispbe->mdev);1616 v4l2_device_unregister(&pispbe->v4l2_dev);1617}1618 1619static int pispbe_runtime_suspend(struct device *dev)1620{1621 struct pispbe_dev *pispbe = dev_get_drvdata(dev);1622 1623 clk_disable_unprepare(pispbe->clk);1624 1625 return 0;1626}1627 1628static int pispbe_runtime_resume(struct device *dev)1629{1630 struct pispbe_dev *pispbe = dev_get_drvdata(dev);1631 int ret;1632 1633 ret = clk_prepare_enable(pispbe->clk);1634 if (ret) {1635 dev_err(dev, "Unable to enable clock\n");1636 return ret;1637 }1638 1639 dev_dbg(dev, "%s: Enabled clock, rate=%lu\n",1640 __func__, clk_get_rate(pispbe->clk));1641 1642 return 0;1643}1644 1645static int pispbe_hw_init(struct pispbe_dev *pispbe)1646{1647 u32 u;1648 1649 /* Check the HW is present and has a known version */1650 u = pispbe_rd(pispbe, PISP_BE_VERSION_REG);1651 dev_dbg(pispbe->dev, "pispbe_probe: HW version: 0x%08x", u);1652 pispbe->hw_version = u;1653 if ((u & ~PISP_BE_VERSION_MINOR_BITS) != PISP_BE_VERSION_2712)1654 return -ENODEV;1655 1656 /* Clear leftover interrupts */1657 pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, 0xFFFFFFFFu);1658 u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG);1659 dev_dbg(pispbe->dev, "pispbe_probe: BatchStatus: 0x%08x", u);1660 1661 pispbe->done = (uint8_t)u;1662 pispbe->started = (uint8_t)(u >> 8);1663 u = pispbe_rd(pispbe, PISP_BE_STATUS_REG);1664 dev_dbg(pispbe->dev, "pispbe_probe: Status: 0x%08x", u);1665 1666 if (u != 0 || pispbe->done != pispbe->started) {1667 dev_err(pispbe->dev, "pispbe_probe: HW is stuck or busy\n");1668 return -EBUSY;1669 }1670 1671 /*1672 * AXI QOS=0, CACHE=4'b0010, PROT=3'b0111673 * Also set "chicken bits" 22:20 which enable sub-64-byte bursts1674 * and AXI AWID/BID variability (on versions which support this).1675 */1676 pispbe_wr(pispbe, PISP_BE_AXI_REG, 0x32703200u);1677 1678 /* Enable both interrupt flags */1679 pispbe_wr(pispbe, PISP_BE_INTERRUPT_EN_REG, 0x00000003u);1680 1681 return 0;1682}1683 1684/* Probe the ISP-BE hardware block, as a single platform device. */1685static int pispbe_probe(struct platform_device *pdev)1686{1687 struct pispbe_dev *pispbe;1688 int ret;1689 1690 pispbe = devm_kzalloc(&pdev->dev, sizeof(*pispbe), GFP_KERNEL);1691 if (!pispbe)1692 return -ENOMEM;1693 1694 dev_set_drvdata(&pdev->dev, pispbe);1695 pispbe->dev = &pdev->dev;1696 platform_set_drvdata(pdev, pispbe);1697 1698 pispbe->be_reg_base = devm_platform_ioremap_resource(pdev, 0);1699 if (IS_ERR(pispbe->be_reg_base)) {1700 dev_err(&pdev->dev, "Failed to get ISP-BE registers address\n");1701 return PTR_ERR(pispbe->be_reg_base);1702 }1703 1704 pispbe->irq = platform_get_irq(pdev, 0);1705 if (pispbe->irq <= 0)1706 return -EINVAL;1707 1708 ret = devm_request_irq(&pdev->dev, pispbe->irq, pispbe_isr, 0,1709 PISPBE_NAME, pispbe);1710 if (ret) {1711 dev_err(&pdev->dev, "Unable to request interrupt\n");1712 return ret;1713 }1714 1715 ret = dma_set_mask_and_coherent(pispbe->dev, DMA_BIT_MASK(36));1716 if (ret)1717 return ret;1718 1719 pispbe->clk = devm_clk_get(&pdev->dev, NULL);1720 if (IS_ERR(pispbe->clk))1721 return dev_err_probe(&pdev->dev, PTR_ERR(pispbe->clk),1722 "Failed to get clock");1723 1724 /* Hardware initialisation */1725 pm_runtime_set_autosuspend_delay(pispbe->dev, 200);1726 pm_runtime_use_autosuspend(pispbe->dev);1727 pm_runtime_enable(pispbe->dev);1728 1729 ret = pispbe_runtime_resume(pispbe->dev);1730 if (ret)1731 goto pm_runtime_disable_err;1732 1733 pispbe->hw_busy = false;1734 spin_lock_init(&pispbe->hw_lock);1735 ret = pispbe_hw_init(pispbe);1736 if (ret)1737 goto pm_runtime_suspend_err;1738 1739 ret = pispbe_init_devices(pispbe);1740 if (ret)1741 goto disable_devs_err;1742 1743 pm_runtime_mark_last_busy(pispbe->dev);1744 pm_runtime_put_autosuspend(pispbe->dev);1745 1746 return 0;1747 1748disable_devs_err:1749 pispbe_destroy_devices(pispbe);1750pm_runtime_suspend_err:1751 pispbe_runtime_suspend(pispbe->dev);1752pm_runtime_disable_err:1753 pm_runtime_dont_use_autosuspend(pispbe->dev);1754 pm_runtime_disable(pispbe->dev);1755 1756 return ret;1757}1758 1759static void pispbe_remove(struct platform_device *pdev)1760{1761 struct pispbe_dev *pispbe = platform_get_drvdata(pdev);1762 1763 pispbe_destroy_devices(pispbe);1764 1765 pispbe_runtime_suspend(pispbe->dev);1766 pm_runtime_dont_use_autosuspend(pispbe->dev);1767 pm_runtime_disable(pispbe->dev);1768}1769 1770static const struct dev_pm_ops pispbe_pm_ops = {1771 SET_RUNTIME_PM_OPS(pispbe_runtime_suspend, pispbe_runtime_resume, NULL)1772};1773 1774static const struct of_device_id pispbe_of_match[] = {1775 {1776 .compatible = "raspberrypi,pispbe",1777 },1778 { /* sentinel */ },1779};1780MODULE_DEVICE_TABLE(of, pispbe_of_match);1781 1782static struct platform_driver pispbe_pdrv = {1783 .probe = pispbe_probe,1784 .remove_new = pispbe_remove,1785 .driver = {1786 .name = PISPBE_NAME,1787 .of_match_table = pispbe_of_match,1788 .pm = &pispbe_pm_ops,1789 },1790};1791 1792module_platform_driver(pispbe_pdrv);1793 1794MODULE_DESCRIPTION("PiSP Back End driver");1795MODULE_AUTHOR("David Plowman <david.plowman@raspberrypi.com>");1796MODULE_AUTHOR("Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>");1797MODULE_LICENSE("GPL");1798