1617 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2012 Samsung Electronics Co.Ltd4 * Authors: Joonyoung Shim <jy0922.shim@samsung.com>5 */6 7#include <linux/refcount.h>8#include <linux/clk.h>9#include <linux/component.h>10#include <linux/delay.h>11#include <linux/dma-mapping.h>12#include <linux/err.h>13#include <linux/interrupt.h>14#include <linux/io.h>15#include <linux/kernel.h>16#include <linux/of.h>17#include <linux/platform_device.h>18#include <linux/pm_runtime.h>19#include <linux/slab.h>20#include <linux/uaccess.h>21#include <linux/workqueue.h>22 23#include <drm/drm_file.h>24#include <drm/exynos_drm.h>25 26#include "exynos_drm_drv.h"27#include "exynos_drm_g2d.h"28#include "exynos_drm_gem.h"29 30#define G2D_HW_MAJOR_VER 431#define G2D_HW_MINOR_VER 132 33/* vaild register range set from user: 0x0104 ~ 0x0880 */34#define G2D_VALID_START 0x010435#define G2D_VALID_END 0x088036 37/* general registers */38#define G2D_SOFT_RESET 0x000039#define G2D_INTEN 0x000440#define G2D_INTC_PEND 0x000C41#define G2D_DMA_SFR_BASE_ADDR 0x008042#define G2D_DMA_COMMAND 0x008443#define G2D_DMA_STATUS 0x008C44#define G2D_DMA_HOLD_CMD 0x009045 46/* command registers */47#define G2D_BITBLT_START 0x010048 49/* registers for base address */50#define G2D_SRC_BASE_ADDR 0x030451#define G2D_SRC_STRIDE 0x030852#define G2D_SRC_COLOR_MODE 0x030C53#define G2D_SRC_LEFT_TOP 0x031054#define G2D_SRC_RIGHT_BOTTOM 0x031455#define G2D_SRC_PLANE2_BASE_ADDR 0x031856#define G2D_DST_BASE_ADDR 0x040457#define G2D_DST_STRIDE 0x040858#define G2D_DST_COLOR_MODE 0x040C59#define G2D_DST_LEFT_TOP 0x041060#define G2D_DST_RIGHT_BOTTOM 0x041461#define G2D_DST_PLANE2_BASE_ADDR 0x041862#define G2D_PAT_BASE_ADDR 0x050063#define G2D_MSK_BASE_ADDR 0x052064 65/* G2D_SOFT_RESET */66#define G2D_SFRCLEAR (1 << 1)67#define G2D_R (1 << 0)68 69/* G2D_INTEN */70#define G2D_INTEN_ACF (1 << 3)71#define G2D_INTEN_UCF (1 << 2)72#define G2D_INTEN_GCF (1 << 1)73#define G2D_INTEN_SCF (1 << 0)74 75/* G2D_INTC_PEND */76#define G2D_INTP_ACMD_FIN (1 << 3)77#define G2D_INTP_UCMD_FIN (1 << 2)78#define G2D_INTP_GCMD_FIN (1 << 1)79#define G2D_INTP_SCMD_FIN (1 << 0)80 81/* G2D_DMA_COMMAND */82#define G2D_DMA_HALT (1 << 2)83#define G2D_DMA_CONTINUE (1 << 1)84#define G2D_DMA_START (1 << 0)85 86/* G2D_DMA_STATUS */87#define G2D_DMA_LIST_DONE_COUNT (0xFF << 17)88#define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1)89#define G2D_DMA_DONE (1 << 0)90#define G2D_DMA_LIST_DONE_COUNT_OFFSET 1791 92/* G2D_DMA_HOLD_CMD */93#define G2D_USER_HOLD (1 << 2)94#define G2D_LIST_HOLD (1 << 1)95#define G2D_BITBLT_HOLD (1 << 0)96 97/* G2D_BITBLT_START */98#define G2D_START_CASESEL (1 << 2)99#define G2D_START_NHOLT (1 << 1)100#define G2D_START_BITBLT (1 << 0)101 102/* buffer color format */103#define G2D_FMT_XRGB8888 0104#define G2D_FMT_ARGB8888 1105#define G2D_FMT_RGB565 2106#define G2D_FMT_XRGB1555 3107#define G2D_FMT_ARGB1555 4108#define G2D_FMT_XRGB4444 5109#define G2D_FMT_ARGB4444 6110#define G2D_FMT_PACKED_RGB888 7111#define G2D_FMT_A8 11112#define G2D_FMT_L8 12113 114/* buffer valid length */115#define G2D_LEN_MIN 1116#define G2D_LEN_MAX 8000117 118#define G2D_CMDLIST_SIZE (PAGE_SIZE / 4)119#define G2D_CMDLIST_NUM 64120#define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)121#define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2)122 123/* maximum buffer pool size of userptr is 64MB as default */124#define MAX_POOL (64 * 1024 * 1024)125 126enum {127 BUF_TYPE_GEM = 1,128 BUF_TYPE_USERPTR,129};130 131enum g2d_reg_type {132 REG_TYPE_NONE = -1,133 REG_TYPE_SRC,134 REG_TYPE_SRC_PLANE2,135 REG_TYPE_DST,136 REG_TYPE_DST_PLANE2,137 REG_TYPE_PAT,138 REG_TYPE_MSK,139 MAX_REG_TYPE_NR140};141 142enum g2d_flag_bits {143 /*144 * If set, suspends the runqueue worker after the currently145 * processed node is finished.146 */147 G2D_BIT_SUSPEND_RUNQUEUE,148 /*149 * If set, indicates that the engine is currently busy.150 */151 G2D_BIT_ENGINE_BUSY,152};153 154/* cmdlist data structure */155struct g2d_cmdlist {156 u32 head;157 unsigned long data[G2D_CMDLIST_DATA_NUM];158 u32 last; /* last data offset */159};160 161/*162 * A structure of buffer description163 *164 * @format: color format165 * @stride: buffer stride/pitch in bytes166 * @left_x: the x coordinates of left top corner167 * @top_y: the y coordinates of left top corner168 * @right_x: the x coordinates of right bottom corner169 * @bottom_y: the y coordinates of right bottom corner170 *171 */172struct g2d_buf_desc {173 unsigned int format;174 unsigned int stride;175 unsigned int left_x;176 unsigned int top_y;177 unsigned int right_x;178 unsigned int bottom_y;179};180 181/*182 * A structure of buffer information183 *184 * @map_nr: manages the number of mapped buffers185 * @reg_types: stores regitster type in the order of requested command186 * @handles: stores buffer handle in its reg_type position187 * @types: stores buffer type in its reg_type position188 * @descs: stores buffer description in its reg_type position189 *190 */191struct g2d_buf_info {192 unsigned int map_nr;193 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR];194 void *obj[MAX_REG_TYPE_NR];195 unsigned int types[MAX_REG_TYPE_NR];196 struct g2d_buf_desc descs[MAX_REG_TYPE_NR];197};198 199struct drm_exynos_pending_g2d_event {200 struct drm_pending_event base;201 struct drm_exynos_g2d_event event;202};203 204struct g2d_cmdlist_userptr {205 struct list_head list;206 dma_addr_t dma_addr;207 unsigned long userptr;208 unsigned long size;209 struct page **pages;210 unsigned int npages;211 struct sg_table *sgt;212 refcount_t refcount;213 bool in_pool;214 bool out_of_list;215};216struct g2d_cmdlist_node {217 struct list_head list;218 struct g2d_cmdlist *cmdlist;219 dma_addr_t dma_addr;220 struct g2d_buf_info buf_info;221 222 struct drm_exynos_pending_g2d_event *event;223};224 225struct g2d_runqueue_node {226 struct list_head list;227 struct list_head run_cmdlist;228 struct list_head event_list;229 struct drm_file *filp;230 pid_t pid;231 struct completion complete;232 int async;233};234 235struct g2d_data {236 struct device *dev;237 void *dma_priv;238 struct clk *gate_clk;239 void __iomem *regs;240 int irq;241 struct workqueue_struct *g2d_workq;242 struct work_struct runqueue_work;243 struct drm_device *drm_dev;244 unsigned long flags;245 246 /* cmdlist */247 struct g2d_cmdlist_node *cmdlist_node;248 struct list_head free_cmdlist;249 struct mutex cmdlist_mutex;250 dma_addr_t cmdlist_pool;251 void *cmdlist_pool_virt;252 unsigned long cmdlist_dma_attrs;253 254 /* runqueue*/255 struct g2d_runqueue_node *runqueue_node;256 struct list_head runqueue;257 struct mutex runqueue_mutex;258 struct kmem_cache *runqueue_slab;259 260 unsigned long current_pool;261 unsigned long max_pool;262};263 264static inline void g2d_hw_reset(struct g2d_data *g2d)265{266 writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET);267 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);268}269 270static int g2d_init_cmdlist(struct g2d_data *g2d)271{272 struct device *dev = g2d->dev;273 struct g2d_cmdlist_node *node;274 int nr;275 int ret;276 struct g2d_buf_info *buf_info;277 278 g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;279 280 g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev),281 G2D_CMDLIST_POOL_SIZE,282 &g2d->cmdlist_pool, GFP_KERNEL,283 g2d->cmdlist_dma_attrs);284 if (!g2d->cmdlist_pool_virt) {285 dev_err(dev, "failed to allocate dma memory\n");286 return -ENOMEM;287 }288 289 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);290 if (!node) {291 ret = -ENOMEM;292 goto err;293 }294 295 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {296 unsigned int i;297 298 node[nr].cmdlist =299 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;300 node[nr].dma_addr =301 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;302 303 buf_info = &node[nr].buf_info;304 for (i = 0; i < MAX_REG_TYPE_NR; i++)305 buf_info->reg_types[i] = REG_TYPE_NONE;306 307 list_add_tail(&node[nr].list, &g2d->free_cmdlist);308 }309 310 return 0;311 312err:313 dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,314 g2d->cmdlist_pool_virt,315 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);316 return ret;317}318 319static void g2d_fini_cmdlist(struct g2d_data *g2d)320{321 kfree(g2d->cmdlist_node);322 323 if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {324 dma_free_attrs(to_dma_dev(g2d->drm_dev),325 G2D_CMDLIST_POOL_SIZE,326 g2d->cmdlist_pool_virt,327 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);328 }329}330 331static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)332{333 struct device *dev = g2d->dev;334 struct g2d_cmdlist_node *node;335 336 mutex_lock(&g2d->cmdlist_mutex);337 if (list_empty(&g2d->free_cmdlist)) {338 dev_err(dev, "there is no free cmdlist\n");339 mutex_unlock(&g2d->cmdlist_mutex);340 return NULL;341 }342 343 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,344 list);345 list_del_init(&node->list);346 mutex_unlock(&g2d->cmdlist_mutex);347 348 return node;349}350 351static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)352{353 mutex_lock(&g2d->cmdlist_mutex);354 list_move_tail(&node->list, &g2d->free_cmdlist);355 mutex_unlock(&g2d->cmdlist_mutex);356}357 358static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,359 struct g2d_cmdlist_node *node)360{361 struct g2d_cmdlist_node *lnode;362 363 if (list_empty(&file_priv->inuse_cmdlist))364 goto add_to_list;365 366 /* this links to base address of new cmdlist */367 lnode = list_entry(file_priv->inuse_cmdlist.prev,368 struct g2d_cmdlist_node, list);369 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;370 371add_to_list:372 list_add_tail(&node->list, &file_priv->inuse_cmdlist);373 374 if (node->event)375 list_add_tail(&node->event->base.link, &file_priv->event_list);376}377 378static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,379 void *obj,380 bool force)381{382 struct g2d_cmdlist_userptr *g2d_userptr = obj;383 384 if (!obj)385 return;386 387 if (force)388 goto out;389 390 refcount_dec(&g2d_userptr->refcount);391 392 if (refcount_read(&g2d_userptr->refcount) > 0)393 return;394 395 if (g2d_userptr->in_pool)396 return;397 398out:399 dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt,400 DMA_BIDIRECTIONAL, 0);401 402 unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages,403 true);404 kvfree(g2d_userptr->pages);405 406 if (!g2d_userptr->out_of_list)407 list_del_init(&g2d_userptr->list);408 409 sg_free_table(g2d_userptr->sgt);410 kfree(g2d_userptr->sgt);411 kfree(g2d_userptr);412}413 414static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,415 unsigned long userptr,416 unsigned long size,417 struct drm_file *filp,418 void **obj)419{420 struct drm_exynos_file_private *file_priv = filp->driver_priv;421 struct g2d_cmdlist_userptr *g2d_userptr;422 struct sg_table *sgt;423 unsigned long start, end;424 unsigned int npages, offset;425 int ret;426 427 if (!size) {428 DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");429 return ERR_PTR(-EINVAL);430 }431 432 /* check if userptr already exists in userptr_list. */433 list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {434 if (g2d_userptr->userptr == userptr) {435 /*436 * also check size because there could be same address437 * and different size.438 */439 if (g2d_userptr->size == size) {440 refcount_inc(&g2d_userptr->refcount);441 *obj = g2d_userptr;442 443 return &g2d_userptr->dma_addr;444 }445 446 /*447 * at this moment, maybe g2d dma is accessing this448 * g2d_userptr memory region so just remove this449 * g2d_userptr object from userptr_list not to be450 * referred again and also except it the userptr451 * pool to be released after the dma access completion.452 */453 g2d_userptr->out_of_list = true;454 g2d_userptr->in_pool = false;455 list_del_init(&g2d_userptr->list);456 457 break;458 }459 }460 461 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);462 if (!g2d_userptr)463 return ERR_PTR(-ENOMEM);464 465 refcount_set(&g2d_userptr->refcount, 1);466 g2d_userptr->size = size;467 468 start = userptr & PAGE_MASK;469 offset = userptr & ~PAGE_MASK;470 end = PAGE_ALIGN(userptr + size);471 npages = (end - start) >> PAGE_SHIFT;472 g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages),473 GFP_KERNEL);474 if (!g2d_userptr->pages) {475 ret = -ENOMEM;476 goto err_free;477 }478 479 ret = pin_user_pages_fast(start, npages,480 FOLL_WRITE | FOLL_LONGTERM,481 g2d_userptr->pages);482 if (ret != npages) {483 DRM_DEV_ERROR(g2d->dev,484 "failed to get user pages from userptr.\n");485 if (ret < 0)486 goto err_destroy_pages;487 npages = ret;488 ret = -EFAULT;489 goto err_unpin_pages;490 }491 g2d_userptr->npages = npages;492 493 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);494 if (!sgt) {495 ret = -ENOMEM;496 goto err_unpin_pages;497 }498 499 ret = sg_alloc_table_from_pages(sgt,500 g2d_userptr->pages,501 npages, offset, size, GFP_KERNEL);502 if (ret < 0) {503 DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");504 goto err_free_sgt;505 }506 507 g2d_userptr->sgt = sgt;508 509 ret = dma_map_sgtable(to_dma_dev(g2d->drm_dev), sgt,510 DMA_BIDIRECTIONAL, 0);511 if (ret) {512 DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");513 goto err_sg_free_table;514 }515 516 g2d_userptr->dma_addr = sgt->sgl[0].dma_address;517 g2d_userptr->userptr = userptr;518 519 list_add_tail(&g2d_userptr->list, &file_priv->userptr_list);520 521 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {522 g2d->current_pool += npages << PAGE_SHIFT;523 g2d_userptr->in_pool = true;524 }525 526 *obj = g2d_userptr;527 528 return &g2d_userptr->dma_addr;529 530err_sg_free_table:531 sg_free_table(sgt);532 533err_free_sgt:534 kfree(sgt);535 536err_unpin_pages:537 unpin_user_pages(g2d_userptr->pages, npages);538 539err_destroy_pages:540 kvfree(g2d_userptr->pages);541 542err_free:543 kfree(g2d_userptr);544 545 return ERR_PTR(ret);546}547 548static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)549{550 struct drm_exynos_file_private *file_priv = filp->driver_priv;551 struct g2d_cmdlist_userptr *g2d_userptr, *n;552 553 list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)554 if (g2d_userptr->in_pool)555 g2d_userptr_put_dma_addr(g2d, g2d_userptr, true);556 557 g2d->current_pool = 0;558}559 560static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)561{562 enum g2d_reg_type reg_type;563 564 switch (reg_offset) {565 case G2D_SRC_BASE_ADDR:566 case G2D_SRC_STRIDE:567 case G2D_SRC_COLOR_MODE:568 case G2D_SRC_LEFT_TOP:569 case G2D_SRC_RIGHT_BOTTOM:570 reg_type = REG_TYPE_SRC;571 break;572 case G2D_SRC_PLANE2_BASE_ADDR:573 reg_type = REG_TYPE_SRC_PLANE2;574 break;575 case G2D_DST_BASE_ADDR:576 case G2D_DST_STRIDE:577 case G2D_DST_COLOR_MODE:578 case G2D_DST_LEFT_TOP:579 case G2D_DST_RIGHT_BOTTOM:580 reg_type = REG_TYPE_DST;581 break;582 case G2D_DST_PLANE2_BASE_ADDR:583 reg_type = REG_TYPE_DST_PLANE2;584 break;585 case G2D_PAT_BASE_ADDR:586 reg_type = REG_TYPE_PAT;587 break;588 case G2D_MSK_BASE_ADDR:589 reg_type = REG_TYPE_MSK;590 break;591 default:592 reg_type = REG_TYPE_NONE;593 DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",594 reg_offset);595 break;596 }597 598 return reg_type;599}600 601static unsigned long g2d_get_buf_bpp(unsigned int format)602{603 unsigned long bpp;604 605 switch (format) {606 case G2D_FMT_XRGB8888:607 case G2D_FMT_ARGB8888:608 bpp = 4;609 break;610 case G2D_FMT_RGB565:611 case G2D_FMT_XRGB1555:612 case G2D_FMT_ARGB1555:613 case G2D_FMT_XRGB4444:614 case G2D_FMT_ARGB4444:615 bpp = 2;616 break;617 case G2D_FMT_PACKED_RGB888:618 bpp = 3;619 break;620 default:621 bpp = 1;622 break;623 }624 625 return bpp;626}627 628static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,629 struct g2d_buf_desc *buf_desc,630 enum g2d_reg_type reg_type,631 unsigned long size)632{633 int width, height;634 unsigned long bpp, last_pos;635 636 /*637 * check source and destination buffers only.638 * so the others are always valid.639 */640 if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST)641 return true;642 643 /* This check also makes sure that right_x > left_x. */644 width = (int)buf_desc->right_x - (int)buf_desc->left_x;645 if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {646 DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);647 return false;648 }649 650 /* This check also makes sure that bottom_y > top_y. */651 height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;652 if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {653 DRM_DEV_ERROR(g2d->dev,654 "height[%d] is out of range!\n", height);655 return false;656 }657 658 bpp = g2d_get_buf_bpp(buf_desc->format);659 660 /* Compute the position of the last byte that the engine accesses. */661 last_pos = ((unsigned long)buf_desc->bottom_y - 1) *662 (unsigned long)buf_desc->stride +663 (unsigned long)buf_desc->right_x * bpp - 1;664 665 /*666 * Since right_x > left_x and bottom_y > top_y we already know667 * that the first_pos < last_pos (first_pos being the position668 * of the first byte the engine accesses), it just remains to669 * check if last_pos is smaller then the buffer size.670 */671 672 if (last_pos >= size) {673 DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "674 "is out of range [%lu]!\n", last_pos, size);675 return false;676 }677 678 return true;679}680 681static int g2d_map_cmdlist_gem(struct g2d_data *g2d,682 struct g2d_cmdlist_node *node,683 struct drm_device *drm_dev,684 struct drm_file *file)685{686 struct g2d_cmdlist *cmdlist = node->cmdlist;687 struct g2d_buf_info *buf_info = &node->buf_info;688 int offset;689 int ret;690 int i;691 692 for (i = 0; i < buf_info->map_nr; i++) {693 struct g2d_buf_desc *buf_desc;694 enum g2d_reg_type reg_type;695 int reg_pos;696 unsigned long handle;697 dma_addr_t *addr;698 699 reg_pos = cmdlist->last - 2 * (i + 1);700 701 offset = cmdlist->data[reg_pos];702 handle = cmdlist->data[reg_pos + 1];703 704 reg_type = g2d_get_reg_type(g2d, offset);705 if (reg_type == REG_TYPE_NONE) {706 ret = -EFAULT;707 goto err;708 }709 710 buf_desc = &buf_info->descs[reg_type];711 712 if (buf_info->types[reg_type] == BUF_TYPE_GEM) {713 struct exynos_drm_gem *exynos_gem;714 715 exynos_gem = exynos_drm_gem_get(file, handle);716 if (!exynos_gem) {717 ret = -EFAULT;718 goto err;719 }720 721 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,722 reg_type, exynos_gem->size)) {723 exynos_drm_gem_put(exynos_gem);724 ret = -EFAULT;725 goto err;726 }727 728 addr = &exynos_gem->dma_addr;729 buf_info->obj[reg_type] = exynos_gem;730 } else {731 struct drm_exynos_g2d_userptr g2d_userptr;732 733 if (copy_from_user(&g2d_userptr, (void __user *)handle,734 sizeof(struct drm_exynos_g2d_userptr))) {735 ret = -EFAULT;736 goto err;737 }738 739 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,740 reg_type,741 g2d_userptr.size)) {742 ret = -EFAULT;743 goto err;744 }745 746 addr = g2d_userptr_get_dma_addr(g2d,747 g2d_userptr.userptr,748 g2d_userptr.size,749 file,750 &buf_info->obj[reg_type]);751 if (IS_ERR(addr)) {752 ret = -EFAULT;753 goto err;754 }755 }756 757 cmdlist->data[reg_pos + 1] = *addr;758 buf_info->reg_types[i] = reg_type;759 }760 761 return 0;762 763err:764 buf_info->map_nr = i;765 return ret;766}767 768static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,769 struct g2d_cmdlist_node *node,770 struct drm_file *filp)771{772 struct g2d_buf_info *buf_info = &node->buf_info;773 int i;774 775 for (i = 0; i < buf_info->map_nr; i++) {776 struct g2d_buf_desc *buf_desc;777 enum g2d_reg_type reg_type;778 void *obj;779 780 reg_type = buf_info->reg_types[i];781 782 buf_desc = &buf_info->descs[reg_type];783 obj = buf_info->obj[reg_type];784 785 if (buf_info->types[reg_type] == BUF_TYPE_GEM)786 exynos_drm_gem_put(obj);787 else788 g2d_userptr_put_dma_addr(g2d, obj, false);789 790 buf_info->reg_types[i] = REG_TYPE_NONE;791 buf_info->obj[reg_type] = NULL;792 buf_info->types[reg_type] = 0;793 memset(buf_desc, 0x00, sizeof(*buf_desc));794 }795 796 buf_info->map_nr = 0;797}798 799static void g2d_dma_start(struct g2d_data *g2d,800 struct g2d_runqueue_node *runqueue_node)801{802 struct g2d_cmdlist_node *node =803 list_first_entry(&runqueue_node->run_cmdlist,804 struct g2d_cmdlist_node, list);805 806 set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);807 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);808 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);809}810 811static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)812{813 struct g2d_runqueue_node *runqueue_node;814 815 if (list_empty(&g2d->runqueue))816 return NULL;817 818 runqueue_node = list_first_entry(&g2d->runqueue,819 struct g2d_runqueue_node, list);820 list_del_init(&runqueue_node->list);821 return runqueue_node;822}823 824static void g2d_free_runqueue_node(struct g2d_data *g2d,825 struct g2d_runqueue_node *runqueue_node)826{827 struct g2d_cmdlist_node *node;828 829 mutex_lock(&g2d->cmdlist_mutex);830 /*831 * commands in run_cmdlist have been completed so unmap all gem832 * objects in each command node so that they are unreferenced.833 */834 list_for_each_entry(node, &runqueue_node->run_cmdlist, list)835 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);836 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);837 mutex_unlock(&g2d->cmdlist_mutex);838 839 kmem_cache_free(g2d->runqueue_slab, runqueue_node);840}841 842/**843 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes844 * @g2d: G2D state object845 * @file: if not zero, only remove items with this DRM file846 *847 * Has to be called under runqueue lock.848 */849static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)850{851 struct g2d_runqueue_node *node, *n;852 853 if (list_empty(&g2d->runqueue))854 return;855 856 list_for_each_entry_safe(node, n, &g2d->runqueue, list) {857 if (file && node->filp != file)858 continue;859 860 list_del_init(&node->list);861 g2d_free_runqueue_node(g2d, node);862 }863}864 865static void g2d_runqueue_worker(struct work_struct *work)866{867 struct g2d_data *g2d = container_of(work, struct g2d_data,868 runqueue_work);869 struct g2d_runqueue_node *runqueue_node;870 871 /*872 * The engine is busy and the completion of the current node is going873 * to poke the runqueue worker, so nothing to do here.874 */875 if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))876 return;877 878 mutex_lock(&g2d->runqueue_mutex);879 880 runqueue_node = g2d->runqueue_node;881 g2d->runqueue_node = NULL;882 883 if (runqueue_node) {884 pm_runtime_mark_last_busy(g2d->dev);885 pm_runtime_put_autosuspend(g2d->dev);886 887 complete(&runqueue_node->complete);888 if (runqueue_node->async)889 g2d_free_runqueue_node(g2d, runqueue_node);890 }891 892 if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {893 g2d->runqueue_node = g2d_get_runqueue_node(g2d);894 895 if (g2d->runqueue_node) {896 int ret;897 898 ret = pm_runtime_resume_and_get(g2d->dev);899 if (ret < 0) {900 dev_err(g2d->dev, "failed to enable G2D device.\n");901 goto out;902 }903 904 g2d_dma_start(g2d, g2d->runqueue_node);905 }906 }907 908out:909 mutex_unlock(&g2d->runqueue_mutex);910}911 912static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)913{914 struct drm_device *drm_dev = g2d->drm_dev;915 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;916 struct drm_exynos_pending_g2d_event *e;917 struct timespec64 now;918 919 if (list_empty(&runqueue_node->event_list))920 return;921 922 e = list_first_entry(&runqueue_node->event_list,923 struct drm_exynos_pending_g2d_event, base.link);924 925 ktime_get_ts64(&now);926 e->event.tv_sec = now.tv_sec;927 e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;928 e->event.cmdlist_no = cmdlist_no;929 930 drm_send_event(drm_dev, &e->base);931}932 933static irqreturn_t g2d_irq_handler(int irq, void *dev_id)934{935 struct g2d_data *g2d = dev_id;936 u32 pending;937 938 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);939 if (pending)940 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);941 942 if (pending & G2D_INTP_GCMD_FIN) {943 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);944 945 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>946 G2D_DMA_LIST_DONE_COUNT_OFFSET;947 948 g2d_finish_event(g2d, cmdlist_no);949 950 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);951 if (!(pending & G2D_INTP_ACMD_FIN)) {952 writel_relaxed(G2D_DMA_CONTINUE,953 g2d->regs + G2D_DMA_COMMAND);954 }955 }956 957 if (pending & G2D_INTP_ACMD_FIN) {958 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);959 queue_work(g2d->g2d_workq, &g2d->runqueue_work);960 }961 962 return IRQ_HANDLED;963}964 965/**966 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node967 * @g2d: G2D state object968 * @file: if not zero, only wait if the current runqueue node belongs969 * to the DRM file970 *971 * Should the engine not become idle after a 100ms timeout, a hardware972 * reset is issued.973 */974static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)975{976 struct device *dev = g2d->dev;977 978 struct g2d_runqueue_node *runqueue_node = NULL;979 unsigned int tries = 10;980 981 mutex_lock(&g2d->runqueue_mutex);982 983 /* If no node is currently processed, we have nothing to do. */984 if (!g2d->runqueue_node)985 goto out;986 987 runqueue_node = g2d->runqueue_node;988 989 /* Check if the currently processed item belongs to us. */990 if (file && runqueue_node->filp != file)991 goto out;992 993 mutex_unlock(&g2d->runqueue_mutex);994 995 /* Wait for the G2D engine to finish. */996 while (tries-- && (g2d->runqueue_node == runqueue_node))997 mdelay(10);998 999 mutex_lock(&g2d->runqueue_mutex);1000 1001 if (g2d->runqueue_node != runqueue_node)1002 goto out;1003 1004 dev_err(dev, "wait timed out, resetting engine...\n");1005 g2d_hw_reset(g2d);1006 1007 /*1008 * After the hardware reset of the engine we are going to loose1009 * the IRQ which triggers the PM runtime put().1010 * So do this manually here.1011 */1012 pm_runtime_mark_last_busy(dev);1013 pm_runtime_put_autosuspend(dev);1014 1015 complete(&runqueue_node->complete);1016 if (runqueue_node->async)1017 g2d_free_runqueue_node(g2d, runqueue_node);1018 1019out:1020 mutex_unlock(&g2d->runqueue_mutex);1021}1022 1023static int g2d_check_reg_offset(struct g2d_data *g2d,1024 struct g2d_cmdlist_node *node,1025 int nr, bool for_addr)1026{1027 struct g2d_cmdlist *cmdlist = node->cmdlist;1028 int reg_offset;1029 int index;1030 int i;1031 1032 for (i = 0; i < nr; i++) {1033 struct g2d_buf_info *buf_info = &node->buf_info;1034 struct g2d_buf_desc *buf_desc;1035 enum g2d_reg_type reg_type;1036 unsigned long value;1037 1038 index = cmdlist->last - 2 * (i + 1);1039 1040 reg_offset = cmdlist->data[index] & ~0xfffff000;1041 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)1042 goto err;1043 if (reg_offset % 4)1044 goto err;1045 1046 switch (reg_offset) {1047 case G2D_SRC_BASE_ADDR:1048 case G2D_SRC_PLANE2_BASE_ADDR:1049 case G2D_DST_BASE_ADDR:1050 case G2D_DST_PLANE2_BASE_ADDR:1051 case G2D_PAT_BASE_ADDR:1052 case G2D_MSK_BASE_ADDR:1053 if (!for_addr)1054 goto err;1055 1056 reg_type = g2d_get_reg_type(g2d, reg_offset);1057 1058 /* check userptr buffer type. */1059 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {1060 buf_info->types[reg_type] = BUF_TYPE_USERPTR;1061 cmdlist->data[index] &= ~G2D_BUF_USERPTR;1062 } else1063 buf_info->types[reg_type] = BUF_TYPE_GEM;1064 break;1065 case G2D_SRC_STRIDE:1066 case G2D_DST_STRIDE:1067 if (for_addr)1068 goto err;1069 1070 reg_type = g2d_get_reg_type(g2d, reg_offset);1071 1072 buf_desc = &buf_info->descs[reg_type];1073 buf_desc->stride = cmdlist->data[index + 1];1074 break;1075 case G2D_SRC_COLOR_MODE:1076 case G2D_DST_COLOR_MODE:1077 if (for_addr)1078 goto err;1079 1080 reg_type = g2d_get_reg_type(g2d, reg_offset);1081 1082 buf_desc = &buf_info->descs[reg_type];1083 value = cmdlist->data[index + 1];1084 1085 buf_desc->format = value & 0xf;1086 break;1087 case G2D_SRC_LEFT_TOP:1088 case G2D_DST_LEFT_TOP:1089 if (for_addr)1090 goto err;1091 1092 reg_type = g2d_get_reg_type(g2d, reg_offset);1093 1094 buf_desc = &buf_info->descs[reg_type];1095 value = cmdlist->data[index + 1];1096 1097 buf_desc->left_x = value & 0x1fff;1098 buf_desc->top_y = (value & 0x1fff0000) >> 16;1099 break;1100 case G2D_SRC_RIGHT_BOTTOM:1101 case G2D_DST_RIGHT_BOTTOM:1102 if (for_addr)1103 goto err;1104 1105 reg_type = g2d_get_reg_type(g2d, reg_offset);1106 1107 buf_desc = &buf_info->descs[reg_type];1108 value = cmdlist->data[index + 1];1109 1110 buf_desc->right_x = value & 0x1fff;1111 buf_desc->bottom_y = (value & 0x1fff0000) >> 16;1112 break;1113 default:1114 if (for_addr)1115 goto err;1116 break;1117 }1118 }1119 1120 return 0;1121 1122err:1123 dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);1124 return -EINVAL;1125}1126 1127/* ioctl functions */1128int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,1129 struct drm_file *file)1130{1131 struct drm_exynos_g2d_get_ver *ver = data;1132 1133 ver->major = G2D_HW_MAJOR_VER;1134 ver->minor = G2D_HW_MINOR_VER;1135 1136 return 0;1137}1138 1139int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,1140 struct drm_file *file)1141{1142 struct drm_exynos_file_private *file_priv = file->driver_priv;1143 struct exynos_drm_private *priv = drm_dev->dev_private;1144 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);1145 struct drm_exynos_g2d_set_cmdlist *req = data;1146 struct drm_exynos_g2d_cmd *cmd;1147 struct drm_exynos_pending_g2d_event *e;1148 struct g2d_cmdlist_node *node;1149 struct g2d_cmdlist *cmdlist;1150 int size;1151 int ret;1152 1153 node = g2d_get_cmdlist(g2d);1154 if (!node)1155 return -ENOMEM;1156 1157 /*1158 * To avoid an integer overflow for the later size computations, we1159 * enforce a maximum number of submitted commands here. This limit is1160 * sufficient for all conceivable usage cases of the G2D.1161 */1162 if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||1163 req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {1164 dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");1165 return -EINVAL;1166 }1167 1168 node->event = NULL;1169 1170 if (req->event_type != G2D_EVENT_NOT) {1171 e = kzalloc(sizeof(*node->event), GFP_KERNEL);1172 if (!e) {1173 ret = -ENOMEM;1174 goto err;1175 }1176 1177 e->event.base.type = DRM_EXYNOS_G2D_EVENT;1178 e->event.base.length = sizeof(e->event);1179 e->event.user_data = req->user_data;1180 1181 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base);1182 if (ret) {1183 kfree(e);1184 goto err;1185 }1186 1187 node->event = e;1188 }1189 1190 cmdlist = node->cmdlist;1191 1192 cmdlist->last = 0;1193 1194 /*1195 * If don't clear SFR registers, the cmdlist is affected by register1196 * values of previous cmdlist. G2D hw executes SFR clear command and1197 * a next command at the same time then the next command is ignored and1198 * is executed rightly from next next command, so needs a dummy command1199 * to next command of SFR clear command.1200 */1201 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;1202 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;1203 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;1204 cmdlist->data[cmdlist->last++] = 0;1205 1206 /*1207 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG1208 * and GCF bit should be set to INTEN register if user wants1209 * G2D interrupt event once current command list execution is1210 * finished.1211 * Otherwise only ACF bit should be set to INTEN register so1212 * that one interrupt is occurred after all command lists1213 * have been completed.1214 */1215 if (node->event) {1216 cmdlist->data[cmdlist->last++] = G2D_INTEN;1217 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;1218 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;1219 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;1220 } else {1221 cmdlist->data[cmdlist->last++] = G2D_INTEN;1222 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;1223 }1224 1225 /*1226 * Check the size of cmdlist. The 2 that is added last comes from1227 * the implicit G2D_BITBLT_START that is appended once we have1228 * checked all the submitted commands.1229 */1230 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;1231 if (size > G2D_CMDLIST_DATA_NUM) {1232 dev_err(g2d->dev, "cmdlist size is too big\n");1233 ret = -EINVAL;1234 goto err_free_event;1235 }1236 1237 cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd;1238 1239 if (copy_from_user(cmdlist->data + cmdlist->last,1240 (void __user *)cmd,1241 sizeof(*cmd) * req->cmd_nr)) {1242 ret = -EFAULT;1243 goto err_free_event;1244 }1245 cmdlist->last += req->cmd_nr * 2;1246 1247 ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false);1248 if (ret < 0)1249 goto err_free_event;1250 1251 node->buf_info.map_nr = req->cmd_buf_nr;1252 if (req->cmd_buf_nr) {1253 struct drm_exynos_g2d_cmd *cmd_buf;1254 1255 cmd_buf = (struct drm_exynos_g2d_cmd *)1256 (unsigned long)req->cmd_buf;1257 1258 if (copy_from_user(cmdlist->data + cmdlist->last,1259 (void __user *)cmd_buf,1260 sizeof(*cmd_buf) * req->cmd_buf_nr)) {1261 ret = -EFAULT;1262 goto err_free_event;1263 }1264 cmdlist->last += req->cmd_buf_nr * 2;1265 1266 ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true);1267 if (ret < 0)1268 goto err_free_event;1269 1270 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);1271 if (ret < 0)1272 goto err_unmap;1273 }1274 1275 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;1276 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;1277 1278 /* head */1279 cmdlist->head = cmdlist->last / 2;1280 1281 /* tail */1282 cmdlist->data[cmdlist->last] = 0;1283 1284 g2d_add_cmdlist_to_inuse(file_priv, node);1285 1286 return 0;1287 1288err_unmap:1289 g2d_unmap_cmdlist_gem(g2d, node, file);1290err_free_event:1291 if (node->event)1292 drm_event_cancel_free(drm_dev, &node->event->base);1293err:1294 g2d_put_cmdlist(g2d, node);1295 return ret;1296}1297 1298int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,1299 struct drm_file *file)1300{1301 struct drm_exynos_file_private *file_priv = file->driver_priv;1302 struct exynos_drm_private *priv = drm_dev->dev_private;1303 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);1304 struct drm_exynos_g2d_exec *req = data;1305 struct g2d_runqueue_node *runqueue_node;1306 struct list_head *run_cmdlist;1307 struct list_head *event_list;1308 1309 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);1310 if (!runqueue_node)1311 return -ENOMEM;1312 1313 run_cmdlist = &runqueue_node->run_cmdlist;1314 event_list = &runqueue_node->event_list;1315 INIT_LIST_HEAD(run_cmdlist);1316 INIT_LIST_HEAD(event_list);1317 init_completion(&runqueue_node->complete);1318 runqueue_node->async = req->async;1319 1320 list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist);1321 list_splice_init(&file_priv->event_list, event_list);1322 1323 if (list_empty(run_cmdlist)) {1324 dev_err(g2d->dev, "there is no inuse cmdlist\n");1325 kmem_cache_free(g2d->runqueue_slab, runqueue_node);1326 return -EPERM;1327 }1328 1329 mutex_lock(&g2d->runqueue_mutex);1330 runqueue_node->pid = current->pid;1331 runqueue_node->filp = file;1332 list_add_tail(&runqueue_node->list, &g2d->runqueue);1333 mutex_unlock(&g2d->runqueue_mutex);1334 1335 /* Let the runqueue know that there is work to do. */1336 queue_work(g2d->g2d_workq, &g2d->runqueue_work);1337 1338 if (req->async)1339 goto out;1340 1341 wait_for_completion(&runqueue_node->complete);1342 g2d_free_runqueue_node(g2d, runqueue_node);1343 1344out:1345 return 0;1346}1347 1348int g2d_open(struct drm_device *drm_dev, struct drm_file *file)1349{1350 struct drm_exynos_file_private *file_priv = file->driver_priv;1351 1352 INIT_LIST_HEAD(&file_priv->inuse_cmdlist);1353 INIT_LIST_HEAD(&file_priv->event_list);1354 INIT_LIST_HEAD(&file_priv->userptr_list);1355 1356 return 0;1357}1358 1359void g2d_close(struct drm_device *drm_dev, struct drm_file *file)1360{1361 struct drm_exynos_file_private *file_priv = file->driver_priv;1362 struct exynos_drm_private *priv = drm_dev->dev_private;1363 struct g2d_data *g2d;1364 struct g2d_cmdlist_node *node, *n;1365 1366 if (!priv->g2d_dev)1367 return;1368 1369 g2d = dev_get_drvdata(priv->g2d_dev);1370 1371 /* Remove the runqueue nodes that belong to us. */1372 mutex_lock(&g2d->runqueue_mutex);1373 g2d_remove_runqueue_nodes(g2d, file);1374 mutex_unlock(&g2d->runqueue_mutex);1375 1376 /*1377 * Wait for the runqueue worker to finish its current node.1378 * After this the engine should no longer be accessing any1379 * memory belonging to us.1380 */1381 g2d_wait_finish(g2d, file);1382 1383 /*1384 * Even after the engine is idle, there might still be stale cmdlists1385 * (i.e. cmdlisst which we submitted but never executed) around, with1386 * their corresponding GEM/userptr buffers.1387 * Properly unmap these buffers here.1388 */1389 mutex_lock(&g2d->cmdlist_mutex);1390 list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {1391 g2d_unmap_cmdlist_gem(g2d, node, file);1392 list_move_tail(&node->list, &g2d->free_cmdlist);1393 }1394 mutex_unlock(&g2d->cmdlist_mutex);1395 1396 /* release all g2d_userptr in pool. */1397 g2d_userptr_free_all(g2d, file);1398}1399 1400static int g2d_bind(struct device *dev, struct device *master, void *data)1401{1402 struct g2d_data *g2d = dev_get_drvdata(dev);1403 struct drm_device *drm_dev = data;1404 struct exynos_drm_private *priv = drm_dev->dev_private;1405 int ret;1406 1407 g2d->drm_dev = drm_dev;1408 1409 /* allocate dma-aware cmdlist buffer. */1410 ret = g2d_init_cmdlist(g2d);1411 if (ret < 0) {1412 dev_err(dev, "cmdlist init failed\n");1413 return ret;1414 }1415 1416 ret = exynos_drm_register_dma(drm_dev, dev, &g2d->dma_priv);1417 if (ret < 0) {1418 dev_err(dev, "failed to enable iommu.\n");1419 g2d_fini_cmdlist(g2d);1420 return ret;1421 }1422 priv->g2d_dev = dev;1423 1424 dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",1425 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);1426 return 0;1427}1428 1429static void g2d_unbind(struct device *dev, struct device *master, void *data)1430{1431 struct g2d_data *g2d = dev_get_drvdata(dev);1432 struct drm_device *drm_dev = data;1433 struct exynos_drm_private *priv = drm_dev->dev_private;1434 1435 /* Suspend operation and wait for engine idle. */1436 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);1437 g2d_wait_finish(g2d, NULL);1438 priv->g2d_dev = NULL;1439 1440 cancel_work_sync(&g2d->runqueue_work);1441 exynos_drm_unregister_dma(g2d->drm_dev, dev, &g2d->dma_priv);1442}1443 1444static const struct component_ops g2d_component_ops = {1445 .bind = g2d_bind,1446 .unbind = g2d_unbind,1447};1448 1449static int g2d_probe(struct platform_device *pdev)1450{1451 struct device *dev = &pdev->dev;1452 struct g2d_data *g2d;1453 int ret;1454 1455 g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL);1456 if (!g2d)1457 return -ENOMEM;1458 1459 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",1460 sizeof(struct g2d_runqueue_node), 0, 0, NULL);1461 if (!g2d->runqueue_slab)1462 return -ENOMEM;1463 1464 g2d->dev = dev;1465 1466 g2d->g2d_workq = create_singlethread_workqueue("g2d");1467 if (!g2d->g2d_workq) {1468 dev_err(dev, "failed to create workqueue\n");1469 ret = -EINVAL;1470 goto err_destroy_slab;1471 }1472 1473 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);1474 INIT_LIST_HEAD(&g2d->free_cmdlist);1475 INIT_LIST_HEAD(&g2d->runqueue);1476 1477 mutex_init(&g2d->cmdlist_mutex);1478 mutex_init(&g2d->runqueue_mutex);1479 1480 g2d->gate_clk = devm_clk_get(dev, "fimg2d");1481 if (IS_ERR(g2d->gate_clk)) {1482 dev_err(dev, "failed to get gate clock\n");1483 ret = PTR_ERR(g2d->gate_clk);1484 goto err_destroy_workqueue;1485 }1486 1487 pm_runtime_use_autosuspend(dev);1488 pm_runtime_set_autosuspend_delay(dev, 2000);1489 pm_runtime_enable(dev);1490 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);1491 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);1492 1493 g2d->regs = devm_platform_ioremap_resource(pdev, 0);1494 if (IS_ERR(g2d->regs)) {1495 ret = PTR_ERR(g2d->regs);1496 goto err_put_clk;1497 }1498 1499 g2d->irq = platform_get_irq(pdev, 0);1500 if (g2d->irq < 0) {1501 ret = g2d->irq;1502 goto err_put_clk;1503 }1504 1505 ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0,1506 "drm_g2d", g2d);1507 if (ret < 0) {1508 dev_err(dev, "irq request failed\n");1509 goto err_put_clk;1510 }1511 1512 g2d->max_pool = MAX_POOL;1513 1514 platform_set_drvdata(pdev, g2d);1515 1516 ret = component_add(dev, &g2d_component_ops);1517 if (ret < 0) {1518 dev_err(dev, "failed to register drm g2d device\n");1519 goto err_put_clk;1520 }1521 1522 return 0;1523 1524err_put_clk:1525 pm_runtime_disable(dev);1526err_destroy_workqueue:1527 destroy_workqueue(g2d->g2d_workq);1528err_destroy_slab:1529 kmem_cache_destroy(g2d->runqueue_slab);1530 return ret;1531}1532 1533static void g2d_remove(struct platform_device *pdev)1534{1535 struct g2d_data *g2d = platform_get_drvdata(pdev);1536 1537 component_del(&pdev->dev, &g2d_component_ops);1538 1539 /* There should be no locking needed here. */1540 g2d_remove_runqueue_nodes(g2d, NULL);1541 1542 pm_runtime_dont_use_autosuspend(&pdev->dev);1543 pm_runtime_disable(&pdev->dev);1544 1545 g2d_fini_cmdlist(g2d);1546 destroy_workqueue(g2d->g2d_workq);1547 kmem_cache_destroy(g2d->runqueue_slab);1548}1549 1550static int g2d_suspend(struct device *dev)1551{1552 struct g2d_data *g2d = dev_get_drvdata(dev);1553 1554 /*1555 * Suspend the runqueue worker operation and wait until the G2D1556 * engine is idle.1557 */1558 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);1559 g2d_wait_finish(g2d, NULL);1560 flush_work(&g2d->runqueue_work);1561 1562 return 0;1563}1564 1565static int g2d_resume(struct device *dev)1566{1567 struct g2d_data *g2d = dev_get_drvdata(dev);1568 1569 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);1570 queue_work(g2d->g2d_workq, &g2d->runqueue_work);1571 1572 return 0;1573}1574 1575static int g2d_runtime_suspend(struct device *dev)1576{1577 struct g2d_data *g2d = dev_get_drvdata(dev);1578 1579 clk_disable_unprepare(g2d->gate_clk);1580 1581 return 0;1582}1583 1584static int g2d_runtime_resume(struct device *dev)1585{1586 struct g2d_data *g2d = dev_get_drvdata(dev);1587 int ret;1588 1589 ret = clk_prepare_enable(g2d->gate_clk);1590 if (ret < 0)1591 dev_warn(dev, "failed to enable clock.\n");1592 1593 return ret;1594}1595 1596static const struct dev_pm_ops g2d_pm_ops = {1597 SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume)1598 RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL)1599};1600 1601static const struct of_device_id exynos_g2d_match[] = {1602 { .compatible = "samsung,exynos5250-g2d" },1603 { .compatible = "samsung,exynos4212-g2d" },1604 {},1605};1606MODULE_DEVICE_TABLE(of, exynos_g2d_match);1607 1608struct platform_driver g2d_driver = {1609 .probe = g2d_probe,1610 .remove_new = g2d_remove,1611 .driver = {1612 .name = "exynos-drm-g2d",1613 .pm = pm_ptr(&g2d_pm_ops),1614 .of_match_table = exynos_g2d_match,1615 },1616};1617