585 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Keystone accumulator queue manager4 *5 * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com6 * Author: Sandeep Nair <sandeep_n@ti.com>7 * Cyril Chemparathy <cyril@ti.com>8 * Santosh Shilimkar <santosh.shilimkar@ti.com>9 */10 11#include <linux/dma-mapping.h>12#include <linux/io.h>13#include <linux/interrupt.h>14#include <linux/module.h>15#include <linux/of_address.h>16#include <linux/soc/ti/knav_qmss.h>17 18#include "knav_qmss.h"19 20#define knav_range_offset_to_inst(kdev, range, q) \21 (range->queue_base_inst + (q << kdev->inst_shift))22 23static void __knav_acc_notify(struct knav_range_info *range,24 struct knav_acc_channel *acc)25{26 struct knav_device *kdev = range->kdev;27 struct knav_queue_inst *inst;28 int range_base, queue;29 30 range_base = kdev->base_id + range->queue_base;31 32 if (range->flags & RANGE_MULTI_QUEUE) {33 for (queue = 0; queue < range->num_queues; queue++) {34 inst = knav_range_offset_to_inst(kdev, range,35 queue);36 if (inst->notify_needed) {37 inst->notify_needed = 0;38 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",39 range_base + queue);40 knav_queue_notify(inst);41 }42 }43 } else {44 queue = acc->channel - range->acc_info.start_channel;45 inst = knav_range_offset_to_inst(kdev, range, queue);46 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",47 range_base + queue);48 knav_queue_notify(inst);49 }50}51 52static int knav_acc_set_notify(struct knav_range_info *range,53 struct knav_queue_inst *kq,54 bool enabled)55{56 struct knav_pdsp_info *pdsp = range->acc_info.pdsp;57 struct knav_device *kdev = range->kdev;58 u32 mask, offset;59 60 /*61 * when enabling, we need to re-trigger an interrupt if we62 * have descriptors pending63 */64 if (!enabled || atomic_read(&kq->desc_count) <= 0)65 return 0;66 67 kq->notify_needed = 1;68 atomic_inc(&kq->acc->retrigger_count);69 mask = BIT(kq->acc->channel % 32);70 offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);71 dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",72 kq->acc->name);73 writel_relaxed(mask, pdsp->intd + offset);74 return 0;75}76 77static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)78{79 struct knav_acc_channel *acc;80 struct knav_queue_inst *kq = NULL;81 struct knav_range_info *range;82 struct knav_pdsp_info *pdsp;83 struct knav_acc_info *info;84 struct knav_device *kdev;85 86 u32 *list, *list_cpu, val, idx, notifies;87 int range_base, channel, queue = 0;88 dma_addr_t list_dma;89 90 range = _instdata;91 info = &range->acc_info;92 kdev = range->kdev;93 pdsp = range->acc_info.pdsp;94 acc = range->acc;95 96 range_base = kdev->base_id + range->queue_base;97 if ((range->flags & RANGE_MULTI_QUEUE) == 0) {98 for (queue = 0; queue < range->num_irqs; queue++)99 if (range->irqs[queue].irq == irq)100 break;101 kq = knav_range_offset_to_inst(kdev, range, queue);102 acc += queue;103 }104 105 channel = acc->channel;106 list_dma = acc->list_dma[acc->list_index];107 list_cpu = acc->list_cpu[acc->list_index];108 dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",109 channel, acc->list_index, list_cpu, &list_dma);110 if (atomic_read(&acc->retrigger_count)) {111 atomic_dec(&acc->retrigger_count);112 __knav_acc_notify(range, acc);113 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));114 /* ack the interrupt */115 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,116 pdsp->intd + ACC_INTD_OFFSET_EOI);117 118 return IRQ_HANDLED;119 }120 121 notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));122 WARN_ON(!notifies);123 dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,124 DMA_FROM_DEVICE);125 126 for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));127 list += ACC_LIST_ENTRY_WORDS) {128 if (ACC_LIST_ENTRY_WORDS == 1) {129 dev_dbg(kdev->dev,130 "acc-irq: list %d, entry @%p, %08x\n",131 acc->list_index, list, list[0]);132 } else if (ACC_LIST_ENTRY_WORDS == 2) {133 dev_dbg(kdev->dev,134 "acc-irq: list %d, entry @%p, %08x %08x\n",135 acc->list_index, list, list[0], list[1]);136 } else if (ACC_LIST_ENTRY_WORDS == 4) {137 dev_dbg(kdev->dev,138 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",139 acc->list_index, list, list[0], list[1],140 list[2], list[3]);141 }142 143 val = list[ACC_LIST_ENTRY_DESC_IDX];144 if (!val)145 break;146 147 if (range->flags & RANGE_MULTI_QUEUE) {148 queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;149 if (queue < range_base ||150 queue >= range_base + range->num_queues) {151 dev_err(kdev->dev,152 "bad queue %d, expecting %d-%d\n",153 queue, range_base,154 range_base + range->num_queues);155 break;156 }157 queue -= range_base;158 kq = knav_range_offset_to_inst(kdev, range,159 queue);160 }161 162 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {163 atomic_dec(&kq->desc_count);164 dev_err(kdev->dev,165 "acc-irq: queue %d full, entry dropped\n",166 queue + range_base);167 continue;168 }169 170 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;171 kq->descs[idx] = val;172 kq->notify_needed = 1;173 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",174 val, idx, queue + range_base);175 }176 177 __knav_acc_notify(range, acc);178 memset(list_cpu, 0, info->list_size);179 dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,180 DMA_TO_DEVICE);181 182 /* flip to the other list */183 acc->list_index ^= 1;184 185 /* reset the interrupt counter */186 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));187 188 /* ack the interrupt */189 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,190 pdsp->intd + ACC_INTD_OFFSET_EOI);191 192 return IRQ_HANDLED;193}194 195static int knav_range_setup_acc_irq(struct knav_range_info *range,196 int queue, bool enabled)197{198 struct knav_device *kdev = range->kdev;199 struct knav_acc_channel *acc;200 struct cpumask *cpu_mask;201 int ret = 0, irq;202 u32 old, new;203 204 if (range->flags & RANGE_MULTI_QUEUE) {205 acc = range->acc;206 irq = range->irqs[0].irq;207 cpu_mask = range->irqs[0].cpu_mask;208 } else {209 acc = range->acc + queue;210 irq = range->irqs[queue].irq;211 cpu_mask = range->irqs[queue].cpu_mask;212 }213 214 old = acc->open_mask;215 if (enabled)216 new = old | BIT(queue);217 else218 new = old & ~BIT(queue);219 acc->open_mask = new;220 221 dev_dbg(kdev->dev,222 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",223 old, new, acc->name);224 225 if (likely(new == old))226 return 0;227 228 if (new && !old) {229 dev_dbg(kdev->dev,230 "setup-acc-irq: requesting %s for channel %s\n",231 acc->name, acc->name);232 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,233 range);234 if (!ret && cpu_mask) {235 ret = irq_set_affinity_hint(irq, cpu_mask);236 if (ret) {237 dev_warn(range->kdev->dev,238 "Failed to set IRQ affinity\n");239 return ret;240 }241 }242 }243 244 if (old && !new) {245 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",246 acc->name, acc->name);247 ret = irq_set_affinity_hint(irq, NULL);248 if (ret)249 dev_warn(range->kdev->dev,250 "Failed to set IRQ affinity\n");251 free_irq(irq, range);252 }253 254 return ret;255}256 257static const char *knav_acc_result_str(enum knav_acc_result result)258{259 static const char * const result_str[] = {260 [ACC_RET_IDLE] = "idle",261 [ACC_RET_SUCCESS] = "success",262 [ACC_RET_INVALID_COMMAND] = "invalid command",263 [ACC_RET_INVALID_CHANNEL] = "invalid channel",264 [ACC_RET_INACTIVE_CHANNEL] = "inactive channel",265 [ACC_RET_ACTIVE_CHANNEL] = "active channel",266 [ACC_RET_INVALID_QUEUE] = "invalid queue",267 [ACC_RET_INVALID_RET] = "invalid return code",268 };269 270 if (result >= ARRAY_SIZE(result_str))271 return result_str[ACC_RET_INVALID_RET];272 else273 return result_str[result];274}275 276static enum knav_acc_result277knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,278 struct knav_reg_acc_command *cmd)279{280 u32 result;281 282 dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",283 cmd->command, cmd->queue_mask, cmd->list_dma,284 cmd->queue_num, cmd->timer_config);285 286 writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);287 writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);288 writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);289 writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);290 writel_relaxed(cmd->command, &pdsp->acc_command->command);291 292 /* wait for the command to clear */293 do {294 result = readl_relaxed(&pdsp->acc_command->command);295 } while ((result >> 8) & 0xff);296 297 return (result >> 24) & 0xff;298}299 300static void knav_acc_setup_cmd(struct knav_device *kdev,301 struct knav_range_info *range,302 struct knav_reg_acc_command *cmd,303 int queue)304{305 struct knav_acc_info *info = &range->acc_info;306 struct knav_acc_channel *acc;307 int queue_base;308 u32 queue_mask;309 310 if (range->flags & RANGE_MULTI_QUEUE) {311 acc = range->acc;312 queue_base = range->queue_base;313 queue_mask = BIT(range->num_queues) - 1;314 } else {315 acc = range->acc + queue;316 queue_base = range->queue_base + queue;317 queue_mask = 0;318 }319 320 memset(cmd, 0, sizeof(*cmd));321 cmd->command = acc->channel;322 cmd->queue_mask = queue_mask;323 cmd->list_dma = (u32)acc->list_dma[0];324 cmd->queue_num = info->list_entries << 16;325 cmd->queue_num |= queue_base;326 327 cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;328 if (range->flags & RANGE_MULTI_QUEUE)329 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;330 cmd->timer_config |= info->pacing_mode << 16;331 cmd->timer_config |= info->timer_count;332}333 334static void knav_acc_stop(struct knav_device *kdev,335 struct knav_range_info *range,336 int queue)337{338 struct knav_reg_acc_command cmd;339 struct knav_acc_channel *acc;340 enum knav_acc_result result;341 342 acc = range->acc + queue;343 344 knav_acc_setup_cmd(kdev, range, &cmd, queue);345 cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;346 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);347 348 dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",349 acc->name, knav_acc_result_str(result));350}351 352static enum knav_acc_result knav_acc_start(struct knav_device *kdev,353 struct knav_range_info *range,354 int queue)355{356 struct knav_reg_acc_command cmd;357 struct knav_acc_channel *acc;358 enum knav_acc_result result;359 360 acc = range->acc + queue;361 362 knav_acc_setup_cmd(kdev, range, &cmd, queue);363 cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;364 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);365 366 dev_dbg(kdev->dev, "started acc channel %s, result %s\n",367 acc->name, knav_acc_result_str(result));368 369 return result;370}371 372static int knav_acc_init_range(struct knav_range_info *range)373{374 struct knav_device *kdev = range->kdev;375 struct knav_acc_channel *acc;376 enum knav_acc_result result;377 int queue;378 379 for (queue = 0; queue < range->num_queues; queue++) {380 acc = range->acc + queue;381 382 knav_acc_stop(kdev, range, queue);383 acc->list_index = 0;384 result = knav_acc_start(kdev, range, queue);385 386 if (result != ACC_RET_SUCCESS)387 return -EIO;388 389 if (range->flags & RANGE_MULTI_QUEUE)390 return 0;391 }392 return 0;393}394 395static int knav_acc_init_queue(struct knav_range_info *range,396 struct knav_queue_inst *kq)397{398 unsigned id = kq->id - range->queue_base;399 400 kq->descs = devm_kcalloc(range->kdev->dev,401 ACC_DESCS_MAX, sizeof(u32), GFP_KERNEL);402 if (!kq->descs)403 return -ENOMEM;404 405 kq->acc = range->acc;406 if ((range->flags & RANGE_MULTI_QUEUE) == 0)407 kq->acc += id;408 return 0;409}410 411static int knav_acc_open_queue(struct knav_range_info *range,412 struct knav_queue_inst *inst, unsigned flags)413{414 unsigned id = inst->id - range->queue_base;415 416 return knav_range_setup_acc_irq(range, id, true);417}418 419static int knav_acc_close_queue(struct knav_range_info *range,420 struct knav_queue_inst *inst)421{422 unsigned id = inst->id - range->queue_base;423 424 return knav_range_setup_acc_irq(range, id, false);425}426 427static int knav_acc_free_range(struct knav_range_info *range)428{429 struct knav_device *kdev = range->kdev;430 struct knav_acc_channel *acc;431 struct knav_acc_info *info;432 int channel, channels;433 434 info = &range->acc_info;435 436 if (range->flags & RANGE_MULTI_QUEUE)437 channels = 1;438 else439 channels = range->num_queues;440 441 for (channel = 0; channel < channels; channel++) {442 acc = range->acc + channel;443 if (!acc->list_cpu[0])444 continue;445 dma_unmap_single(kdev->dev, acc->list_dma[0],446 info->mem_size, DMA_BIDIRECTIONAL);447 free_pages_exact(acc->list_cpu[0], info->mem_size);448 }449 devm_kfree(range->kdev->dev, range->acc);450 return 0;451}452 453static const struct knav_range_ops knav_acc_range_ops = {454 .set_notify = knav_acc_set_notify,455 .init_queue = knav_acc_init_queue,456 .open_queue = knav_acc_open_queue,457 .close_queue = knav_acc_close_queue,458 .init_range = knav_acc_init_range,459 .free_range = knav_acc_free_range,460};461 462/**463 * knav_init_acc_range: Initialise accumulator ranges464 *465 * @kdev: qmss device466 * @node: device node467 * @range: qmms range information468 *469 * Return 0 on success or error470 */471int knav_init_acc_range(struct knav_device *kdev,472 struct device_node *node,473 struct knav_range_info *range)474{475 struct knav_acc_channel *acc;476 struct knav_pdsp_info *pdsp;477 struct knav_acc_info *info;478 int ret, channel, channels;479 int list_size, mem_size;480 dma_addr_t list_dma;481 void *list_mem;482 u32 config[5];483 484 range->flags |= RANGE_HAS_ACCUMULATOR;485 info = &range->acc_info;486 487 ret = of_property_read_u32_array(node, "accumulator", config, 5);488 if (ret)489 return ret;490 491 info->pdsp_id = config[0];492 info->start_channel = config[1];493 info->list_entries = config[2];494 info->pacing_mode = config[3];495 info->timer_count = config[4] / ACC_DEFAULT_PERIOD;496 497 if (info->start_channel > ACC_MAX_CHANNEL) {498 dev_err(kdev->dev, "channel %d invalid for range %s\n",499 info->start_channel, range->name);500 return -EINVAL;501 }502 503 if (info->pacing_mode > 3) {504 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",505 info->pacing_mode, range->name);506 return -EINVAL;507 }508 509 pdsp = knav_find_pdsp(kdev, info->pdsp_id);510 if (!pdsp) {511 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",512 info->pdsp_id, range->name);513 return -EINVAL;514 }515 516 if (!pdsp->started) {517 dev_err(kdev->dev, "pdsp id %d not started for range %s\n",518 info->pdsp_id, range->name);519 return -ENODEV;520 }521 522 info->pdsp = pdsp;523 channels = range->num_queues;524 if (of_property_read_bool(node, "multi-queue")) {525 range->flags |= RANGE_MULTI_QUEUE;526 channels = 1;527 if (range->queue_base & (32 - 1)) {528 dev_err(kdev->dev,529 "misaligned multi-queue accumulator range %s\n",530 range->name);531 return -EINVAL;532 }533 if (range->num_queues > 32) {534 dev_err(kdev->dev,535 "too many queues in accumulator range %s\n",536 range->name);537 return -EINVAL;538 }539 }540 541 /* figure out list size */542 list_size = info->list_entries;543 list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);544 info->list_size = list_size;545 mem_size = PAGE_ALIGN(list_size * 2);546 info->mem_size = mem_size;547 range->acc = devm_kcalloc(kdev->dev, channels, sizeof(*range->acc),548 GFP_KERNEL);549 if (!range->acc)550 return -ENOMEM;551 552 for (channel = 0; channel < channels; channel++) {553 acc = range->acc + channel;554 acc->channel = info->start_channel + channel;555 556 /* allocate memory for the two lists */557 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);558 if (!list_mem)559 return -ENOMEM;560 561 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,562 DMA_BIDIRECTIONAL);563 if (dma_mapping_error(kdev->dev, list_dma)) {564 free_pages_exact(list_mem, mem_size);565 return -ENOMEM;566 }567 568 memset(list_mem, 0, mem_size);569 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,570 DMA_TO_DEVICE);571 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",572 acc->channel);573 acc->list_cpu[0] = list_mem;574 acc->list_cpu[1] = list_mem + list_size;575 acc->list_dma[0] = list_dma;576 acc->list_dma[1] = list_dma + list_size;577 dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",578 acc->name, acc->channel, &list_dma, list_mem);579 }580 581 range->ops = &knav_acc_range_ops;582 return 0;583}584EXPORT_SYMBOL_GPL(knav_init_acc_range);585