1155 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/*4 * Copyright 2016-2019 HabanaLabs, Ltd.5 * All Rights Reserved.6 */7 8#include "habanalabs.h"9 10#include <linux/slab.h>11 12/*13 * hl_queue_add_ptr - add to pi or ci and checks if it wraps around14 *15 * @ptr: the current pi/ci value16 * @val: the amount to add17 *18 * Add val to ptr. It can go until twice the queue length.19 */20inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)21{22 ptr += val;23 ptr &= ((HL_QUEUE_LENGTH << 1) - 1);24 return ptr;25}26static inline int queue_ci_get(atomic_t *ci, u32 queue_len)27{28 return atomic_read(ci) & ((queue_len << 1) - 1);29}30 31static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)32{33 int delta = (q->pi - queue_ci_get(&q->ci, queue_len));34 35 if (delta >= 0)36 return (queue_len - delta);37 else38 return (abs(delta) - queue_len);39}40 41void hl_hw_queue_update_ci(struct hl_cs *cs)42{43 struct hl_device *hdev = cs->ctx->hdev;44 struct hl_hw_queue *q;45 int i;46 47 if (hdev->disabled)48 return;49 50 q = &hdev->kernel_queues[0];51 52 /* There are no internal queues if H/W queues are being used */53 if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)54 return;55 56 /* We must increment CI for every queue that will never get a57 * completion, there are 2 scenarios this can happen:58 * 1. All queues of a non completion CS will never get a completion.59 * 2. Internal queues never gets completion.60 */61 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {62 if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT)63 atomic_add(cs->jobs_in_queue_cnt[i], &q->ci);64 }65}66 67/*68 * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a69 * H/W queue.70 * @hdev: pointer to habanalabs device structure71 * @q: pointer to habanalabs queue structure72 * @ctl: BD's control word73 * @len: BD's length74 * @ptr: BD's pointer75 *76 * This function assumes there is enough space on the queue to submit a new77 * BD to it. It initializes the next BD and calls the device specific78 * function to set the pi (and doorbell)79 *80 * This function must be called when the scheduler mutex is taken81 *82 */83void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,84 u32 ctl, u32 len, u64 ptr)85{86 struct hl_bd *bd;87 u64 addr;88 int i;89 90 bd = q->kernel_address;91 bd += hl_pi_2_offset(q->pi);92 bd->ctl = cpu_to_le32(ctl);93 bd->len = cpu_to_le32(len);94 bd->ptr = cpu_to_le64(ptr);95 96 if (q->dram_bd)97 for (i = 0 ; i < 2 ; i++) {98 addr = q->pq_dram_address +99 ((hl_pi_2_offset(q->pi) * sizeof(struct hl_bd)) + (i * sizeof(u64)));100 hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM, addr,101 (u64 *)(bd) + i, DEBUGFS_WRITE64);102 }103 104 q->pi = hl_queue_inc_ptr(q->pi);105 106 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);107}108 109/*110 * ext_queue_sanity_checks - perform some sanity checks on external queue111 *112 * @hdev : pointer to hl_device structure113 * @q : pointer to hl_hw_queue structure114 * @num_of_entries : how many entries to check for space115 * @reserve_cq_entry : whether to reserve an entry in the cq116 *117 * H/W queues spinlock should be taken before calling this function118 *119 * Perform the following:120 * - Make sure we have enough space in the h/w queue121 * - Make sure we have enough space in the completion queue122 * - Reserve space in the completion queue (needs to be reversed if there123 * is a failure down the road before the actual submission of work). Only124 * do this action if reserve_cq_entry is true125 *126 */127static int ext_queue_sanity_checks(struct hl_device *hdev,128 struct hl_hw_queue *q, int num_of_entries,129 bool reserve_cq_entry)130{131 atomic_t *free_slots =132 &hdev->completion_queue[q->cq_id].free_slots_cnt;133 int free_slots_cnt;134 135 /* Check we have enough space in the queue */136 free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);137 138 if (free_slots_cnt < num_of_entries) {139 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",140 q->hw_queue_id, num_of_entries);141 return -EAGAIN;142 }143 144 if (reserve_cq_entry) {145 /*146 * Check we have enough space in the completion queue147 * Add -1 to counter (decrement) unless counter was already 0148 * In that case, CQ is full so we can't submit a new CB because149 * we won't get ack on its completion150 * atomic_add_unless will return 0 if counter was already 0151 */152 if (atomic_add_negative(num_of_entries * -1, free_slots)) {153 dev_dbg(hdev->dev, "No space for %d on CQ %d\n",154 num_of_entries, q->hw_queue_id);155 atomic_add(num_of_entries, free_slots);156 return -EAGAIN;157 }158 }159 160 return 0;161}162 163/*164 * int_queue_sanity_checks - perform some sanity checks on internal queue165 *166 * @hdev : pointer to hl_device structure167 * @q : pointer to hl_hw_queue structure168 * @num_of_entries : how many entries to check for space169 *170 * H/W queues spinlock should be taken before calling this function171 *172 * Perform the following:173 * - Make sure we have enough space in the h/w queue174 *175 */176static int int_queue_sanity_checks(struct hl_device *hdev,177 struct hl_hw_queue *q,178 int num_of_entries)179{180 int free_slots_cnt;181 182 if (num_of_entries > q->int_queue_len) {183 dev_err(hdev->dev,184 "Cannot populate queue %u with %u jobs\n",185 q->hw_queue_id, num_of_entries);186 return -ENOMEM;187 }188 189 /* Check we have enough space in the queue */190 free_slots_cnt = queue_free_slots(q, q->int_queue_len);191 192 if (free_slots_cnt < num_of_entries) {193 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",194 q->hw_queue_id, num_of_entries);195 return -EAGAIN;196 }197 198 return 0;199}200 201/*202 * hw_queue_sanity_checks() - Make sure we have enough space in the h/w queue203 * @hdev: Pointer to hl_device structure.204 * @q: Pointer to hl_hw_queue structure.205 * @num_of_entries: How many entries to check for space.206 *207 * Notice: We do not reserve queue entries so this function mustn't be called208 * more than once per CS for the same queue209 *210 */211static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,212 int num_of_entries)213{214 int free_slots_cnt;215 216 /* Check we have enough space in the queue */217 free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);218 219 if (free_slots_cnt < num_of_entries) {220 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",221 q->hw_queue_id, num_of_entries);222 return -EAGAIN;223 }224 225 return 0;226}227 228/*229 * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion230 *231 * @hdev: pointer to hl_device structure232 * @hw_queue_id: Queue's type233 * @cb_size: size of CB234 * @cb_ptr: pointer to CB location235 *236 * This function sends a single CB, that must NOT generate a completion entry.237 * Sending CPU messages can be done instead via 'hl_hw_queue_submit_bd()'238 */239int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,240 u32 cb_size, u64 cb_ptr)241{242 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];243 int rc = 0;244 245 hdev->asic_funcs->hw_queues_lock(hdev);246 247 if (hdev->disabled) {248 rc = -EPERM;249 goto out;250 }251 252 /*253 * hl_hw_queue_send_cb_no_cmpl() is called for queues of a H/W queue254 * type only on init phase, when the queues are empty and being tested,255 * so there is no need for sanity checks.256 */257 if (q->queue_type != QUEUE_TYPE_HW) {258 rc = ext_queue_sanity_checks(hdev, q, 1, false);259 if (rc)260 goto out;261 }262 263 hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);264 265out:266 hdev->asic_funcs->hw_queues_unlock(hdev);267 268 return rc;269}270 271/*272 * ext_queue_schedule_job - submit a JOB to an external queue273 *274 * @job: pointer to the job that needs to be submitted to the queue275 *276 * This function must be called when the scheduler mutex is taken277 *278 */279static void ext_queue_schedule_job(struct hl_cs_job *job)280{281 struct hl_device *hdev = job->cs->ctx->hdev;282 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];283 struct hl_cq_entry cq_pkt;284 struct hl_cq *cq;285 u64 cq_addr;286 struct hl_cb *cb;287 u32 ctl;288 u32 len;289 u64 ptr;290 291 /*292 * Update the JOB ID inside the BD CTL so the device would know what293 * to write in the completion queue294 */295 ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK);296 297 cb = job->patched_cb;298 len = job->job_cb_size;299 ptr = cb->bus_address;300 301 /* Skip completion flow in case this is a non completion CS */302 if (!cs_needs_completion(job->cs))303 goto submit_bd;304 305 cq_pkt.data = cpu_to_le32(306 ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)307 & CQ_ENTRY_SHADOW_INDEX_MASK) |308 FIELD_PREP(CQ_ENTRY_SHADOW_INDEX_VALID_MASK, 1) |309 FIELD_PREP(CQ_ENTRY_READY_MASK, 1));310 311 /*312 * No need to protect pi_offset because scheduling to the313 * H/W queues is done under the scheduler mutex314 *315 * No need to check if CQ is full because it was already316 * checked in ext_queue_sanity_checks317 */318 cq = &hdev->completion_queue[q->cq_id];319 cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry);320 321 hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,322 job->user_cb_size,323 cq_addr,324 le32_to_cpu(cq_pkt.data),325 q->msi_vec,326 job->contains_dma_pkt);327 328 q->shadow_queue[hl_pi_2_offset(q->pi)] = job;329 330 cq->pi = hl_cq_inc_ptr(cq->pi);331 332submit_bd:333 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);334}335 336/*337 * int_queue_schedule_job - submit a JOB to an internal queue338 *339 * @job: pointer to the job that needs to be submitted to the queue340 *341 * This function must be called when the scheduler mutex is taken342 *343 */344static void int_queue_schedule_job(struct hl_cs_job *job)345{346 struct hl_device *hdev = job->cs->ctx->hdev;347 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];348 struct hl_bd bd;349 __le64 *pi;350 351 bd.ctl = 0;352 bd.len = cpu_to_le32(job->job_cb_size);353 354 if (job->is_kernel_allocated_cb)355 /* bus_address is actually a mmu mapped address356 * allocated from an internal pool357 */358 bd.ptr = cpu_to_le64(job->user_cb->bus_address);359 else360 bd.ptr = cpu_to_le64((u64) (uintptr_t) job->user_cb);361 362 pi = q->kernel_address + (q->pi & (q->int_queue_len - 1)) * sizeof(bd);363 364 q->pi++;365 q->pi &= ((q->int_queue_len << 1) - 1);366 367 hdev->asic_funcs->pqe_write(hdev, pi, &bd);368 369 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);370}371 372/*373 * hw_queue_schedule_job - submit a JOB to a H/W queue374 *375 * @job: pointer to the job that needs to be submitted to the queue376 *377 * This function must be called when the scheduler mutex is taken378 *379 */380static void hw_queue_schedule_job(struct hl_cs_job *job)381{382 struct hl_device *hdev = job->cs->ctx->hdev;383 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];384 u64 ptr;385 u32 offset, ctl, len;386 387 /*388 * Upon PQE completion, COMP_DATA is used as the write data to the389 * completion queue (QMAN HBW message), and COMP_OFFSET is used as the390 * write address offset in the SM block (QMAN LBW message).391 * The write address offset is calculated as "COMP_OFFSET << 2".392 */393 offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1);394 ctl = ((offset << BD_CTL_COMP_OFFSET_SHIFT) & BD_CTL_COMP_OFFSET_MASK) |395 ((q->pi << BD_CTL_COMP_DATA_SHIFT) & BD_CTL_COMP_DATA_MASK);396 397 len = job->job_cb_size;398 399 /*400 * A patched CB is created only if a user CB was allocated by driver and401 * MMU is disabled. If MMU is enabled, the user CB should be used402 * instead. If the user CB wasn't allocated by driver, assume that it403 * holds an address.404 */405 if (job->patched_cb)406 ptr = job->patched_cb->bus_address;407 else if (job->is_kernel_allocated_cb)408 ptr = job->user_cb->bus_address;409 else410 ptr = (u64) (uintptr_t) job->user_cb;411 412 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);413}414 415static int init_signal_cs(struct hl_device *hdev,416 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)417{418 struct hl_sync_stream_properties *prop;419 struct hl_hw_sob *hw_sob;420 u32 q_idx;421 int rc = 0;422 423 q_idx = job->hw_queue_id;424 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;425 hw_sob = &prop->hw_sob[prop->curr_sob_offset];426 427 cs_cmpl->hw_sob = hw_sob;428 cs_cmpl->sob_val = prop->next_sob_val;429 430 dev_dbg(hdev->dev,431 "generate signal CB, sob_id: %d, sob val: %u, q_idx: %d, seq: %llu\n",432 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx,433 cs_cmpl->cs_seq);434 435 /* we set an EB since we must make sure all oeprations are done436 * when sending the signal437 */438 hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,439 cs_cmpl->hw_sob->sob_id, 0, true);440 441 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1,442 false);443 444 job->cs->sob_addr_offset = hw_sob->sob_addr;445 job->cs->initial_sob_count = prop->next_sob_val - 1;446 447 return rc;448}449 450void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,451 struct hl_cs *cs, struct hl_cs_job *job,452 struct hl_cs_compl *cs_cmpl)453{454 struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl;455 u32 offset = 0;456 457 cs_cmpl->hw_sob = handle->hw_sob;458 459 /* Note that encaps_sig_wait_offset was validated earlier in the flow460 * for offset value which exceeds the max reserved signal count.461 * always decrement 1 of the offset since when the user462 * set offset 1 for example he mean to wait only for the first463 * signal only, which will be pre_sob_val, and if he set offset 2464 * then the value required is (pre_sob_val + 1) and so on...465 * if user set wait offset to 0, then treat it as legacy wait cs,466 * wait for the next signal.467 */468 if (job->encaps_sig_wait_offset)469 offset = job->encaps_sig_wait_offset - 1;470 471 cs_cmpl->sob_val = handle->pre_sob_val + offset;472}473 474static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,475 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)476{477 struct hl_gen_wait_properties wait_prop;478 struct hl_sync_stream_properties *prop;479 struct hl_cs_compl *signal_cs_cmpl;480 u32 q_idx;481 482 q_idx = job->hw_queue_id;483 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;484 485 signal_cs_cmpl = container_of(cs->signal_fence,486 struct hl_cs_compl,487 base_fence);488 489 if (cs->encaps_signals) {490 /* use the encaps signal handle stored earlier in the flow491 * and set the SOB information from the encaps492 * signals handle493 */494 hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl);495 496 dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, offset: %u\n",497 cs->encaps_sig_hdl->q_idx,498 cs->encaps_sig_hdl->cs_seq,499 cs_cmpl->sob_val,500 job->encaps_sig_wait_offset);501 } else {502 /* Copy the SOB id and value of the signal CS */503 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;504 cs_cmpl->sob_val = signal_cs_cmpl->sob_val;505 }506 507 /* check again if the signal cs already completed.508 * if yes then don't send any wait cs since the hw_sob509 * could be in reset already. if signal is not completed510 * then get refcount to hw_sob to prevent resetting the sob511 * while wait cs is not submitted.512 * note that this check is protected by two locks,513 * hw queue lock and completion object lock,514 * and the same completion object lock also protects515 * the hw_sob reset handler function.516 * The hw_queue lock prevent out of sync of hw_sob517 * refcount value, changed by signal/wait flows.518 */519 spin_lock(&signal_cs_cmpl->lock);520 521 if (completion_done(&cs->signal_fence->completion)) {522 spin_unlock(&signal_cs_cmpl->lock);523 return -EINVAL;524 }525 526 kref_get(&cs_cmpl->hw_sob->kref);527 528 spin_unlock(&signal_cs_cmpl->lock);529 530 dev_dbg(hdev->dev,531 "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d, seq: %llu\n",532 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,533 prop->base_mon_id, q_idx, cs->sequence);534 535 wait_prop.data = (void *) job->patched_cb;536 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;537 wait_prop.sob_mask = 0x1;538 wait_prop.sob_val = cs_cmpl->sob_val;539 wait_prop.mon_id = prop->base_mon_id;540 wait_prop.q_idx = q_idx;541 wait_prop.size = 0;542 543 hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);544 545 mb();546 hl_fence_put(cs->signal_fence);547 cs->signal_fence = NULL;548 549 return 0;550}551 552/*553 * init_signal_wait_cs - initialize a signal/wait CS554 * @cs: pointer to the signal/wait CS555 *556 * H/W queues spinlock should be taken before calling this function557 */558static int init_signal_wait_cs(struct hl_cs *cs)559{560 struct hl_ctx *ctx = cs->ctx;561 struct hl_device *hdev = ctx->hdev;562 struct hl_cs_job *job;563 struct hl_cs_compl *cs_cmpl =564 container_of(cs->fence, struct hl_cs_compl, base_fence);565 int rc = 0;566 567 /* There is only one job in a signal/wait CS */568 job = list_first_entry(&cs->job_list, struct hl_cs_job,569 cs_node);570 571 if (cs->type & CS_TYPE_SIGNAL)572 rc = init_signal_cs(hdev, job, cs_cmpl);573 else if (cs->type & CS_TYPE_WAIT)574 rc = init_wait_cs(hdev, cs, job, cs_cmpl);575 576 return rc;577}578 579static int encaps_sig_first_staged_cs_handler580 (struct hl_device *hdev, struct hl_cs *cs)581{582 struct hl_cs_compl *cs_cmpl =583 container_of(cs->fence,584 struct hl_cs_compl, base_fence);585 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;586 struct hl_encaps_signals_mgr *mgr;587 int rc = 0;588 589 mgr = &cs->ctx->sig_mgr;590 591 spin_lock(&mgr->lock);592 encaps_sig_hdl = idr_find(&mgr->handles, cs->encaps_sig_hdl_id);593 if (encaps_sig_hdl) {594 /*595 * Set handler CS sequence,596 * the CS which contains the encapsulated signals.597 */598 encaps_sig_hdl->cs_seq = cs->sequence;599 /* store the handle and set encaps signal indication,600 * to be used later in cs_do_release to put the last601 * reference to encaps signals handlers.602 */603 cs_cmpl->encaps_signals = true;604 cs_cmpl->encaps_sig_hdl = encaps_sig_hdl;605 606 /* set hw_sob pointer in completion object607 * since it's used in cs_do_release flow to put608 * refcount to sob609 */610 cs_cmpl->hw_sob = encaps_sig_hdl->hw_sob;611 cs_cmpl->sob_val = encaps_sig_hdl->pre_sob_val +612 encaps_sig_hdl->count;613 614 dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob(%u), val(%u)\n",615 cs->sequence, encaps_sig_hdl->id,616 encaps_sig_hdl->count,617 encaps_sig_hdl->q_idx,618 cs_cmpl->hw_sob->sob_id,619 cs_cmpl->sob_val);620 621 } else {622 dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n",623 cs->encaps_sig_hdl_id);624 rc = -EINVAL;625 }626 627 spin_unlock(&mgr->lock);628 629 return rc;630}631 632/*633 * hl_hw_queue_schedule_cs - schedule a command submission634 * @cs: pointer to the CS635 */636int hl_hw_queue_schedule_cs(struct hl_cs *cs)637{638 enum hl_device_status status;639 struct hl_cs_counters_atomic *cntr;640 struct hl_ctx *ctx = cs->ctx;641 struct hl_device *hdev = ctx->hdev;642 struct hl_cs_job *job, *tmp;643 struct hl_hw_queue *q;644 int rc = 0, i, cq_cnt;645 bool first_entry;646 u32 max_queues;647 648 cntr = &hdev->aggregated_cs_counters;649 650 hdev->asic_funcs->hw_queues_lock(hdev);651 652 if (!hl_device_operational(hdev, &status)) {653 atomic64_inc(&cntr->device_in_reset_drop_cnt);654 atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt);655 dev_err(hdev->dev,656 "device is %s, CS rejected!\n", hdev->status[status]);657 rc = -EPERM;658 goto out;659 }660 661 max_queues = hdev->asic_prop.max_queues;662 663 q = &hdev->kernel_queues[0];664 for (i = 0, cq_cnt = 0 ; i < max_queues ; i++, q++) {665 if (cs->jobs_in_queue_cnt[i]) {666 switch (q->queue_type) {667 case QUEUE_TYPE_EXT:668 rc = ext_queue_sanity_checks(hdev, q,669 cs->jobs_in_queue_cnt[i],670 cs_needs_completion(cs) ?671 true : false);672 break;673 case QUEUE_TYPE_INT:674 rc = int_queue_sanity_checks(hdev, q,675 cs->jobs_in_queue_cnt[i]);676 break;677 case QUEUE_TYPE_HW:678 rc = hw_queue_sanity_checks(hdev, q,679 cs->jobs_in_queue_cnt[i]);680 break;681 default:682 dev_err(hdev->dev, "Queue type %d is invalid\n",683 q->queue_type);684 rc = -EINVAL;685 break;686 }687 688 if (rc) {689 atomic64_inc(690 &ctx->cs_counters.queue_full_drop_cnt);691 atomic64_inc(&cntr->queue_full_drop_cnt);692 goto unroll_cq_resv;693 }694 695 if (q->queue_type == QUEUE_TYPE_EXT)696 cq_cnt++;697 }698 }699 700 if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) {701 rc = init_signal_wait_cs(cs);702 if (rc)703 goto unroll_cq_resv;704 } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) {705 rc = hdev->asic_funcs->collective_wait_init_cs(cs);706 if (rc)707 goto unroll_cq_resv;708 }709 710 rc = hdev->asic_funcs->pre_schedule_cs(cs);711 if (rc) {712 dev_err(hdev->dev,713 "Failed in pre-submission operations of CS %d.%llu\n",714 ctx->asid, cs->sequence);715 goto unroll_cq_resv;716 }717 718 hdev->shadow_cs_queue[cs->sequence &719 (hdev->asic_prop.max_pending_cs - 1)] = cs;720 721 if (cs->encaps_signals && cs->staged_first) {722 rc = encaps_sig_first_staged_cs_handler(hdev, cs);723 if (rc)724 goto unroll_cq_resv;725 }726 727 spin_lock(&hdev->cs_mirror_lock);728 729 /* Verify staged CS exists and add to the staged list */730 if (cs->staged_cs && !cs->staged_first) {731 struct hl_cs *staged_cs;732 733 staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);734 if (!staged_cs) {735 dev_err(hdev->dev,736 "Cannot find staged submission sequence %llu",737 cs->staged_sequence);738 rc = -EINVAL;739 goto unlock_cs_mirror;740 }741 742 if (is_staged_cs_last_exists(hdev, staged_cs)) {743 dev_err(hdev->dev,744 "Staged submission sequence %llu already submitted",745 cs->staged_sequence);746 rc = -EINVAL;747 goto unlock_cs_mirror;748 }749 750 list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);751 752 /* update stream map of the first CS */753 if (hdev->supports_wait_for_multi_cs)754 staged_cs->fence->stream_master_qid_map |=755 cs->fence->stream_master_qid_map;756 }757 758 list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);759 760 /* Queue TDR if the CS is the first entry and if timeout is wanted */761 first_entry = list_first_entry(&hdev->cs_mirror_list,762 struct hl_cs, mirror_node) == cs;763 if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&764 first_entry && cs_needs_timeout(cs)) {765 cs->tdr_active = true;766 schedule_delayed_work(&cs->work_tdr, cs->timeout_jiffies);767 768 }769 770 spin_unlock(&hdev->cs_mirror_lock);771 772 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)773 switch (job->queue_type) {774 case QUEUE_TYPE_EXT:775 ext_queue_schedule_job(job);776 break;777 case QUEUE_TYPE_INT:778 int_queue_schedule_job(job);779 break;780 case QUEUE_TYPE_HW:781 hw_queue_schedule_job(job);782 break;783 default:784 break;785 }786 787 cs->submitted = true;788 789 goto out;790 791unlock_cs_mirror:792 spin_unlock(&hdev->cs_mirror_lock);793unroll_cq_resv:794 q = &hdev->kernel_queues[0];795 for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) {796 if ((q->queue_type == QUEUE_TYPE_EXT) &&797 (cs->jobs_in_queue_cnt[i])) {798 atomic_t *free_slots =799 &hdev->completion_queue[i].free_slots_cnt;800 atomic_add(cs->jobs_in_queue_cnt[i], free_slots);801 cq_cnt--;802 }803 }804 805out:806 hdev->asic_funcs->hw_queues_unlock(hdev);807 808 return rc;809}810 811/*812 * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue813 *814 * @hdev: pointer to hl_device structure815 * @hw_queue_id: which queue to increment its ci816 */817void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)818{819 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];820 821 atomic_inc(&q->ci);822}823 824static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,825 bool is_cpu_queue)826{827 void *p;828 int rc;829 830 if (is_cpu_queue)831 p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address);832 else833 p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,834 GFP_KERNEL | __GFP_ZERO);835 if (!p)836 return -ENOMEM;837 838 q->kernel_address = p;839 840 q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH, sizeof(struct hl_cs_job *), GFP_KERNEL);841 if (!q->shadow_queue) {842 dev_err(hdev->dev,843 "Failed to allocate shadow queue for H/W queue %d\n",844 q->hw_queue_id);845 rc = -ENOMEM;846 goto free_queue;847 }848 849 /* Make sure read/write pointers are initialized to start of queue */850 atomic_set(&q->ci, 0);851 q->pi = 0;852 853 return 0;854 855free_queue:856 if (is_cpu_queue)857 hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);858 else859 hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,860 q->bus_address);861 862 return rc;863}864 865static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)866{867 void *p;868 869 p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,870 &q->bus_address, &q->int_queue_len);871 if (!p) {872 dev_err(hdev->dev,873 "Failed to get base address for internal queue %d\n",874 q->hw_queue_id);875 return -EFAULT;876 }877 878 q->kernel_address = p;879 q->pi = 0;880 atomic_set(&q->ci, 0);881 882 return 0;883}884 885static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)886{887 return ext_and_cpu_queue_init(hdev, q, true);888}889 890static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)891{892 return ext_and_cpu_queue_init(hdev, q, false);893}894 895static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)896{897 void *p;898 899 p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,900 GFP_KERNEL | __GFP_ZERO);901 if (!p)902 return -ENOMEM;903 904 q->kernel_address = p;905 906 /* Make sure read/write pointers are initialized to start of queue */907 atomic_set(&q->ci, 0);908 q->pi = 0;909 910 return 0;911}912 913static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx)914{915 struct hl_sync_stream_properties *sync_stream_prop;916 struct asic_fixed_properties *prop = &hdev->asic_prop;917 struct hl_hw_sob *hw_sob;918 int sob, reserved_mon_idx, queue_idx;919 920 sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop;921 922 /* We use 'collective_mon_idx' as a running index in order to reserve923 * monitors for collective master/slave queues.924 * collective master queue gets 2 reserved monitors925 * collective slave queue gets 1 reserved monitor926 */927 if (hdev->kernel_queues[q_idx].collective_mode ==928 HL_COLLECTIVE_MASTER) {929 reserved_mon_idx = hdev->collective_mon_idx;930 931 /* reserve the first monitor for collective master queue */932 sync_stream_prop->collective_mstr_mon_id[0] =933 prop->collective_first_mon + reserved_mon_idx;934 935 /* reserve the second monitor for collective master queue */936 sync_stream_prop->collective_mstr_mon_id[1] =937 prop->collective_first_mon + reserved_mon_idx + 1;938 939 hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS;940 } else if (hdev->kernel_queues[q_idx].collective_mode ==941 HL_COLLECTIVE_SLAVE) {942 reserved_mon_idx = hdev->collective_mon_idx++;943 944 /* reserve a monitor for collective slave queue */945 sync_stream_prop->collective_slave_mon_id =946 prop->collective_first_mon + reserved_mon_idx;947 }948 949 if (!hdev->kernel_queues[q_idx].supports_sync_stream)950 return;951 952 queue_idx = hdev->sync_stream_queue_idx++;953 954 sync_stream_prop->base_sob_id = prop->sync_stream_first_sob +955 (queue_idx * HL_RSVD_SOBS);956 sync_stream_prop->base_mon_id = prop->sync_stream_first_mon +957 (queue_idx * HL_RSVD_MONS);958 sync_stream_prop->next_sob_val = 1;959 sync_stream_prop->curr_sob_offset = 0;960 961 for (sob = 0 ; sob < HL_RSVD_SOBS ; sob++) {962 hw_sob = &sync_stream_prop->hw_sob[sob];963 hw_sob->hdev = hdev;964 hw_sob->sob_id = sync_stream_prop->base_sob_id + sob;965 hw_sob->sob_addr =966 hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);967 hw_sob->q_idx = q_idx;968 kref_init(&hw_sob->kref);969 }970}971 972static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx)973{974 struct hl_sync_stream_properties *prop =975 &hdev->kernel_queues[q_idx].sync_stream_prop;976 977 /*978 * In case we got here due to a stuck CS, the refcnt might be bigger979 * than 1 and therefore we reset it.980 */981 kref_init(&prop->hw_sob[prop->curr_sob_offset].kref);982 prop->curr_sob_offset = 0;983 prop->next_sob_val = 1;984}985 986/*987 * queue_init - main initialization function for H/W queue object988 *989 * @hdev: pointer to hl_device device structure990 * @q: pointer to hl_hw_queue queue structure991 * @hw_queue_id: The id of the H/W queue992 *993 * Allocate dma-able memory for the queue and initialize fields994 * Returns 0 on success995 */996static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,997 u32 hw_queue_id)998{999 int rc;1000 1001 q->hw_queue_id = hw_queue_id;1002 1003 switch (q->queue_type) {1004 case QUEUE_TYPE_EXT:1005 rc = ext_queue_init(hdev, q);1006 break;1007 case QUEUE_TYPE_INT:1008 rc = int_queue_init(hdev, q);1009 break;1010 case QUEUE_TYPE_CPU:1011 rc = cpu_queue_init(hdev, q);1012 break;1013 case QUEUE_TYPE_HW:1014 rc = hw_queue_init(hdev, q);1015 break;1016 case QUEUE_TYPE_NA:1017 q->valid = 0;1018 return 0;1019 default:1020 dev_crit(hdev->dev, "wrong queue type %d during init\n",1021 q->queue_type);1022 rc = -EINVAL;1023 break;1024 }1025 1026 sync_stream_queue_init(hdev, q->hw_queue_id);1027 1028 if (rc)1029 return rc;1030 1031 q->valid = 1;1032 1033 return 0;1034}1035 1036/*1037 * hw_queue_fini - destroy queue1038 *1039 * @hdev: pointer to hl_device device structure1040 * @q: pointer to hl_hw_queue queue structure1041 *1042 * Free the queue memory1043 */1044static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)1045{1046 if (!q->valid)1047 return;1048 1049 /*1050 * If we arrived here, there are no jobs waiting on this queue1051 * so we can safely remove it.1052 * This is because this function can only called when:1053 * 1. Either a context is deleted, which only can occur if all its1054 * jobs were finished1055 * 2. A context wasn't able to be created due to failure or timeout,1056 * which means there are no jobs on the queue yet1057 *1058 * The only exception are the queues of the kernel context, but1059 * if they are being destroyed, it means that the entire module is1060 * being removed. If the module is removed, it means there is no open1061 * user context. It also means that if a job was submitted by1062 * the kernel driver (e.g. context creation), the job itself was1063 * released by the kernel driver when a timeout occurred on its1064 * Completion. Thus, we don't need to release it again.1065 */1066 1067 if (q->queue_type == QUEUE_TYPE_INT)1068 return;1069 1070 kfree(q->shadow_queue);1071 1072 if (q->queue_type == QUEUE_TYPE_CPU)1073 hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);1074 else1075 hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,1076 q->bus_address);1077}1078 1079int hl_hw_queues_create(struct hl_device *hdev)1080{1081 struct asic_fixed_properties *asic = &hdev->asic_prop;1082 struct hl_hw_queue *q;1083 int i, rc, q_ready_cnt;1084 1085 hdev->kernel_queues = kcalloc(asic->max_queues,1086 sizeof(*hdev->kernel_queues), GFP_KERNEL);1087 1088 if (!hdev->kernel_queues) {1089 dev_err(hdev->dev, "Not enough memory for H/W queues\n");1090 return -ENOMEM;1091 }1092 1093 /* Initialize the H/W queues */1094 for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;1095 i < asic->max_queues ; i++, q_ready_cnt++, q++) {1096 1097 q->queue_type = asic->hw_queues_props[i].type;1098 q->supports_sync_stream =1099 asic->hw_queues_props[i].supports_sync_stream;1100 q->collective_mode = asic->hw_queues_props[i].collective_mode;1101 q->dram_bd = asic->hw_queues_props[i].dram_bd;1102 1103 rc = queue_init(hdev, q, i);1104 if (rc) {1105 dev_err(hdev->dev,1106 "failed to initialize queue %d\n", i);1107 goto release_queues;1108 }1109 1110 /* Set DRAM PQ address for the queue if it should be at DRAM */1111 if (q->dram_bd)1112 q->pq_dram_address = asic->hw_queues_props[i].q_dram_bd_address;1113 }1114 1115 return 0;1116 1117release_queues:1118 for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)1119 queue_fini(hdev, q);1120 1121 kfree(hdev->kernel_queues);1122 1123 return rc;1124}1125 1126void hl_hw_queues_destroy(struct hl_device *hdev)1127{1128 struct hl_hw_queue *q;1129 u32 max_queues = hdev->asic_prop.max_queues;1130 int i;1131 1132 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)1133 queue_fini(hdev, q);1134 1135 kfree(hdev->kernel_queues);1136}1137 1138void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)1139{1140 struct hl_hw_queue *q;1141 u32 max_queues = hdev->asic_prop.max_queues;1142 int i;1143 1144 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {1145 if ((!q->valid) ||1146 ((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))1147 continue;1148 q->pi = 0;1149 atomic_set(&q->ci, 0);1150 1151 if (q->supports_sync_stream)1152 sync_stream_queue_reset(hdev, q->hw_queue_id);1153 }1154}1155