576 lines · c
1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)2/* Copyright(c) 2014 - 2020 Intel Corporation */3#include <linux/delay.h>4#include <linux/nospec.h>5#include "adf_accel_devices.h"6#include "adf_transport_internal.h"7#include "adf_transport_access_macros.h"8#include "adf_cfg.h"9#include "adf_common_drv.h"10 11#define ADF_MAX_RING_THRESHOLD 8012#define ADF_PERCENT(tot, percent) (((tot) * (percent)) / 100)13 14static inline u32 adf_modulo(u32 data, u32 shift)15{16 u32 div = data >> shift;17 u32 mult = div << shift;18 19 return data - mult;20}21 22static inline int adf_check_ring_alignment(u64 addr, u64 size)23{24 if (((size - 1) & addr) != 0)25 return -EFAULT;26 return 0;27}28 29static int adf_verify_ring_size(u32 msg_size, u32 msg_num)30{31 int i = ADF_MIN_RING_SIZE;32 33 for (; i <= ADF_MAX_RING_SIZE; i++)34 if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i))35 return i;36 37 return ADF_DEFAULT_RING_SIZE;38}39 40static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring)41{42 spin_lock(&bank->lock);43 if (bank->ring_mask & (1 << ring)) {44 spin_unlock(&bank->lock);45 return -EFAULT;46 }47 bank->ring_mask |= (1 << ring);48 spin_unlock(&bank->lock);49 return 0;50}51 52static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring)53{54 spin_lock(&bank->lock);55 bank->ring_mask &= ~(1 << ring);56 spin_unlock(&bank->lock);57}58 59static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)60{61 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);62 63 spin_lock_bh(&bank->lock);64 bank->irq_mask |= (1 << ring);65 spin_unlock_bh(&bank->lock);66 csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number,67 bank->irq_mask);68 csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number,69 bank->irq_coalesc_timer);70}71 72static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)73{74 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);75 76 spin_lock_bh(&bank->lock);77 bank->irq_mask &= ~(1 << ring);78 spin_unlock_bh(&bank->lock);79 csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number,80 bank->irq_mask);81}82 83bool adf_ring_nearly_full(struct adf_etr_ring_data *ring)84{85 return atomic_read(ring->inflights) > ring->threshold;86}87 88int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)89{90 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);91 92 if (atomic_add_return(1, ring->inflights) >93 ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {94 atomic_dec(ring->inflights);95 return -EAGAIN;96 }97 spin_lock_bh(&ring->lock);98 memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg,99 ADF_MSG_SIZE_TO_BYTES(ring->msg_size));100 101 ring->tail = adf_modulo(ring->tail +102 ADF_MSG_SIZE_TO_BYTES(ring->msg_size),103 ADF_RING_SIZE_MODULO(ring->ring_size));104 csr_ops->write_csr_ring_tail(ring->bank->csr_addr,105 ring->bank->bank_number, ring->ring_number,106 ring->tail);107 spin_unlock_bh(&ring->lock);108 109 return 0;110}111 112static int adf_handle_response(struct adf_etr_ring_data *ring)113{114 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);115 u32 msg_counter = 0;116 u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);117 118 while (*msg != ADF_RING_EMPTY_SIG) {119 ring->callback((u32 *)msg);120 atomic_dec(ring->inflights);121 *msg = ADF_RING_EMPTY_SIG;122 ring->head = adf_modulo(ring->head +123 ADF_MSG_SIZE_TO_BYTES(ring->msg_size),124 ADF_RING_SIZE_MODULO(ring->ring_size));125 msg_counter++;126 msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);127 }128 if (msg_counter > 0) {129 csr_ops->write_csr_ring_head(ring->bank->csr_addr,130 ring->bank->bank_number,131 ring->ring_number, ring->head);132 }133 return 0;134}135 136static void adf_configure_tx_ring(struct adf_etr_ring_data *ring)137{138 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);139 u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);140 141 csr_ops->write_csr_ring_config(ring->bank->csr_addr,142 ring->bank->bank_number,143 ring->ring_number, ring_config);144 145}146 147static void adf_configure_rx_ring(struct adf_etr_ring_data *ring)148{149 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);150 u32 ring_config =151 BUILD_RESP_RING_CONFIG(ring->ring_size,152 ADF_RING_NEAR_WATERMARK_512,153 ADF_RING_NEAR_WATERMARK_0);154 155 csr_ops->write_csr_ring_config(ring->bank->csr_addr,156 ring->bank->bank_number,157 ring->ring_number, ring_config);158}159 160static int adf_init_ring(struct adf_etr_ring_data *ring)161{162 struct adf_etr_bank_data *bank = ring->bank;163 struct adf_accel_dev *accel_dev = bank->accel_dev;164 struct adf_hw_device_data *hw_data = accel_dev->hw_device;165 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);166 u64 ring_base;167 u32 ring_size_bytes =168 ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);169 170 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);171 ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev),172 ring_size_bytes, &ring->dma_addr,173 GFP_KERNEL);174 if (!ring->base_addr)175 return -ENOMEM;176 177 memset(ring->base_addr, 0x7F, ring_size_bytes);178 /* The base_addr has to be aligned to the size of the buffer */179 if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) {180 dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n");181 dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes,182 ring->base_addr, ring->dma_addr);183 ring->base_addr = NULL;184 return -EFAULT;185 }186 187 if (hw_data->tx_rings_mask & (1 << ring->ring_number))188 adf_configure_tx_ring(ring);189 190 else191 adf_configure_rx_ring(ring);192 193 ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr,194 ring->ring_size);195 196 csr_ops->write_csr_ring_base(ring->bank->csr_addr,197 ring->bank->bank_number, ring->ring_number,198 ring_base);199 spin_lock_init(&ring->lock);200 return 0;201}202 203static void adf_cleanup_ring(struct adf_etr_ring_data *ring)204{205 u32 ring_size_bytes =206 ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);207 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);208 209 if (ring->base_addr) {210 memset(ring->base_addr, 0x7F, ring_size_bytes);211 dma_free_coherent(&GET_DEV(ring->bank->accel_dev),212 ring_size_bytes, ring->base_addr,213 ring->dma_addr);214 }215}216 217int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,218 u32 bank_num, u32 num_msgs,219 u32 msg_size, const char *ring_name,220 adf_callback_fn callback, int poll_mode,221 struct adf_etr_ring_data **ring_ptr)222{223 struct adf_etr_data *transport_data = accel_dev->transport;224 u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev);225 struct adf_etr_bank_data *bank;226 struct adf_etr_ring_data *ring;227 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];228 int max_inflights;229 u32 ring_num;230 int ret;231 232 if (bank_num >= GET_MAX_BANKS(accel_dev)) {233 dev_err(&GET_DEV(accel_dev), "Invalid bank number\n");234 return -EFAULT;235 }236 if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {237 dev_err(&GET_DEV(accel_dev), "Invalid msg size\n");238 return -EFAULT;239 }240 if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs),241 ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) {242 dev_err(&GET_DEV(accel_dev),243 "Invalid ring size for given msg size\n");244 return -EFAULT;245 }246 if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) {247 dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n",248 section, ring_name);249 return -EFAULT;250 }251 if (kstrtouint(val, 10, &ring_num)) {252 dev_err(&GET_DEV(accel_dev), "Can't get ring number\n");253 return -EFAULT;254 }255 if (ring_num >= num_rings_per_bank) {256 dev_err(&GET_DEV(accel_dev), "Invalid ring number\n");257 return -EFAULT;258 }259 260 ring_num = array_index_nospec(ring_num, num_rings_per_bank);261 bank = &transport_data->banks[bank_num];262 if (adf_reserve_ring(bank, ring_num)) {263 dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n",264 ring_num, ring_name);265 return -EFAULT;266 }267 ring = &bank->rings[ring_num];268 ring->ring_number = ring_num;269 ring->bank = bank;270 ring->callback = callback;271 ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size);272 ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);273 ring->head = 0;274 ring->tail = 0;275 max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);276 ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD);277 atomic_set(ring->inflights, 0);278 ret = adf_init_ring(ring);279 if (ret)280 goto err;281 282 /* Enable HW arbitration for the given ring */283 adf_update_ring_arb(ring);284 285 if (adf_ring_debugfs_add(ring, ring_name)) {286 dev_err(&GET_DEV(accel_dev),287 "Couldn't add ring debugfs entry\n");288 ret = -EFAULT;289 goto err;290 }291 292 /* Enable interrupts if needed */293 if (callback && (!poll_mode))294 adf_enable_ring_irq(bank, ring->ring_number);295 *ring_ptr = ring;296 return 0;297err:298 adf_cleanup_ring(ring);299 adf_unreserve_ring(bank, ring_num);300 adf_update_ring_arb(ring);301 return ret;302}303 304void adf_remove_ring(struct adf_etr_ring_data *ring)305{306 struct adf_etr_bank_data *bank = ring->bank;307 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);308 309 /* Disable interrupts for the given ring */310 adf_disable_ring_irq(bank, ring->ring_number);311 312 /* Clear PCI config space */313 314 csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number,315 ring->ring_number, 0);316 csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number,317 ring->ring_number, 0);318 adf_ring_debugfs_rm(ring);319 adf_unreserve_ring(bank, ring->ring_number);320 /* Disable HW arbitration for the given ring */321 adf_update_ring_arb(ring);322 adf_cleanup_ring(ring);323}324 325static void adf_ring_response_handler(struct adf_etr_bank_data *bank)326{327 struct adf_accel_dev *accel_dev = bank->accel_dev;328 u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev);329 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);330 unsigned long empty_rings;331 int i;332 333 empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr,334 bank->bank_number);335 empty_rings = ~empty_rings & bank->irq_mask;336 337 for_each_set_bit(i, &empty_rings, num_rings_per_bank)338 adf_handle_response(&bank->rings[i]);339}340 341void adf_response_handler(uintptr_t bank_addr)342{343 struct adf_etr_bank_data *bank = (void *)bank_addr;344 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);345 346 /* Handle all the responses and reenable IRQs */347 adf_ring_response_handler(bank);348 349 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number,350 bank->irq_mask);351}352 353static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev,354 const char *section, const char *format,355 u32 key, u32 *value)356{357 char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES];358 char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES];359 360 snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key);361 362 if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf))363 return -EFAULT;364 365 if (kstrtouint(val_buf, 10, value))366 return -EFAULT;367 return 0;368}369 370static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank,371 const char *section,372 u32 bank_num_in_accel)373{374 if (adf_get_cfg_int(bank->accel_dev, section,375 ADF_ETRMGR_COALESCE_TIMER_FORMAT,376 bank_num_in_accel, &bank->irq_coalesc_timer))377 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME;378 379 if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer ||380 ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer)381 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME;382}383 384static int adf_init_bank(struct adf_accel_dev *accel_dev,385 struct adf_etr_bank_data *bank,386 u32 bank_num, void __iomem *csr_addr)387{388 struct adf_hw_device_data *hw_data = accel_dev->hw_device;389 u8 num_rings_per_bank = hw_data->num_rings_per_bank;390 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops;391 u32 irq_mask = BIT(num_rings_per_bank) - 1;392 struct adf_etr_ring_data *ring;393 struct adf_etr_ring_data *tx_ring;394 u32 i, coalesc_enabled = 0;395 unsigned long ring_mask;396 int size;397 398 memset(bank, 0, sizeof(*bank));399 bank->bank_number = bank_num;400 bank->csr_addr = csr_addr;401 bank->accel_dev = accel_dev;402 spin_lock_init(&bank->lock);403 404 /* Allocate the rings in the bank */405 size = num_rings_per_bank * sizeof(struct adf_etr_ring_data);406 bank->rings = kzalloc_node(size, GFP_KERNEL,407 dev_to_node(&GET_DEV(accel_dev)));408 if (!bank->rings)409 return -ENOMEM;410 411 /* Enable IRQ coalescing always. This will allow to use412 * the optimised flag and coalesc register.413 * If it is disabled in the config file just use min time value */414 if ((adf_get_cfg_int(accel_dev, "Accelerator0",415 ADF_ETRMGR_COALESCING_ENABLED_FORMAT, bank_num,416 &coalesc_enabled) == 0) && coalesc_enabled)417 adf_get_coalesc_timer(bank, "Accelerator0", bank_num);418 else419 bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME;420 421 for (i = 0; i < num_rings_per_bank; i++) {422 csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0);423 csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0);424 425 ring = &bank->rings[i];426 if (hw_data->tx_rings_mask & (1 << i)) {427 ring->inflights =428 kzalloc_node(sizeof(atomic_t),429 GFP_KERNEL,430 dev_to_node(&GET_DEV(accel_dev)));431 if (!ring->inflights)432 goto err;433 } else {434 if (i < hw_data->tx_rx_gap) {435 dev_err(&GET_DEV(accel_dev),436 "Invalid tx rings mask config\n");437 goto err;438 }439 tx_ring = &bank->rings[i - hw_data->tx_rx_gap];440 ring->inflights = tx_ring->inflights;441 }442 }443 if (adf_bank_debugfs_add(bank)) {444 dev_err(&GET_DEV(accel_dev),445 "Failed to add bank debugfs entry\n");446 goto err;447 }448 449 csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask);450 csr_ops->write_csr_int_srcsel(csr_addr, bank_num);451 452 return 0;453err:454 ring_mask = hw_data->tx_rings_mask;455 for_each_set_bit(i, &ring_mask, num_rings_per_bank) {456 ring = &bank->rings[i];457 kfree(ring->inflights);458 ring->inflights = NULL;459 }460 kfree(bank->rings);461 return -ENOMEM;462}463 464/**465 * adf_init_etr_data() - Initialize transport rings for acceleration device466 * @accel_dev: Pointer to acceleration device.467 *468 * Function is the initializes the communications channels (rings) to the469 * acceleration device accel_dev.470 * To be used by QAT device specific drivers.471 *472 * Return: 0 on success, error code otherwise.473 */474int adf_init_etr_data(struct adf_accel_dev *accel_dev)475{476 struct adf_etr_data *etr_data;477 void __iomem *csr_addr;478 u32 size;479 u32 num_banks = 0;480 int i, ret;481 482 etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL,483 dev_to_node(&GET_DEV(accel_dev)));484 if (!etr_data)485 return -ENOMEM;486 487 num_banks = GET_MAX_BANKS(accel_dev);488 size = num_banks * sizeof(struct adf_etr_bank_data);489 etr_data->banks = kzalloc_node(size, GFP_KERNEL,490 dev_to_node(&GET_DEV(accel_dev)));491 if (!etr_data->banks) {492 ret = -ENOMEM;493 goto err_bank;494 }495 496 accel_dev->transport = etr_data;497 csr_addr = adf_get_etr_base(accel_dev);498 499 /* accel_dev->debugfs_dir should always be non-NULL here */500 etr_data->debug = debugfs_create_dir("transport",501 accel_dev->debugfs_dir);502 503 for (i = 0; i < num_banks; i++) {504 ret = adf_init_bank(accel_dev, &etr_data->banks[i], i,505 csr_addr);506 if (ret)507 goto err_bank_all;508 }509 510 return 0;511 512err_bank_all:513 debugfs_remove(etr_data->debug);514 kfree(etr_data->banks);515err_bank:516 kfree(etr_data);517 accel_dev->transport = NULL;518 return ret;519}520EXPORT_SYMBOL_GPL(adf_init_etr_data);521 522static void cleanup_bank(struct adf_etr_bank_data *bank)523{524 struct adf_accel_dev *accel_dev = bank->accel_dev;525 struct adf_hw_device_data *hw_data = accel_dev->hw_device;526 u8 num_rings_per_bank = hw_data->num_rings_per_bank;527 u32 i;528 529 for (i = 0; i < num_rings_per_bank; i++) {530 struct adf_etr_ring_data *ring = &bank->rings[i];531 532 if (bank->ring_mask & (1 << i))533 adf_cleanup_ring(ring);534 535 if (hw_data->tx_rings_mask & (1 << i))536 kfree(ring->inflights);537 }538 kfree(bank->rings);539 adf_bank_debugfs_rm(bank);540 memset(bank, 0, sizeof(*bank));541}542 543static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev)544{545 struct adf_etr_data *etr_data = accel_dev->transport;546 u32 i, num_banks = GET_MAX_BANKS(accel_dev);547 548 for (i = 0; i < num_banks; i++)549 cleanup_bank(&etr_data->banks[i]);550}551 552/**553 * adf_cleanup_etr_data() - Clear transport rings for acceleration device554 * @accel_dev: Pointer to acceleration device.555 *556 * Function is the clears the communications channels (rings) of the557 * acceleration device accel_dev.558 * To be used by QAT device specific drivers.559 *560 * Return: void561 */562void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev)563{564 struct adf_etr_data *etr_data = accel_dev->transport;565 566 if (etr_data) {567 adf_cleanup_etr_handles(accel_dev);568 debugfs_remove(etr_data->debug);569 kfree(etr_data->banks->rings);570 kfree(etr_data->banks);571 kfree(etr_data);572 accel_dev->transport = NULL;573 }574}575EXPORT_SYMBOL_GPL(adf_cleanup_etr_data);576