297 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2022 Intel Corporation */3#include <linux/module.h>4#include <linux/slab.h>5#include "adf_accel_devices.h"6#include "adf_common_drv.h"7#include "adf_transport.h"8#include "adf_transport_access_macros.h"9#include "adf_cfg.h"10#include "adf_cfg_strings.h"11#include "qat_compression.h"12#include "icp_qat_fw.h"13 14#define SEC ADF_KERNEL_SEC15 16static struct service_hndl qat_compression;17 18void qat_compression_put_instance(struct qat_compression_instance *inst)19{20 atomic_dec(&inst->refctr);21 adf_dev_put(inst->accel_dev);22}23 24static int qat_compression_free_instances(struct adf_accel_dev *accel_dev)25{26 struct qat_compression_instance *inst;27 struct list_head *list_ptr, *tmp;28 int i;29 30 list_for_each_safe(list_ptr, tmp, &accel_dev->compression_list) {31 inst = list_entry(list_ptr,32 struct qat_compression_instance, list);33 34 for (i = 0; i < atomic_read(&inst->refctr); i++)35 qat_compression_put_instance(inst);36 37 if (inst->dc_tx)38 adf_remove_ring(inst->dc_tx);39 40 if (inst->dc_rx)41 adf_remove_ring(inst->dc_rx);42 43 list_del(list_ptr);44 kfree(inst);45 }46 return 0;47}48 49struct qat_compression_instance *qat_compression_get_instance_node(int node)50{51 struct qat_compression_instance *inst = NULL;52 struct adf_accel_dev *accel_dev = NULL;53 unsigned long best = ~0;54 struct list_head *itr;55 56 list_for_each(itr, adf_devmgr_get_head()) {57 struct adf_accel_dev *tmp_dev;58 unsigned long ctr;59 int tmp_dev_node;60 61 tmp_dev = list_entry(itr, struct adf_accel_dev, list);62 tmp_dev_node = dev_to_node(&GET_DEV(tmp_dev));63 64 if ((node == tmp_dev_node || tmp_dev_node < 0) &&65 adf_dev_started(tmp_dev) && !list_empty(&tmp_dev->compression_list)) {66 ctr = atomic_read(&tmp_dev->ref_count);67 if (best > ctr) {68 accel_dev = tmp_dev;69 best = ctr;70 }71 }72 }73 74 if (!accel_dev) {75 pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node);76 /* Get any started device */77 list_for_each(itr, adf_devmgr_get_head()) {78 struct adf_accel_dev *tmp_dev;79 80 tmp_dev = list_entry(itr, struct adf_accel_dev, list);81 if (adf_dev_started(tmp_dev) &&82 !list_empty(&tmp_dev->compression_list)) {83 accel_dev = tmp_dev;84 break;85 }86 }87 }88 89 if (!accel_dev)90 return NULL;91 92 best = ~0;93 list_for_each(itr, &accel_dev->compression_list) {94 struct qat_compression_instance *tmp_inst;95 unsigned long ctr;96 97 tmp_inst = list_entry(itr, struct qat_compression_instance, list);98 ctr = atomic_read(&tmp_inst->refctr);99 if (best > ctr) {100 inst = tmp_inst;101 best = ctr;102 }103 }104 if (inst) {105 if (adf_dev_get(accel_dev)) {106 dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n");107 return NULL;108 }109 atomic_inc(&inst->refctr);110 }111 return inst;112}113 114static int qat_compression_create_instances(struct adf_accel_dev *accel_dev)115{116 struct qat_compression_instance *inst;117 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];118 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];119 unsigned long num_inst, num_msg_dc;120 unsigned long bank;121 int msg_size;122 int ret;123 int i;124 125 INIT_LIST_HEAD(&accel_dev->compression_list);126 strscpy(key, ADF_NUM_DC, sizeof(key));127 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);128 if (ret)129 return ret;130 131 ret = kstrtoul(val, 10, &num_inst);132 if (ret)133 return ret;134 135 for (i = 0; i < num_inst; i++) {136 inst = kzalloc_node(sizeof(*inst), GFP_KERNEL,137 dev_to_node(&GET_DEV(accel_dev)));138 if (!inst) {139 ret = -ENOMEM;140 goto err;141 }142 143 list_add_tail(&inst->list, &accel_dev->compression_list);144 inst->id = i;145 atomic_set(&inst->refctr, 0);146 inst->accel_dev = accel_dev;147 inst->build_deflate_ctx = GET_DC_OPS(accel_dev)->build_deflate_ctx;148 149 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i);150 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);151 if (ret)152 return ret;153 154 ret = kstrtoul(val, 10, &bank);155 if (ret)156 return ret;157 158 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i);159 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);160 if (ret)161 return ret;162 163 ret = kstrtoul(val, 10, &num_msg_dc);164 if (ret)165 return ret;166 167 msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ;168 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i);169 ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc,170 msg_size, key, NULL, 0, &inst->dc_tx);171 if (ret)172 return ret;173 174 msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ;175 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i);176 ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc,177 msg_size, key, qat_comp_alg_callback, 0,178 &inst->dc_rx);179 if (ret)180 return ret;181 182 inst->dc_data = accel_dev->dc_data;183 INIT_LIST_HEAD(&inst->backlog.list);184 spin_lock_init(&inst->backlog.lock);185 }186 return 0;187err:188 qat_compression_free_instances(accel_dev);189 return ret;190}191 192static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev)193{194 struct device *dev = &GET_DEV(accel_dev);195 dma_addr_t obuff_p = DMA_MAPPING_ERROR;196 size_t ovf_buff_sz = QAT_COMP_MAX_SKID;197 struct adf_dc_data *dc_data = NULL;198 u8 *obuff = NULL;199 200 dc_data = devm_kzalloc(dev, sizeof(*dc_data), GFP_KERNEL);201 if (!dc_data)202 goto err;203 204 obuff = kzalloc_node(ovf_buff_sz, GFP_KERNEL, dev_to_node(dev));205 if (!obuff)206 goto err;207 208 obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_FROM_DEVICE);209 if (unlikely(dma_mapping_error(dev, obuff_p)))210 goto err;211 212 dc_data->ovf_buff = obuff;213 dc_data->ovf_buff_p = obuff_p;214 dc_data->ovf_buff_sz = ovf_buff_sz;215 216 accel_dev->dc_data = dc_data;217 218 return 0;219 220err:221 accel_dev->dc_data = NULL;222 kfree(obuff);223 devm_kfree(dev, dc_data);224 return -ENOMEM;225}226 227static void qat_free_dc_data(struct adf_accel_dev *accel_dev)228{229 struct adf_dc_data *dc_data = accel_dev->dc_data;230 struct device *dev = &GET_DEV(accel_dev);231 232 if (!dc_data)233 return;234 235 dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz,236 DMA_FROM_DEVICE);237 kfree_sensitive(dc_data->ovf_buff);238 devm_kfree(dev, dc_data);239 accel_dev->dc_data = NULL;240}241 242static int qat_compression_init(struct adf_accel_dev *accel_dev)243{244 int ret;245 246 ret = qat_compression_alloc_dc_data(accel_dev);247 if (ret)248 return ret;249 250 ret = qat_compression_create_instances(accel_dev);251 if (ret)252 qat_free_dc_data(accel_dev);253 254 return ret;255}256 257static int qat_compression_shutdown(struct adf_accel_dev *accel_dev)258{259 qat_free_dc_data(accel_dev);260 return qat_compression_free_instances(accel_dev);261}262 263static int qat_compression_event_handler(struct adf_accel_dev *accel_dev,264 enum adf_event event)265{266 int ret;267 268 switch (event) {269 case ADF_EVENT_INIT:270 ret = qat_compression_init(accel_dev);271 break;272 case ADF_EVENT_SHUTDOWN:273 ret = qat_compression_shutdown(accel_dev);274 break;275 case ADF_EVENT_RESTARTING:276 case ADF_EVENT_RESTARTED:277 case ADF_EVENT_START:278 case ADF_EVENT_STOP:279 default:280 ret = 0;281 }282 return ret;283}284 285int qat_compression_register(void)286{287 memset(&qat_compression, 0, sizeof(qat_compression));288 qat_compression.event_hld = qat_compression_event_handler;289 qat_compression.name = "qat_compression";290 return adf_service_register(&qat_compression);291}292 293int qat_compression_unregister(void)294{295 return adf_service_unregister(&qat_compression);296}297