1992 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */3#include <linux/init.h>4#include <linux/kernel.h>5#include <linux/module.h>6#include <linux/pci.h>7#include <linux/device.h>8#include <linux/io-64-nonatomic-lo-hi.h>9#include <uapi/linux/idxd.h>10#include "registers.h"11#include "idxd.h"12 13static char *idxd_wq_type_names[] = {14 [IDXD_WQT_NONE] = "none",15 [IDXD_WQT_KERNEL] = "kernel",16 [IDXD_WQT_USER] = "user",17};18 19/* IDXD engine attributes */20static ssize_t engine_group_id_show(struct device *dev,21 struct device_attribute *attr, char *buf)22{23 struct idxd_engine *engine = confdev_to_engine(dev);24 25 if (engine->group)26 return sysfs_emit(buf, "%d\n", engine->group->id);27 else28 return sysfs_emit(buf, "%d\n", -1);29}30 31static ssize_t engine_group_id_store(struct device *dev,32 struct device_attribute *attr,33 const char *buf, size_t count)34{35 struct idxd_engine *engine = confdev_to_engine(dev);36 struct idxd_device *idxd = engine->idxd;37 long id;38 int rc;39 struct idxd_group *prevg;40 41 rc = kstrtol(buf, 10, &id);42 if (rc < 0)43 return -EINVAL;44 45 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))46 return -EPERM;47 48 if (id > idxd->max_groups - 1 || id < -1)49 return -EINVAL;50 51 if (id == -1) {52 if (engine->group) {53 engine->group->num_engines--;54 engine->group = NULL;55 }56 return count;57 }58 59 prevg = engine->group;60 61 if (prevg)62 prevg->num_engines--;63 engine->group = idxd->groups[id];64 engine->group->num_engines++;65 66 return count;67}68 69static struct device_attribute dev_attr_engine_group =70 __ATTR(group_id, 0644, engine_group_id_show,71 engine_group_id_store);72 73static struct attribute *idxd_engine_attributes[] = {74 &dev_attr_engine_group.attr,75 NULL,76};77 78static const struct attribute_group idxd_engine_attribute_group = {79 .attrs = idxd_engine_attributes,80};81 82static const struct attribute_group *idxd_engine_attribute_groups[] = {83 &idxd_engine_attribute_group,84 NULL,85};86 87static void idxd_conf_engine_release(struct device *dev)88{89 struct idxd_engine *engine = confdev_to_engine(dev);90 91 kfree(engine);92}93 94const struct device_type idxd_engine_device_type = {95 .name = "engine",96 .release = idxd_conf_engine_release,97 .groups = idxd_engine_attribute_groups,98};99 100/* Group attributes */101 102static void idxd_set_free_rdbufs(struct idxd_device *idxd)103{104 int i, rdbufs;105 106 for (i = 0, rdbufs = 0; i < idxd->max_groups; i++) {107 struct idxd_group *g = idxd->groups[i];108 109 rdbufs += g->rdbufs_reserved;110 }111 112 idxd->nr_rdbufs = idxd->max_rdbufs - rdbufs;113}114 115static ssize_t group_read_buffers_reserved_show(struct device *dev,116 struct device_attribute *attr,117 char *buf)118{119 struct idxd_group *group = confdev_to_group(dev);120 121 return sysfs_emit(buf, "%u\n", group->rdbufs_reserved);122}123 124static ssize_t group_tokens_reserved_show(struct device *dev,125 struct device_attribute *attr,126 char *buf)127{128 dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n");129 return group_read_buffers_reserved_show(dev, attr, buf);130}131 132static ssize_t group_read_buffers_reserved_store(struct device *dev,133 struct device_attribute *attr,134 const char *buf, size_t count)135{136 struct idxd_group *group = confdev_to_group(dev);137 struct idxd_device *idxd = group->idxd;138 unsigned long val;139 int rc;140 141 rc = kstrtoul(buf, 10, &val);142 if (rc < 0)143 return -EINVAL;144 145 if (idxd->data->type == IDXD_TYPE_IAX)146 return -EOPNOTSUPP;147 148 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))149 return -EPERM;150 151 if (idxd->state == IDXD_DEV_ENABLED)152 return -EPERM;153 154 if (val > idxd->max_rdbufs)155 return -EINVAL;156 157 if (val > idxd->nr_rdbufs + group->rdbufs_reserved)158 return -EINVAL;159 160 group->rdbufs_reserved = val;161 idxd_set_free_rdbufs(idxd);162 return count;163}164 165static ssize_t group_tokens_reserved_store(struct device *dev,166 struct device_attribute *attr,167 const char *buf, size_t count)168{169 dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n");170 return group_read_buffers_reserved_store(dev, attr, buf, count);171}172 173static struct device_attribute dev_attr_group_tokens_reserved =174 __ATTR(tokens_reserved, 0644, group_tokens_reserved_show,175 group_tokens_reserved_store);176 177static struct device_attribute dev_attr_group_read_buffers_reserved =178 __ATTR(read_buffers_reserved, 0644, group_read_buffers_reserved_show,179 group_read_buffers_reserved_store);180 181static ssize_t group_read_buffers_allowed_show(struct device *dev,182 struct device_attribute *attr,183 char *buf)184{185 struct idxd_group *group = confdev_to_group(dev);186 187 return sysfs_emit(buf, "%u\n", group->rdbufs_allowed);188}189 190static ssize_t group_tokens_allowed_show(struct device *dev,191 struct device_attribute *attr,192 char *buf)193{194 dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n");195 return group_read_buffers_allowed_show(dev, attr, buf);196}197 198static ssize_t group_read_buffers_allowed_store(struct device *dev,199 struct device_attribute *attr,200 const char *buf, size_t count)201{202 struct idxd_group *group = confdev_to_group(dev);203 struct idxd_device *idxd = group->idxd;204 unsigned long val;205 int rc;206 207 rc = kstrtoul(buf, 10, &val);208 if (rc < 0)209 return -EINVAL;210 211 if (idxd->data->type == IDXD_TYPE_IAX)212 return -EOPNOTSUPP;213 214 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))215 return -EPERM;216 217 if (idxd->state == IDXD_DEV_ENABLED)218 return -EPERM;219 220 if (val < 4 * group->num_engines ||221 val > group->rdbufs_reserved + idxd->nr_rdbufs)222 return -EINVAL;223 224 group->rdbufs_allowed = val;225 return count;226}227 228static ssize_t group_tokens_allowed_store(struct device *dev,229 struct device_attribute *attr,230 const char *buf, size_t count)231{232 dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n");233 return group_read_buffers_allowed_store(dev, attr, buf, count);234}235 236static struct device_attribute dev_attr_group_tokens_allowed =237 __ATTR(tokens_allowed, 0644, group_tokens_allowed_show,238 group_tokens_allowed_store);239 240static struct device_attribute dev_attr_group_read_buffers_allowed =241 __ATTR(read_buffers_allowed, 0644, group_read_buffers_allowed_show,242 group_read_buffers_allowed_store);243 244static ssize_t group_use_read_buffer_limit_show(struct device *dev,245 struct device_attribute *attr,246 char *buf)247{248 struct idxd_group *group = confdev_to_group(dev);249 250 return sysfs_emit(buf, "%u\n", group->use_rdbuf_limit);251}252 253static ssize_t group_use_token_limit_show(struct device *dev,254 struct device_attribute *attr,255 char *buf)256{257 dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n");258 return group_use_read_buffer_limit_show(dev, attr, buf);259}260 261static ssize_t group_use_read_buffer_limit_store(struct device *dev,262 struct device_attribute *attr,263 const char *buf, size_t count)264{265 struct idxd_group *group = confdev_to_group(dev);266 struct idxd_device *idxd = group->idxd;267 unsigned long val;268 int rc;269 270 rc = kstrtoul(buf, 10, &val);271 if (rc < 0)272 return -EINVAL;273 274 if (idxd->data->type == IDXD_TYPE_IAX)275 return -EOPNOTSUPP;276 277 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))278 return -EPERM;279 280 if (idxd->state == IDXD_DEV_ENABLED)281 return -EPERM;282 283 if (idxd->rdbuf_limit == 0)284 return -EPERM;285 286 group->use_rdbuf_limit = !!val;287 return count;288}289 290static ssize_t group_use_token_limit_store(struct device *dev,291 struct device_attribute *attr,292 const char *buf, size_t count)293{294 dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n");295 return group_use_read_buffer_limit_store(dev, attr, buf, count);296}297 298static struct device_attribute dev_attr_group_use_token_limit =299 __ATTR(use_token_limit, 0644, group_use_token_limit_show,300 group_use_token_limit_store);301 302static struct device_attribute dev_attr_group_use_read_buffer_limit =303 __ATTR(use_read_buffer_limit, 0644, group_use_read_buffer_limit_show,304 group_use_read_buffer_limit_store);305 306static ssize_t group_engines_show(struct device *dev,307 struct device_attribute *attr, char *buf)308{309 struct idxd_group *group = confdev_to_group(dev);310 int i, rc = 0;311 struct idxd_device *idxd = group->idxd;312 313 for (i = 0; i < idxd->max_engines; i++) {314 struct idxd_engine *engine = idxd->engines[i];315 316 if (!engine->group)317 continue;318 319 if (engine->group->id == group->id)320 rc += sysfs_emit_at(buf, rc, "engine%d.%d ", idxd->id, engine->id);321 }322 323 if (!rc)324 return 0;325 rc--;326 rc += sysfs_emit_at(buf, rc, "\n");327 328 return rc;329}330 331static struct device_attribute dev_attr_group_engines =332 __ATTR(engines, 0444, group_engines_show, NULL);333 334static ssize_t group_work_queues_show(struct device *dev,335 struct device_attribute *attr, char *buf)336{337 struct idxd_group *group = confdev_to_group(dev);338 int i, rc = 0;339 struct idxd_device *idxd = group->idxd;340 341 for (i = 0; i < idxd->max_wqs; i++) {342 struct idxd_wq *wq = idxd->wqs[i];343 344 if (!wq->group)345 continue;346 347 if (wq->group->id == group->id)348 rc += sysfs_emit_at(buf, rc, "wq%d.%d ", idxd->id, wq->id);349 }350 351 if (!rc)352 return 0;353 rc--;354 rc += sysfs_emit_at(buf, rc, "\n");355 356 return rc;357}358 359static struct device_attribute dev_attr_group_work_queues =360 __ATTR(work_queues, 0444, group_work_queues_show, NULL);361 362static ssize_t group_traffic_class_a_show(struct device *dev,363 struct device_attribute *attr,364 char *buf)365{366 struct idxd_group *group = confdev_to_group(dev);367 368 return sysfs_emit(buf, "%d\n", group->tc_a);369}370 371static ssize_t group_traffic_class_a_store(struct device *dev,372 struct device_attribute *attr,373 const char *buf, size_t count)374{375 struct idxd_group *group = confdev_to_group(dev);376 struct idxd_device *idxd = group->idxd;377 long val;378 int rc;379 380 rc = kstrtol(buf, 10, &val);381 if (rc < 0)382 return -EINVAL;383 384 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))385 return -EPERM;386 387 if (idxd->state == IDXD_DEV_ENABLED)388 return -EPERM;389 390 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override)391 return -EPERM;392 393 if (val < 0 || val > 7)394 return -EINVAL;395 396 group->tc_a = val;397 return count;398}399 400static struct device_attribute dev_attr_group_traffic_class_a =401 __ATTR(traffic_class_a, 0644, group_traffic_class_a_show,402 group_traffic_class_a_store);403 404static ssize_t group_traffic_class_b_show(struct device *dev,405 struct device_attribute *attr,406 char *buf)407{408 struct idxd_group *group = confdev_to_group(dev);409 410 return sysfs_emit(buf, "%d\n", group->tc_b);411}412 413static ssize_t group_traffic_class_b_store(struct device *dev,414 struct device_attribute *attr,415 const char *buf, size_t count)416{417 struct idxd_group *group = confdev_to_group(dev);418 struct idxd_device *idxd = group->idxd;419 long val;420 int rc;421 422 rc = kstrtol(buf, 10, &val);423 if (rc < 0)424 return -EINVAL;425 426 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))427 return -EPERM;428 429 if (idxd->state == IDXD_DEV_ENABLED)430 return -EPERM;431 432 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override)433 return -EPERM;434 435 if (val < 0 || val > 7)436 return -EINVAL;437 438 group->tc_b = val;439 return count;440}441 442static struct device_attribute dev_attr_group_traffic_class_b =443 __ATTR(traffic_class_b, 0644, group_traffic_class_b_show,444 group_traffic_class_b_store);445 446static ssize_t group_desc_progress_limit_show(struct device *dev,447 struct device_attribute *attr,448 char *buf)449{450 struct idxd_group *group = confdev_to_group(dev);451 452 return sysfs_emit(buf, "%d\n", group->desc_progress_limit);453}454 455static ssize_t group_desc_progress_limit_store(struct device *dev,456 struct device_attribute *attr,457 const char *buf, size_t count)458{459 struct idxd_group *group = confdev_to_group(dev);460 int val, rc;461 462 rc = kstrtoint(buf, 10, &val);463 if (rc < 0)464 return -EINVAL;465 466 if (val & ~GENMASK(1, 0))467 return -EINVAL;468 469 group->desc_progress_limit = val;470 return count;471}472 473static struct device_attribute dev_attr_group_desc_progress_limit =474 __ATTR(desc_progress_limit, 0644, group_desc_progress_limit_show,475 group_desc_progress_limit_store);476 477static ssize_t group_batch_progress_limit_show(struct device *dev,478 struct device_attribute *attr,479 char *buf)480{481 struct idxd_group *group = confdev_to_group(dev);482 483 return sysfs_emit(buf, "%d\n", group->batch_progress_limit);484}485 486static ssize_t group_batch_progress_limit_store(struct device *dev,487 struct device_attribute *attr,488 const char *buf, size_t count)489{490 struct idxd_group *group = confdev_to_group(dev);491 int val, rc;492 493 rc = kstrtoint(buf, 10, &val);494 if (rc < 0)495 return -EINVAL;496 497 if (val & ~GENMASK(1, 0))498 return -EINVAL;499 500 group->batch_progress_limit = val;501 return count;502}503 504static struct device_attribute dev_attr_group_batch_progress_limit =505 __ATTR(batch_progress_limit, 0644, group_batch_progress_limit_show,506 group_batch_progress_limit_store);507static struct attribute *idxd_group_attributes[] = {508 &dev_attr_group_work_queues.attr,509 &dev_attr_group_engines.attr,510 &dev_attr_group_use_token_limit.attr,511 &dev_attr_group_use_read_buffer_limit.attr,512 &dev_attr_group_tokens_allowed.attr,513 &dev_attr_group_read_buffers_allowed.attr,514 &dev_attr_group_tokens_reserved.attr,515 &dev_attr_group_read_buffers_reserved.attr,516 &dev_attr_group_traffic_class_a.attr,517 &dev_attr_group_traffic_class_b.attr,518 &dev_attr_group_desc_progress_limit.attr,519 &dev_attr_group_batch_progress_limit.attr,520 NULL,521};522 523static bool idxd_group_attr_progress_limit_invisible(struct attribute *attr,524 struct idxd_device *idxd)525{526 return (attr == &dev_attr_group_desc_progress_limit.attr ||527 attr == &dev_attr_group_batch_progress_limit.attr) &&528 !idxd->hw.group_cap.progress_limit;529}530 531static bool idxd_group_attr_read_buffers_invisible(struct attribute *attr,532 struct idxd_device *idxd)533{534 /*535 * Intel IAA does not support Read Buffer allocation control,536 * make these attributes invisible.537 */538 return (attr == &dev_attr_group_use_token_limit.attr ||539 attr == &dev_attr_group_use_read_buffer_limit.attr ||540 attr == &dev_attr_group_tokens_allowed.attr ||541 attr == &dev_attr_group_read_buffers_allowed.attr ||542 attr == &dev_attr_group_tokens_reserved.attr ||543 attr == &dev_attr_group_read_buffers_reserved.attr) &&544 idxd->data->type == IDXD_TYPE_IAX;545}546 547static umode_t idxd_group_attr_visible(struct kobject *kobj,548 struct attribute *attr, int n)549{550 struct device *dev = container_of(kobj, struct device, kobj);551 struct idxd_group *group = confdev_to_group(dev);552 struct idxd_device *idxd = group->idxd;553 554 if (idxd_group_attr_progress_limit_invisible(attr, idxd))555 return 0;556 557 if (idxd_group_attr_read_buffers_invisible(attr, idxd))558 return 0;559 560 return attr->mode;561}562 563static const struct attribute_group idxd_group_attribute_group = {564 .attrs = idxd_group_attributes,565 .is_visible = idxd_group_attr_visible,566};567 568static const struct attribute_group *idxd_group_attribute_groups[] = {569 &idxd_group_attribute_group,570 NULL,571};572 573static void idxd_conf_group_release(struct device *dev)574{575 struct idxd_group *group = confdev_to_group(dev);576 577 kfree(group);578}579 580const struct device_type idxd_group_device_type = {581 .name = "group",582 .release = idxd_conf_group_release,583 .groups = idxd_group_attribute_groups,584};585 586/* IDXD work queue attribs */587static ssize_t wq_clients_show(struct device *dev,588 struct device_attribute *attr, char *buf)589{590 struct idxd_wq *wq = confdev_to_wq(dev);591 592 return sysfs_emit(buf, "%d\n", wq->client_count);593}594 595static struct device_attribute dev_attr_wq_clients =596 __ATTR(clients, 0444, wq_clients_show, NULL);597 598static ssize_t wq_state_show(struct device *dev,599 struct device_attribute *attr, char *buf)600{601 struct idxd_wq *wq = confdev_to_wq(dev);602 603 switch (wq->state) {604 case IDXD_WQ_DISABLED:605 return sysfs_emit(buf, "disabled\n");606 case IDXD_WQ_ENABLED:607 return sysfs_emit(buf, "enabled\n");608 }609 610 return sysfs_emit(buf, "unknown\n");611}612 613static struct device_attribute dev_attr_wq_state =614 __ATTR(state, 0444, wq_state_show, NULL);615 616static ssize_t wq_group_id_show(struct device *dev,617 struct device_attribute *attr, char *buf)618{619 struct idxd_wq *wq = confdev_to_wq(dev);620 621 if (wq->group)622 return sysfs_emit(buf, "%u\n", wq->group->id);623 else624 return sysfs_emit(buf, "-1\n");625}626 627static ssize_t wq_group_id_store(struct device *dev,628 struct device_attribute *attr,629 const char *buf, size_t count)630{631 struct idxd_wq *wq = confdev_to_wq(dev);632 struct idxd_device *idxd = wq->idxd;633 long id;634 int rc;635 struct idxd_group *prevg, *group;636 637 rc = kstrtol(buf, 10, &id);638 if (rc < 0)639 return -EINVAL;640 641 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))642 return -EPERM;643 644 if (wq->state != IDXD_WQ_DISABLED)645 return -EPERM;646 647 if (id > idxd->max_groups - 1 || id < -1)648 return -EINVAL;649 650 if (id == -1) {651 if (wq->group) {652 wq->group->num_wqs--;653 wq->group = NULL;654 }655 return count;656 }657 658 group = idxd->groups[id];659 prevg = wq->group;660 661 if (prevg)662 prevg->num_wqs--;663 wq->group = group;664 group->num_wqs++;665 return count;666}667 668static struct device_attribute dev_attr_wq_group_id =669 __ATTR(group_id, 0644, wq_group_id_show, wq_group_id_store);670 671static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr,672 char *buf)673{674 struct idxd_wq *wq = confdev_to_wq(dev);675 676 return sysfs_emit(buf, "%s\n", wq_dedicated(wq) ? "dedicated" : "shared");677}678 679static ssize_t wq_mode_store(struct device *dev,680 struct device_attribute *attr, const char *buf,681 size_t count)682{683 struct idxd_wq *wq = confdev_to_wq(dev);684 struct idxd_device *idxd = wq->idxd;685 686 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))687 return -EPERM;688 689 if (wq->state != IDXD_WQ_DISABLED)690 return -EPERM;691 692 if (sysfs_streq(buf, "dedicated")) {693 set_bit(WQ_FLAG_DEDICATED, &wq->flags);694 wq->threshold = 0;695 } else if (sysfs_streq(buf, "shared")) {696 clear_bit(WQ_FLAG_DEDICATED, &wq->flags);697 } else {698 return -EINVAL;699 }700 701 return count;702}703 704static struct device_attribute dev_attr_wq_mode =705 __ATTR(mode, 0644, wq_mode_show, wq_mode_store);706 707static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr,708 char *buf)709{710 struct idxd_wq *wq = confdev_to_wq(dev);711 712 return sysfs_emit(buf, "%u\n", wq->size);713}714 715static int total_claimed_wq_size(struct idxd_device *idxd)716{717 int i;718 int wq_size = 0;719 720 for (i = 0; i < idxd->max_wqs; i++) {721 struct idxd_wq *wq = idxd->wqs[i];722 723 wq_size += wq->size;724 }725 726 return wq_size;727}728 729static ssize_t wq_size_store(struct device *dev,730 struct device_attribute *attr, const char *buf,731 size_t count)732{733 struct idxd_wq *wq = confdev_to_wq(dev);734 unsigned long size;735 struct idxd_device *idxd = wq->idxd;736 int rc;737 738 rc = kstrtoul(buf, 10, &size);739 if (rc < 0)740 return -EINVAL;741 742 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))743 return -EPERM;744 745 if (idxd->state == IDXD_DEV_ENABLED)746 return -EPERM;747 748 if (size + total_claimed_wq_size(idxd) - wq->size > idxd->max_wq_size)749 return -EINVAL;750 751 wq->size = size;752 return count;753}754 755static struct device_attribute dev_attr_wq_size =756 __ATTR(size, 0644, wq_size_show, wq_size_store);757 758static ssize_t wq_priority_show(struct device *dev,759 struct device_attribute *attr, char *buf)760{761 struct idxd_wq *wq = confdev_to_wq(dev);762 763 return sysfs_emit(buf, "%u\n", wq->priority);764}765 766static ssize_t wq_priority_store(struct device *dev,767 struct device_attribute *attr,768 const char *buf, size_t count)769{770 struct idxd_wq *wq = confdev_to_wq(dev);771 unsigned long prio;772 struct idxd_device *idxd = wq->idxd;773 int rc;774 775 rc = kstrtoul(buf, 10, &prio);776 if (rc < 0)777 return -EINVAL;778 779 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))780 return -EPERM;781 782 if (wq->state != IDXD_WQ_DISABLED)783 return -EPERM;784 785 if (prio > IDXD_MAX_PRIORITY)786 return -EINVAL;787 788 wq->priority = prio;789 return count;790}791 792static struct device_attribute dev_attr_wq_priority =793 __ATTR(priority, 0644, wq_priority_show, wq_priority_store);794 795static ssize_t wq_block_on_fault_show(struct device *dev,796 struct device_attribute *attr, char *buf)797{798 struct idxd_wq *wq = confdev_to_wq(dev);799 800 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags));801}802 803static ssize_t wq_block_on_fault_store(struct device *dev,804 struct device_attribute *attr,805 const char *buf, size_t count)806{807 struct idxd_wq *wq = confdev_to_wq(dev);808 struct idxd_device *idxd = wq->idxd;809 bool bof;810 int rc;811 812 if (!idxd->hw.gen_cap.block_on_fault)813 return -EOPNOTSUPP;814 815 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))816 return -EPERM;817 818 if (wq->state != IDXD_WQ_DISABLED)819 return -ENXIO;820 821 rc = kstrtobool(buf, &bof);822 if (rc < 0)823 return rc;824 825 if (bof) {826 if (test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags))827 return -EOPNOTSUPP;828 829 set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);830 } else {831 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);832 }833 834 return count;835}836 837static struct device_attribute dev_attr_wq_block_on_fault =838 __ATTR(block_on_fault, 0644, wq_block_on_fault_show,839 wq_block_on_fault_store);840 841static ssize_t wq_threshold_show(struct device *dev,842 struct device_attribute *attr, char *buf)843{844 struct idxd_wq *wq = confdev_to_wq(dev);845 846 return sysfs_emit(buf, "%u\n", wq->threshold);847}848 849static ssize_t wq_threshold_store(struct device *dev,850 struct device_attribute *attr,851 const char *buf, size_t count)852{853 struct idxd_wq *wq = confdev_to_wq(dev);854 struct idxd_device *idxd = wq->idxd;855 unsigned int val;856 int rc;857 858 rc = kstrtouint(buf, 0, &val);859 if (rc < 0)860 return -EINVAL;861 862 if (val > wq->size || val <= 0)863 return -EINVAL;864 865 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))866 return -EPERM;867 868 if (wq->state != IDXD_WQ_DISABLED)869 return -ENXIO;870 871 if (test_bit(WQ_FLAG_DEDICATED, &wq->flags))872 return -EINVAL;873 874 wq->threshold = val;875 876 return count;877}878 879static struct device_attribute dev_attr_wq_threshold =880 __ATTR(threshold, 0644, wq_threshold_show, wq_threshold_store);881 882static ssize_t wq_type_show(struct device *dev,883 struct device_attribute *attr, char *buf)884{885 struct idxd_wq *wq = confdev_to_wq(dev);886 887 switch (wq->type) {888 case IDXD_WQT_KERNEL:889 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_KERNEL]);890 case IDXD_WQT_USER:891 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_USER]);892 case IDXD_WQT_NONE:893 default:894 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_NONE]);895 }896 897 return -EINVAL;898}899 900static ssize_t wq_type_store(struct device *dev,901 struct device_attribute *attr, const char *buf,902 size_t count)903{904 struct idxd_wq *wq = confdev_to_wq(dev);905 enum idxd_wq_type old_type;906 907 if (wq->state != IDXD_WQ_DISABLED)908 return -EPERM;909 910 old_type = wq->type;911 if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_NONE]))912 wq->type = IDXD_WQT_NONE;913 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_KERNEL]))914 wq->type = IDXD_WQT_KERNEL;915 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_USER]))916 wq->type = IDXD_WQT_USER;917 else918 return -EINVAL;919 920 /* If we are changing queue type, clear the name */921 if (wq->type != old_type)922 memset(wq->name, 0, WQ_NAME_SIZE + 1);923 924 return count;925}926 927static struct device_attribute dev_attr_wq_type =928 __ATTR(type, 0644, wq_type_show, wq_type_store);929 930static ssize_t wq_name_show(struct device *dev,931 struct device_attribute *attr, char *buf)932{933 struct idxd_wq *wq = confdev_to_wq(dev);934 935 return sysfs_emit(buf, "%s\n", wq->name);936}937 938static ssize_t wq_name_store(struct device *dev,939 struct device_attribute *attr, const char *buf,940 size_t count)941{942 struct idxd_wq *wq = confdev_to_wq(dev);943 char *input, *pos;944 945 if (wq->state != IDXD_WQ_DISABLED)946 return -EPERM;947 948 if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0)949 return -EINVAL;950 951 input = kstrndup(buf, count, GFP_KERNEL);952 if (!input)953 return -ENOMEM;954 955 pos = strim(input);956 memset(wq->name, 0, WQ_NAME_SIZE + 1);957 sprintf(wq->name, "%s", pos);958 kfree(input);959 return count;960}961 962static struct device_attribute dev_attr_wq_name =963 __ATTR(name, 0644, wq_name_show, wq_name_store);964 965static ssize_t wq_cdev_minor_show(struct device *dev,966 struct device_attribute *attr, char *buf)967{968 struct idxd_wq *wq = confdev_to_wq(dev);969 int minor = -1;970 971 mutex_lock(&wq->wq_lock);972 if (wq->idxd_cdev)973 minor = wq->idxd_cdev->minor;974 mutex_unlock(&wq->wq_lock);975 976 if (minor == -1)977 return -ENXIO;978 return sysfs_emit(buf, "%d\n", minor);979}980 981static struct device_attribute dev_attr_wq_cdev_minor =982 __ATTR(cdev_minor, 0444, wq_cdev_minor_show, NULL);983 984static int __get_sysfs_u64(const char *buf, u64 *val)985{986 int rc;987 988 rc = kstrtou64(buf, 0, val);989 if (rc < 0)990 return -EINVAL;991 992 if (*val == 0)993 return -EINVAL;994 995 *val = roundup_pow_of_two(*val);996 return 0;997}998 999static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attribute *attr,1000 char *buf)1001{1002 struct idxd_wq *wq = confdev_to_wq(dev);1003 1004 return sysfs_emit(buf, "%llu\n", wq->max_xfer_bytes);1005}1006 1007static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr,1008 const char *buf, size_t count)1009{1010 struct idxd_wq *wq = confdev_to_wq(dev);1011 struct idxd_device *idxd = wq->idxd;1012 u64 xfer_size;1013 int rc;1014 1015 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))1016 return -EPERM;1017 1018 if (wq->state != IDXD_WQ_DISABLED)1019 return -EPERM;1020 1021 rc = __get_sysfs_u64(buf, &xfer_size);1022 if (rc < 0)1023 return rc;1024 1025 if (xfer_size > idxd->max_xfer_bytes)1026 return -EINVAL;1027 1028 wq->max_xfer_bytes = xfer_size;1029 1030 return count;1031}1032 1033static struct device_attribute dev_attr_wq_max_transfer_size =1034 __ATTR(max_transfer_size, 0644,1035 wq_max_transfer_size_show, wq_max_transfer_size_store);1036 1037static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf)1038{1039 struct idxd_wq *wq = confdev_to_wq(dev);1040 1041 return sysfs_emit(buf, "%u\n", wq->max_batch_size);1042}1043 1044static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr,1045 const char *buf, size_t count)1046{1047 struct idxd_wq *wq = confdev_to_wq(dev);1048 struct idxd_device *idxd = wq->idxd;1049 u64 batch_size;1050 int rc;1051 1052 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))1053 return -EPERM;1054 1055 if (wq->state != IDXD_WQ_DISABLED)1056 return -EPERM;1057 1058 rc = __get_sysfs_u64(buf, &batch_size);1059 if (rc < 0)1060 return rc;1061 1062 if (batch_size > idxd->max_batch_size)1063 return -EINVAL;1064 1065 idxd_wq_set_max_batch_size(idxd->data->type, wq, (u32)batch_size);1066 1067 return count;1068}1069 1070static struct device_attribute dev_attr_wq_max_batch_size =1071 __ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store);1072 1073static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf)1074{1075 struct idxd_wq *wq = confdev_to_wq(dev);1076 1077 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags));1078}1079 1080static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr,1081 const char *buf, size_t count)1082{1083 struct idxd_wq *wq = confdev_to_wq(dev);1084 struct idxd_device *idxd = wq->idxd;1085 bool ats_dis;1086 int rc;1087 1088 if (wq->state != IDXD_WQ_DISABLED)1089 return -EPERM;1090 1091 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))1092 return -EPERM;1093 1094 rc = kstrtobool(buf, &ats_dis);1095 if (rc < 0)1096 return rc;1097 1098 if (ats_dis)1099 set_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);1100 else1101 clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);1102 1103 return count;1104}1105 1106static struct device_attribute dev_attr_wq_ats_disable =1107 __ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store);1108 1109static ssize_t wq_prs_disable_show(struct device *dev, struct device_attribute *attr, char *buf)1110{1111 struct idxd_wq *wq = confdev_to_wq(dev);1112 1113 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags));1114}1115 1116static ssize_t wq_prs_disable_store(struct device *dev, struct device_attribute *attr,1117 const char *buf, size_t count)1118{1119 struct idxd_wq *wq = confdev_to_wq(dev);1120 struct idxd_device *idxd = wq->idxd;1121 bool prs_dis;1122 int rc;1123 1124 if (wq->state != IDXD_WQ_DISABLED)1125 return -EPERM;1126 1127 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))1128 return -EPERM;1129 1130 rc = kstrtobool(buf, &prs_dis);1131 if (rc < 0)1132 return rc;1133 1134 if (prs_dis) {1135 set_bit(WQ_FLAG_PRS_DISABLE, &wq->flags);1136 /* when PRS is disabled, BOF needs to be off as well */1137 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);1138 } else {1139 clear_bit(WQ_FLAG_PRS_DISABLE, &wq->flags);1140 }1141 return count;1142}1143 1144static struct device_attribute dev_attr_wq_prs_disable =1145 __ATTR(prs_disable, 0644, wq_prs_disable_show, wq_prs_disable_store);1146 1147static ssize_t wq_occupancy_show(struct device *dev, struct device_attribute *attr, char *buf)1148{1149 struct idxd_wq *wq = confdev_to_wq(dev);1150 struct idxd_device *idxd = wq->idxd;1151 u32 occup, offset;1152 1153 if (!idxd->hw.wq_cap.occupancy)1154 return -EOPNOTSUPP;1155 1156 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_OCCUP_IDX);1157 occup = ioread32(idxd->reg_base + offset) & WQCFG_OCCUP_MASK;1158 1159 return sysfs_emit(buf, "%u\n", occup);1160}1161 1162static struct device_attribute dev_attr_wq_occupancy =1163 __ATTR(occupancy, 0444, wq_occupancy_show, NULL);1164 1165static ssize_t wq_enqcmds_retries_show(struct device *dev,1166 struct device_attribute *attr, char *buf)1167{1168 struct idxd_wq *wq = confdev_to_wq(dev);1169 1170 if (wq_dedicated(wq))1171 return -EOPNOTSUPP;1172 1173 return sysfs_emit(buf, "%u\n", wq->enqcmds_retries);1174}1175 1176static ssize_t wq_enqcmds_retries_store(struct device *dev, struct device_attribute *attr,1177 const char *buf, size_t count)1178{1179 struct idxd_wq *wq = confdev_to_wq(dev);1180 int rc;1181 unsigned int retries;1182 1183 if (wq_dedicated(wq))1184 return -EOPNOTSUPP;1185 1186 rc = kstrtouint(buf, 10, &retries);1187 if (rc < 0)1188 return rc;1189 1190 if (retries > IDXD_ENQCMDS_MAX_RETRIES)1191 retries = IDXD_ENQCMDS_MAX_RETRIES;1192 1193 wq->enqcmds_retries = retries;1194 return count;1195}1196 1197static struct device_attribute dev_attr_wq_enqcmds_retries =1198 __ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store);1199 1200static ssize_t op_cap_show_common(struct device *dev, char *buf, unsigned long *opcap_bmap)1201{1202 ssize_t pos;1203 int i;1204 1205 pos = 0;1206 for (i = IDXD_MAX_OPCAP_BITS/64 - 1; i >= 0; i--) {1207 unsigned long val = opcap_bmap[i];1208 1209 /* On systems where direct user submissions are not safe, we need to clear out1210 * the BATCH capability from the capability mask in sysfs since we cannot support1211 * that command on such systems.1212 */1213 if (i == DSA_OPCODE_BATCH/64 && !confdev_to_idxd(dev)->user_submission_safe)1214 clear_bit(DSA_OPCODE_BATCH % 64, &val);1215 1216 pos += sysfs_emit_at(buf, pos, "%*pb", 64, &val);1217 pos += sysfs_emit_at(buf, pos, "%c", i == 0 ? '\n' : ',');1218 }1219 1220 return pos;1221}1222 1223static ssize_t wq_op_config_show(struct device *dev,1224 struct device_attribute *attr, char *buf)1225{1226 struct idxd_wq *wq = confdev_to_wq(dev);1227 1228 return op_cap_show_common(dev, buf, wq->opcap_bmap);1229}1230 1231static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask)1232{1233 int bit;1234 1235 /*1236 * The OPCAP is defined as 256 bits that represents each operation the device1237 * supports per bit. Iterate through all the bits and check if the input mask1238 * is set for bits that are not set in the OPCAP for the device. If no OPCAP1239 * bit is set and input mask has the bit set, then return error.1240 */1241 for_each_set_bit(bit, opmask, IDXD_MAX_OPCAP_BITS) {1242 if (!test_bit(bit, idxd->opcap_bmap))1243 return -EINVAL;1244 }1245 1246 return 0;1247}1248 1249static ssize_t wq_op_config_store(struct device *dev, struct device_attribute *attr,1250 const char *buf, size_t count)1251{1252 struct idxd_wq *wq = confdev_to_wq(dev);1253 struct idxd_device *idxd = wq->idxd;1254 unsigned long *opmask;1255 int rc;1256 1257 if (wq->state != IDXD_WQ_DISABLED)1258 return -EPERM;1259 1260 opmask = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL);1261 if (!opmask)1262 return -ENOMEM;1263 1264 rc = bitmap_parse(buf, count, opmask, IDXD_MAX_OPCAP_BITS);1265 if (rc < 0)1266 goto err;1267 1268 rc = idxd_verify_supported_opcap(idxd, opmask);1269 if (rc < 0)1270 goto err;1271 1272 bitmap_copy(wq->opcap_bmap, opmask, IDXD_MAX_OPCAP_BITS);1273 1274 bitmap_free(opmask);1275 return count;1276 1277err:1278 bitmap_free(opmask);1279 return rc;1280}1281 1282static struct device_attribute dev_attr_wq_op_config =1283 __ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store);1284 1285static ssize_t wq_driver_name_show(struct device *dev, struct device_attribute *attr, char *buf)1286{1287 struct idxd_wq *wq = confdev_to_wq(dev);1288 1289 return sysfs_emit(buf, "%s\n", wq->driver_name);1290}1291 1292static ssize_t wq_driver_name_store(struct device *dev, struct device_attribute *attr,1293 const char *buf, size_t count)1294{1295 struct idxd_wq *wq = confdev_to_wq(dev);1296 char *input, *pos;1297 1298 if (wq->state != IDXD_WQ_DISABLED)1299 return -EPERM;1300 1301 if (strlen(buf) > DRIVER_NAME_SIZE || strlen(buf) == 0)1302 return -EINVAL;1303 1304 input = kstrndup(buf, count, GFP_KERNEL);1305 if (!input)1306 return -ENOMEM;1307 1308 pos = strim(input);1309 memset(wq->driver_name, 0, DRIVER_NAME_SIZE + 1);1310 sprintf(wq->driver_name, "%s", pos);1311 kfree(input);1312 return count;1313}1314 1315static struct device_attribute dev_attr_wq_driver_name =1316 __ATTR(driver_name, 0644, wq_driver_name_show, wq_driver_name_store);1317 1318static struct attribute *idxd_wq_attributes[] = {1319 &dev_attr_wq_clients.attr,1320 &dev_attr_wq_state.attr,1321 &dev_attr_wq_group_id.attr,1322 &dev_attr_wq_mode.attr,1323 &dev_attr_wq_size.attr,1324 &dev_attr_wq_priority.attr,1325 &dev_attr_wq_block_on_fault.attr,1326 &dev_attr_wq_threshold.attr,1327 &dev_attr_wq_type.attr,1328 &dev_attr_wq_name.attr,1329 &dev_attr_wq_cdev_minor.attr,1330 &dev_attr_wq_max_transfer_size.attr,1331 &dev_attr_wq_max_batch_size.attr,1332 &dev_attr_wq_ats_disable.attr,1333 &dev_attr_wq_prs_disable.attr,1334 &dev_attr_wq_occupancy.attr,1335 &dev_attr_wq_enqcmds_retries.attr,1336 &dev_attr_wq_op_config.attr,1337 &dev_attr_wq_driver_name.attr,1338 NULL,1339};1340 1341/* A WQ attr is invisible if the feature is not supported in WQCAP. */1342#define idxd_wq_attr_invisible(name, cap_field, a, idxd) \1343 ((a) == &dev_attr_wq_##name.attr && !(idxd)->hw.wq_cap.cap_field)1344 1345static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr,1346 struct idxd_device *idxd)1347{1348 /* Intel IAA does not support batch processing, make it invisible */1349 return attr == &dev_attr_wq_max_batch_size.attr &&1350 idxd->data->type == IDXD_TYPE_IAX;1351}1352 1353static umode_t idxd_wq_attr_visible(struct kobject *kobj,1354 struct attribute *attr, int n)1355{1356 struct device *dev = container_of(kobj, struct device, kobj);1357 struct idxd_wq *wq = confdev_to_wq(dev);1358 struct idxd_device *idxd = wq->idxd;1359 1360 if (idxd_wq_attr_invisible(op_config, op_config, attr, idxd))1361 return 0;1362 1363 if (idxd_wq_attr_max_batch_size_invisible(attr, idxd))1364 return 0;1365 1366 if (idxd_wq_attr_invisible(prs_disable, wq_prs_support, attr, idxd))1367 return 0;1368 1369 if (idxd_wq_attr_invisible(ats_disable, wq_ats_support, attr, idxd))1370 return 0;1371 1372 return attr->mode;1373}1374 1375static const struct attribute_group idxd_wq_attribute_group = {1376 .attrs = idxd_wq_attributes,1377 .is_visible = idxd_wq_attr_visible,1378};1379 1380static const struct attribute_group *idxd_wq_attribute_groups[] = {1381 &idxd_wq_attribute_group,1382 NULL,1383};1384 1385static void idxd_conf_wq_release(struct device *dev)1386{1387 struct idxd_wq *wq = confdev_to_wq(dev);1388 1389 bitmap_free(wq->opcap_bmap);1390 kfree(wq->wqcfg);1391 xa_destroy(&wq->upasid_xa);1392 kfree(wq);1393}1394 1395const struct device_type idxd_wq_device_type = {1396 .name = "wq",1397 .release = idxd_conf_wq_release,1398 .groups = idxd_wq_attribute_groups,1399};1400 1401/* IDXD device attribs */1402static ssize_t version_show(struct device *dev, struct device_attribute *attr,1403 char *buf)1404{1405 struct idxd_device *idxd = confdev_to_idxd(dev);1406 1407 return sysfs_emit(buf, "%#x\n", idxd->hw.version);1408}1409static DEVICE_ATTR_RO(version);1410 1411static ssize_t max_work_queues_size_show(struct device *dev,1412 struct device_attribute *attr,1413 char *buf)1414{1415 struct idxd_device *idxd = confdev_to_idxd(dev);1416 1417 return sysfs_emit(buf, "%u\n", idxd->max_wq_size);1418}1419static DEVICE_ATTR_RO(max_work_queues_size);1420 1421static ssize_t max_groups_show(struct device *dev,1422 struct device_attribute *attr, char *buf)1423{1424 struct idxd_device *idxd = confdev_to_idxd(dev);1425 1426 return sysfs_emit(buf, "%u\n", idxd->max_groups);1427}1428static DEVICE_ATTR_RO(max_groups);1429 1430static ssize_t max_work_queues_show(struct device *dev,1431 struct device_attribute *attr, char *buf)1432{1433 struct idxd_device *idxd = confdev_to_idxd(dev);1434 1435 return sysfs_emit(buf, "%u\n", idxd->max_wqs);1436}1437static DEVICE_ATTR_RO(max_work_queues);1438 1439static ssize_t max_engines_show(struct device *dev,1440 struct device_attribute *attr, char *buf)1441{1442 struct idxd_device *idxd = confdev_to_idxd(dev);1443 1444 return sysfs_emit(buf, "%u\n", idxd->max_engines);1445}1446static DEVICE_ATTR_RO(max_engines);1447 1448static ssize_t numa_node_show(struct device *dev,1449 struct device_attribute *attr, char *buf)1450{1451 struct idxd_device *idxd = confdev_to_idxd(dev);1452 1453 return sysfs_emit(buf, "%d\n", dev_to_node(&idxd->pdev->dev));1454}1455static DEVICE_ATTR_RO(numa_node);1456 1457static ssize_t max_batch_size_show(struct device *dev,1458 struct device_attribute *attr, char *buf)1459{1460 struct idxd_device *idxd = confdev_to_idxd(dev);1461 1462 return sysfs_emit(buf, "%u\n", idxd->max_batch_size);1463}1464static DEVICE_ATTR_RO(max_batch_size);1465 1466static ssize_t max_transfer_size_show(struct device *dev,1467 struct device_attribute *attr,1468 char *buf)1469{1470 struct idxd_device *idxd = confdev_to_idxd(dev);1471 1472 return sysfs_emit(buf, "%llu\n", idxd->max_xfer_bytes);1473}1474static DEVICE_ATTR_RO(max_transfer_size);1475 1476static ssize_t op_cap_show(struct device *dev,1477 struct device_attribute *attr, char *buf)1478{1479 struct idxd_device *idxd = confdev_to_idxd(dev);1480 1481 return op_cap_show_common(dev, buf, idxd->opcap_bmap);1482}1483static DEVICE_ATTR_RO(op_cap);1484 1485static ssize_t gen_cap_show(struct device *dev,1486 struct device_attribute *attr, char *buf)1487{1488 struct idxd_device *idxd = confdev_to_idxd(dev);1489 1490 return sysfs_emit(buf, "%#llx\n", idxd->hw.gen_cap.bits);1491}1492static DEVICE_ATTR_RO(gen_cap);1493 1494static ssize_t configurable_show(struct device *dev,1495 struct device_attribute *attr, char *buf)1496{1497 struct idxd_device *idxd = confdev_to_idxd(dev);1498 1499 return sysfs_emit(buf, "%u\n", test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags));1500}1501static DEVICE_ATTR_RO(configurable);1502 1503static ssize_t clients_show(struct device *dev,1504 struct device_attribute *attr, char *buf)1505{1506 struct idxd_device *idxd = confdev_to_idxd(dev);1507 int count = 0, i;1508 1509 spin_lock(&idxd->dev_lock);1510 for (i = 0; i < idxd->max_wqs; i++) {1511 struct idxd_wq *wq = idxd->wqs[i];1512 1513 count += wq->client_count;1514 }1515 spin_unlock(&idxd->dev_lock);1516 1517 return sysfs_emit(buf, "%d\n", count);1518}1519static DEVICE_ATTR_RO(clients);1520 1521static ssize_t pasid_enabled_show(struct device *dev,1522 struct device_attribute *attr, char *buf)1523{1524 struct idxd_device *idxd = confdev_to_idxd(dev);1525 1526 return sysfs_emit(buf, "%u\n", device_user_pasid_enabled(idxd));1527}1528static DEVICE_ATTR_RO(pasid_enabled);1529 1530static ssize_t state_show(struct device *dev,1531 struct device_attribute *attr, char *buf)1532{1533 struct idxd_device *idxd = confdev_to_idxd(dev);1534 1535 switch (idxd->state) {1536 case IDXD_DEV_DISABLED:1537 return sysfs_emit(buf, "disabled\n");1538 case IDXD_DEV_ENABLED:1539 return sysfs_emit(buf, "enabled\n");1540 case IDXD_DEV_HALTED:1541 return sysfs_emit(buf, "halted\n");1542 }1543 1544 return sysfs_emit(buf, "unknown\n");1545}1546static DEVICE_ATTR_RO(state);1547 1548static ssize_t errors_show(struct device *dev,1549 struct device_attribute *attr, char *buf)1550{1551 struct idxd_device *idxd = confdev_to_idxd(dev);1552 DECLARE_BITMAP(swerr_bmap, 256);1553 1554 bitmap_zero(swerr_bmap, 256);1555 spin_lock(&idxd->dev_lock);1556 multi_u64_to_bmap(swerr_bmap, &idxd->sw_err.bits[0], 4);1557 spin_unlock(&idxd->dev_lock);1558 return sysfs_emit(buf, "%*pb\n", 256, swerr_bmap);1559}1560static DEVICE_ATTR_RO(errors);1561 1562static ssize_t max_read_buffers_show(struct device *dev,1563 struct device_attribute *attr, char *buf)1564{1565 struct idxd_device *idxd = confdev_to_idxd(dev);1566 1567 return sysfs_emit(buf, "%u\n", idxd->max_rdbufs);1568}1569 1570static ssize_t max_tokens_show(struct device *dev,1571 struct device_attribute *attr, char *buf)1572{1573 dev_warn_once(dev, "attribute deprecated, see max_read_buffers.\n");1574 return max_read_buffers_show(dev, attr, buf);1575}1576 1577static DEVICE_ATTR_RO(max_tokens); /* deprecated */1578static DEVICE_ATTR_RO(max_read_buffers);1579 1580static ssize_t read_buffer_limit_show(struct device *dev,1581 struct device_attribute *attr, char *buf)1582{1583 struct idxd_device *idxd = confdev_to_idxd(dev);1584 1585 return sysfs_emit(buf, "%u\n", idxd->rdbuf_limit);1586}1587 1588static ssize_t token_limit_show(struct device *dev,1589 struct device_attribute *attr, char *buf)1590{1591 dev_warn_once(dev, "attribute deprecated, see read_buffer_limit.\n");1592 return read_buffer_limit_show(dev, attr, buf);1593}1594 1595static ssize_t read_buffer_limit_store(struct device *dev,1596 struct device_attribute *attr,1597 const char *buf, size_t count)1598{1599 struct idxd_device *idxd = confdev_to_idxd(dev);1600 unsigned long val;1601 int rc;1602 1603 rc = kstrtoul(buf, 10, &val);1604 if (rc < 0)1605 return -EINVAL;1606 1607 if (idxd->state == IDXD_DEV_ENABLED)1608 return -EPERM;1609 1610 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))1611 return -EPERM;1612 1613 if (!idxd->hw.group_cap.rdbuf_limit)1614 return -EPERM;1615 1616 if (val > idxd->hw.group_cap.total_rdbufs)1617 return -EINVAL;1618 1619 idxd->rdbuf_limit = val;1620 return count;1621}1622 1623static ssize_t token_limit_store(struct device *dev,1624 struct device_attribute *attr,1625 const char *buf, size_t count)1626{1627 dev_warn_once(dev, "attribute deprecated, see read_buffer_limit\n");1628 return read_buffer_limit_store(dev, attr, buf, count);1629}1630 1631static DEVICE_ATTR_RW(token_limit); /* deprecated */1632static DEVICE_ATTR_RW(read_buffer_limit);1633 1634static ssize_t cdev_major_show(struct device *dev,1635 struct device_attribute *attr, char *buf)1636{1637 struct idxd_device *idxd = confdev_to_idxd(dev);1638 1639 return sysfs_emit(buf, "%u\n", idxd->major);1640}1641static DEVICE_ATTR_RO(cdev_major);1642 1643static ssize_t cmd_status_show(struct device *dev,1644 struct device_attribute *attr, char *buf)1645{1646 struct idxd_device *idxd = confdev_to_idxd(dev);1647 1648 return sysfs_emit(buf, "%#x\n", idxd->cmd_status);1649}1650 1651static ssize_t cmd_status_store(struct device *dev, struct device_attribute *attr,1652 const char *buf, size_t count)1653{1654 struct idxd_device *idxd = confdev_to_idxd(dev);1655 1656 idxd->cmd_status = 0;1657 return count;1658}1659static DEVICE_ATTR_RW(cmd_status);1660 1661static ssize_t iaa_cap_show(struct device *dev,1662 struct device_attribute *attr, char *buf)1663{1664 struct idxd_device *idxd = confdev_to_idxd(dev);1665 1666 if (idxd->hw.version < DEVICE_VERSION_2)1667 return -EOPNOTSUPP;1668 1669 return sysfs_emit(buf, "%#llx\n", idxd->hw.iaa_cap.bits);1670}1671static DEVICE_ATTR_RO(iaa_cap);1672 1673static ssize_t event_log_size_show(struct device *dev,1674 struct device_attribute *attr, char *buf)1675{1676 struct idxd_device *idxd = confdev_to_idxd(dev);1677 1678 if (!idxd->evl)1679 return -EOPNOTSUPP;1680 1681 return sysfs_emit(buf, "%u\n", idxd->evl->size);1682}1683 1684static ssize_t event_log_size_store(struct device *dev,1685 struct device_attribute *attr,1686 const char *buf, size_t count)1687{1688 struct idxd_device *idxd = confdev_to_idxd(dev);1689 unsigned long val;1690 int rc;1691 1692 if (!idxd->evl)1693 return -EOPNOTSUPP;1694 1695 rc = kstrtoul(buf, 10, &val);1696 if (rc < 0)1697 return -EINVAL;1698 1699 if (idxd->state == IDXD_DEV_ENABLED)1700 return -EPERM;1701 1702 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))1703 return -EPERM;1704 1705 if (val < IDXD_EVL_SIZE_MIN || val > IDXD_EVL_SIZE_MAX ||1706 (val * evl_ent_size(idxd) > ULONG_MAX - idxd->evl->dma))1707 return -EINVAL;1708 1709 idxd->evl->size = val;1710 return count;1711}1712static DEVICE_ATTR_RW(event_log_size);1713 1714static bool idxd_device_attr_max_batch_size_invisible(struct attribute *attr,1715 struct idxd_device *idxd)1716{1717 /* Intel IAA does not support batch processing, make it invisible */1718 return attr == &dev_attr_max_batch_size.attr &&1719 idxd->data->type == IDXD_TYPE_IAX;1720}1721 1722static bool idxd_device_attr_read_buffers_invisible(struct attribute *attr,1723 struct idxd_device *idxd)1724{1725 /*1726 * Intel IAA does not support Read Buffer allocation control,1727 * make these attributes invisible.1728 */1729 return (attr == &dev_attr_max_tokens.attr ||1730 attr == &dev_attr_max_read_buffers.attr ||1731 attr == &dev_attr_token_limit.attr ||1732 attr == &dev_attr_read_buffer_limit.attr) &&1733 idxd->data->type == IDXD_TYPE_IAX;1734}1735 1736static bool idxd_device_attr_iaa_cap_invisible(struct attribute *attr,1737 struct idxd_device *idxd)1738{1739 return attr == &dev_attr_iaa_cap.attr &&1740 (idxd->data->type != IDXD_TYPE_IAX ||1741 idxd->hw.version < DEVICE_VERSION_2);1742}1743 1744static bool idxd_device_attr_event_log_size_invisible(struct attribute *attr,1745 struct idxd_device *idxd)1746{1747 return (attr == &dev_attr_event_log_size.attr &&1748 !idxd->hw.gen_cap.evl_support);1749}1750 1751static umode_t idxd_device_attr_visible(struct kobject *kobj,1752 struct attribute *attr, int n)1753{1754 struct device *dev = container_of(kobj, struct device, kobj);1755 struct idxd_device *idxd = confdev_to_idxd(dev);1756 1757 if (idxd_device_attr_max_batch_size_invisible(attr, idxd))1758 return 0;1759 1760 if (idxd_device_attr_read_buffers_invisible(attr, idxd))1761 return 0;1762 1763 if (idxd_device_attr_iaa_cap_invisible(attr, idxd))1764 return 0;1765 1766 if (idxd_device_attr_event_log_size_invisible(attr, idxd))1767 return 0;1768 1769 return attr->mode;1770}1771 1772static struct attribute *idxd_device_attributes[] = {1773 &dev_attr_version.attr,1774 &dev_attr_max_groups.attr,1775 &dev_attr_max_work_queues.attr,1776 &dev_attr_max_work_queues_size.attr,1777 &dev_attr_max_engines.attr,1778 &dev_attr_numa_node.attr,1779 &dev_attr_max_batch_size.attr,1780 &dev_attr_max_transfer_size.attr,1781 &dev_attr_op_cap.attr,1782 &dev_attr_gen_cap.attr,1783 &dev_attr_configurable.attr,1784 &dev_attr_clients.attr,1785 &dev_attr_pasid_enabled.attr,1786 &dev_attr_state.attr,1787 &dev_attr_errors.attr,1788 &dev_attr_max_tokens.attr,1789 &dev_attr_max_read_buffers.attr,1790 &dev_attr_token_limit.attr,1791 &dev_attr_read_buffer_limit.attr,1792 &dev_attr_cdev_major.attr,1793 &dev_attr_cmd_status.attr,1794 &dev_attr_iaa_cap.attr,1795 &dev_attr_event_log_size.attr,1796 NULL,1797};1798 1799static const struct attribute_group idxd_device_attribute_group = {1800 .attrs = idxd_device_attributes,1801 .is_visible = idxd_device_attr_visible,1802};1803 1804static const struct attribute_group *idxd_attribute_groups[] = {1805 &idxd_device_attribute_group,1806 NULL,1807};1808 1809static void idxd_conf_device_release(struct device *dev)1810{1811 struct idxd_device *idxd = confdev_to_idxd(dev);1812 1813 kfree(idxd->groups);1814 bitmap_free(idxd->wq_enable_map);1815 kfree(idxd->wqs);1816 kfree(idxd->engines);1817 kfree(idxd->evl);1818 kmem_cache_destroy(idxd->evl_cache);1819 ida_free(&idxd_ida, idxd->id);1820 bitmap_free(idxd->opcap_bmap);1821 kfree(idxd);1822}1823 1824const struct device_type dsa_device_type = {1825 .name = "dsa",1826 .release = idxd_conf_device_release,1827 .groups = idxd_attribute_groups,1828};1829 1830const struct device_type iax_device_type = {1831 .name = "iax",1832 .release = idxd_conf_device_release,1833 .groups = idxd_attribute_groups,1834};1835 1836static int idxd_register_engine_devices(struct idxd_device *idxd)1837{1838 struct idxd_engine *engine;1839 int i, j, rc;1840 1841 for (i = 0; i < idxd->max_engines; i++) {1842 engine = idxd->engines[i];1843 rc = device_add(engine_confdev(engine));1844 if (rc < 0)1845 goto cleanup;1846 }1847 1848 return 0;1849 1850cleanup:1851 j = i - 1;1852 for (; i < idxd->max_engines; i++) {1853 engine = idxd->engines[i];1854 put_device(engine_confdev(engine));1855 }1856 1857 while (j--) {1858 engine = idxd->engines[j];1859 device_unregister(engine_confdev(engine));1860 }1861 return rc;1862}1863 1864static int idxd_register_group_devices(struct idxd_device *idxd)1865{1866 struct idxd_group *group;1867 int i, j, rc;1868 1869 for (i = 0; i < idxd->max_groups; i++) {1870 group = idxd->groups[i];1871 rc = device_add(group_confdev(group));1872 if (rc < 0)1873 goto cleanup;1874 }1875 1876 return 0;1877 1878cleanup:1879 j = i - 1;1880 for (; i < idxd->max_groups; i++) {1881 group = idxd->groups[i];1882 put_device(group_confdev(group));1883 }1884 1885 while (j--) {1886 group = idxd->groups[j];1887 device_unregister(group_confdev(group));1888 }1889 return rc;1890}1891 1892static int idxd_register_wq_devices(struct idxd_device *idxd)1893{1894 struct idxd_wq *wq;1895 int i, rc, j;1896 1897 for (i = 0; i < idxd->max_wqs; i++) {1898 wq = idxd->wqs[i];1899 rc = device_add(wq_confdev(wq));1900 if (rc < 0)1901 goto cleanup;1902 }1903 1904 return 0;1905 1906cleanup:1907 j = i - 1;1908 for (; i < idxd->max_wqs; i++) {1909 wq = idxd->wqs[i];1910 put_device(wq_confdev(wq));1911 }1912 1913 while (j--) {1914 wq = idxd->wqs[j];1915 device_unregister(wq_confdev(wq));1916 }1917 return rc;1918}1919 1920int idxd_register_devices(struct idxd_device *idxd)1921{1922 struct device *dev = &idxd->pdev->dev;1923 int rc, i;1924 1925 rc = device_add(idxd_confdev(idxd));1926 if (rc < 0)1927 return rc;1928 1929 rc = idxd_register_wq_devices(idxd);1930 if (rc < 0) {1931 dev_dbg(dev, "WQ devices registering failed: %d\n", rc);1932 goto err_wq;1933 }1934 1935 rc = idxd_register_engine_devices(idxd);1936 if (rc < 0) {1937 dev_dbg(dev, "Engine devices registering failed: %d\n", rc);1938 goto err_engine;1939 }1940 1941 rc = idxd_register_group_devices(idxd);1942 if (rc < 0) {1943 dev_dbg(dev, "Group device registering failed: %d\n", rc);1944 goto err_group;1945 }1946 1947 return 0;1948 1949 err_group:1950 for (i = 0; i < idxd->max_engines; i++)1951 device_unregister(engine_confdev(idxd->engines[i]));1952 err_engine:1953 for (i = 0; i < idxd->max_wqs; i++)1954 device_unregister(wq_confdev(idxd->wqs[i]));1955 err_wq:1956 device_del(idxd_confdev(idxd));1957 return rc;1958}1959 1960void idxd_unregister_devices(struct idxd_device *idxd)1961{1962 int i;1963 1964 for (i = 0; i < idxd->max_wqs; i++) {1965 struct idxd_wq *wq = idxd->wqs[i];1966 1967 device_unregister(wq_confdev(wq));1968 }1969 1970 for (i = 0; i < idxd->max_engines; i++) {1971 struct idxd_engine *engine = idxd->engines[i];1972 1973 device_unregister(engine_confdev(engine));1974 }1975 1976 for (i = 0; i < idxd->max_groups; i++) {1977 struct idxd_group *group = idxd->groups[i];1978 1979 device_unregister(group_confdev(group));1980 }1981}1982 1983int idxd_register_bus_type(void)1984{1985 return bus_register(&dsa_bus_type);1986}1987 1988void idxd_unregister_bus_type(void)1989{1990 bus_unregister(&dsa_bus_type);1991}1992