623 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2#include <linux/virtio.h>3#include <linux/spinlock.h>4#include <linux/virtio_config.h>5#include <linux/virtio_anchor.h>6#include <linux/module.h>7#include <linux/idr.h>8#include <linux/of.h>9#include <uapi/linux/virtio_ids.h>10 11/* Unique numbering for virtio devices. */12static DEFINE_IDA(virtio_index_ida);13 14static ssize_t device_show(struct device *_d,15 struct device_attribute *attr, char *buf)16{17 struct virtio_device *dev = dev_to_virtio(_d);18 return sysfs_emit(buf, "0x%04x\n", dev->id.device);19}20static DEVICE_ATTR_RO(device);21 22static ssize_t vendor_show(struct device *_d,23 struct device_attribute *attr, char *buf)24{25 struct virtio_device *dev = dev_to_virtio(_d);26 return sysfs_emit(buf, "0x%04x\n", dev->id.vendor);27}28static DEVICE_ATTR_RO(vendor);29 30static ssize_t status_show(struct device *_d,31 struct device_attribute *attr, char *buf)32{33 struct virtio_device *dev = dev_to_virtio(_d);34 return sysfs_emit(buf, "0x%08x\n", dev->config->get_status(dev));35}36static DEVICE_ATTR_RO(status);37 38static ssize_t modalias_show(struct device *_d,39 struct device_attribute *attr, char *buf)40{41 struct virtio_device *dev = dev_to_virtio(_d);42 return sysfs_emit(buf, "virtio:d%08Xv%08X\n",43 dev->id.device, dev->id.vendor);44}45static DEVICE_ATTR_RO(modalias);46 47static ssize_t features_show(struct device *_d,48 struct device_attribute *attr, char *buf)49{50 struct virtio_device *dev = dev_to_virtio(_d);51 unsigned int i;52 ssize_t len = 0;53 54 /* We actually represent this as a bitstring, as it could be55 * arbitrary length in future. */56 for (i = 0; i < sizeof(dev->features)*8; i++)57 len += sysfs_emit_at(buf, len, "%c",58 __virtio_test_bit(dev, i) ? '1' : '0');59 len += sysfs_emit_at(buf, len, "\n");60 return len;61}62static DEVICE_ATTR_RO(features);63 64static struct attribute *virtio_dev_attrs[] = {65 &dev_attr_device.attr,66 &dev_attr_vendor.attr,67 &dev_attr_status.attr,68 &dev_attr_modalias.attr,69 &dev_attr_features.attr,70 NULL,71};72ATTRIBUTE_GROUPS(virtio_dev);73 74static inline int virtio_id_match(const struct virtio_device *dev,75 const struct virtio_device_id *id)76{77 if (id->device != dev->id.device && id->device != VIRTIO_DEV_ANY_ID)78 return 0;79 80 return id->vendor == VIRTIO_DEV_ANY_ID || id->vendor == dev->id.vendor;81}82 83/* This looks through all the IDs a driver claims to support. If any of them84 * match, we return 1 and the kernel will call virtio_dev_probe(). */85static int virtio_dev_match(struct device *_dv, const struct device_driver *_dr)86{87 unsigned int i;88 struct virtio_device *dev = dev_to_virtio(_dv);89 const struct virtio_device_id *ids;90 91 ids = drv_to_virtio(_dr)->id_table;92 for (i = 0; ids[i].device; i++)93 if (virtio_id_match(dev, &ids[i]))94 return 1;95 return 0;96}97 98static int virtio_uevent(const struct device *_dv, struct kobj_uevent_env *env)99{100 const struct virtio_device *dev = dev_to_virtio(_dv);101 102 return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",103 dev->id.device, dev->id.vendor);104}105 106void virtio_check_driver_offered_feature(const struct virtio_device *vdev,107 unsigned int fbit)108{109 unsigned int i;110 struct virtio_driver *drv = drv_to_virtio(vdev->dev.driver);111 112 for (i = 0; i < drv->feature_table_size; i++)113 if (drv->feature_table[i] == fbit)114 return;115 116 if (drv->feature_table_legacy) {117 for (i = 0; i < drv->feature_table_size_legacy; i++)118 if (drv->feature_table_legacy[i] == fbit)119 return;120 }121 122 BUG();123}124EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature);125 126static void __virtio_config_changed(struct virtio_device *dev)127{128 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);129 130 if (!dev->config_core_enabled || dev->config_driver_disabled)131 dev->config_change_pending = true;132 else if (drv && drv->config_changed) {133 drv->config_changed(dev);134 dev->config_change_pending = false;135 }136}137 138void virtio_config_changed(struct virtio_device *dev)139{140 unsigned long flags;141 142 spin_lock_irqsave(&dev->config_lock, flags);143 __virtio_config_changed(dev);144 spin_unlock_irqrestore(&dev->config_lock, flags);145}146EXPORT_SYMBOL_GPL(virtio_config_changed);147 148/**149 * virtio_config_driver_disable - disable config change reporting by drivers150 * @dev: the device to reset151 *152 * This is only allowed to be called by a driver and disabling can't153 * be nested.154 */155void virtio_config_driver_disable(struct virtio_device *dev)156{157 spin_lock_irq(&dev->config_lock);158 dev->config_driver_disabled = true;159 spin_unlock_irq(&dev->config_lock);160}161EXPORT_SYMBOL_GPL(virtio_config_driver_disable);162 163/**164 * virtio_config_driver_enable - enable config change reporting by drivers165 * @dev: the device to reset166 *167 * This is only allowed to be called by a driver and enabling can't168 * be nested.169 */170void virtio_config_driver_enable(struct virtio_device *dev)171{172 spin_lock_irq(&dev->config_lock);173 dev->config_driver_disabled = false;174 if (dev->config_change_pending)175 __virtio_config_changed(dev);176 spin_unlock_irq(&dev->config_lock);177}178EXPORT_SYMBOL_GPL(virtio_config_driver_enable);179 180static void virtio_config_core_disable(struct virtio_device *dev)181{182 spin_lock_irq(&dev->config_lock);183 dev->config_core_enabled = false;184 spin_unlock_irq(&dev->config_lock);185}186 187static void virtio_config_core_enable(struct virtio_device *dev)188{189 spin_lock_irq(&dev->config_lock);190 dev->config_core_enabled = true;191 if (dev->config_change_pending)192 __virtio_config_changed(dev);193 spin_unlock_irq(&dev->config_lock);194}195 196void virtio_add_status(struct virtio_device *dev, unsigned int status)197{198 might_sleep();199 dev->config->set_status(dev, dev->config->get_status(dev) | status);200}201EXPORT_SYMBOL_GPL(virtio_add_status);202 203/* Do some validation, then set FEATURES_OK */204static int virtio_features_ok(struct virtio_device *dev)205{206 unsigned int status;207 208 might_sleep();209 210 if (virtio_check_mem_acc_cb(dev)) {211 if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) {212 dev_warn(&dev->dev,213 "device must provide VIRTIO_F_VERSION_1\n");214 return -ENODEV;215 }216 217 if (!virtio_has_feature(dev, VIRTIO_F_ACCESS_PLATFORM)) {218 dev_warn(&dev->dev,219 "device must provide VIRTIO_F_ACCESS_PLATFORM\n");220 return -ENODEV;221 }222 }223 224 if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1))225 return 0;226 227 virtio_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);228 status = dev->config->get_status(dev);229 if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {230 dev_err(&dev->dev, "virtio: device refuses features: %x\n",231 status);232 return -ENODEV;233 }234 return 0;235}236 237/**238 * virtio_reset_device - quiesce device for removal239 * @dev: the device to reset240 *241 * Prevents device from sending interrupts and accessing memory.242 *243 * Generally used for cleanup during driver / device removal.244 *245 * Once this has been invoked, caller must ensure that246 * virtqueue_notify / virtqueue_kick are not in progress.247 *248 * Note: this guarantees that vq callbacks are not in progress, however caller249 * is responsible for preventing access from other contexts, such as a system250 * call/workqueue/bh. Invoking virtio_break_device then flushing any such251 * contexts is one way to handle that.252 * */253void virtio_reset_device(struct virtio_device *dev)254{255#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION256 /*257 * The below virtio_synchronize_cbs() guarantees that any258 * interrupt for this line arriving after259 * virtio_synchronize_vqs() has completed is guaranteed to see260 * vq->broken as true.261 */262 virtio_break_device(dev);263 virtio_synchronize_cbs(dev);264#endif265 266 dev->config->reset(dev);267}268EXPORT_SYMBOL_GPL(virtio_reset_device);269 270static int virtio_dev_probe(struct device *_d)271{272 int err, i;273 struct virtio_device *dev = dev_to_virtio(_d);274 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);275 u64 device_features;276 u64 driver_features;277 u64 driver_features_legacy;278 279 /* We have a driver! */280 virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);281 282 /* Figure out what features the device supports. */283 device_features = dev->config->get_features(dev);284 285 /* Figure out what features the driver supports. */286 driver_features = 0;287 for (i = 0; i < drv->feature_table_size; i++) {288 unsigned int f = drv->feature_table[i];289 BUG_ON(f >= 64);290 driver_features |= (1ULL << f);291 }292 293 /* Some drivers have a separate feature table for virtio v1.0 */294 if (drv->feature_table_legacy) {295 driver_features_legacy = 0;296 for (i = 0; i < drv->feature_table_size_legacy; i++) {297 unsigned int f = drv->feature_table_legacy[i];298 BUG_ON(f >= 64);299 driver_features_legacy |= (1ULL << f);300 }301 } else {302 driver_features_legacy = driver_features;303 }304 305 if (device_features & (1ULL << VIRTIO_F_VERSION_1))306 dev->features = driver_features & device_features;307 else308 dev->features = driver_features_legacy & device_features;309 310 /* When debugging, user may filter some features by hand. */311 virtio_debug_device_filter_features(dev);312 313 /* Transport features always preserved to pass to finalize_features. */314 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++)315 if (device_features & (1ULL << i))316 __virtio_set_bit(dev, i);317 318 err = dev->config->finalize_features(dev);319 if (err)320 goto err;321 322 if (drv->validate) {323 u64 features = dev->features;324 325 err = drv->validate(dev);326 if (err)327 goto err;328 329 /* Did validation change any features? Then write them again. */330 if (features != dev->features) {331 err = dev->config->finalize_features(dev);332 if (err)333 goto err;334 }335 }336 337 err = virtio_features_ok(dev);338 if (err)339 goto err;340 341 err = drv->probe(dev);342 if (err)343 goto err;344 345 /* If probe didn't do it, mark device DRIVER_OK ourselves. */346 if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))347 virtio_device_ready(dev);348 349 if (drv->scan)350 drv->scan(dev);351 352 virtio_config_core_enable(dev);353 354 return 0;355 356err:357 virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);358 return err;359 360}361 362static void virtio_dev_remove(struct device *_d)363{364 struct virtio_device *dev = dev_to_virtio(_d);365 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);366 367 virtio_config_core_disable(dev);368 369 drv->remove(dev);370 371 /* Driver should have reset device. */372 WARN_ON_ONCE(dev->config->get_status(dev));373 374 /* Acknowledge the device's existence again. */375 virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);376 377 of_node_put(dev->dev.of_node);378}379 380static const struct bus_type virtio_bus = {381 .name = "virtio",382 .match = virtio_dev_match,383 .dev_groups = virtio_dev_groups,384 .uevent = virtio_uevent,385 .probe = virtio_dev_probe,386 .remove = virtio_dev_remove,387};388 389int __register_virtio_driver(struct virtio_driver *driver, struct module *owner)390{391 /* Catch this early. */392 BUG_ON(driver->feature_table_size && !driver->feature_table);393 driver->driver.bus = &virtio_bus;394 driver->driver.owner = owner;395 396 return driver_register(&driver->driver);397}398EXPORT_SYMBOL_GPL(__register_virtio_driver);399 400void unregister_virtio_driver(struct virtio_driver *driver)401{402 driver_unregister(&driver->driver);403}404EXPORT_SYMBOL_GPL(unregister_virtio_driver);405 406static int virtio_device_of_init(struct virtio_device *dev)407{408 struct device_node *np, *pnode = dev_of_node(dev->dev.parent);409 char compat[] = "virtio,deviceXXXXXXXX";410 int ret, count;411 412 if (!pnode)413 return 0;414 415 count = of_get_available_child_count(pnode);416 if (!count)417 return 0;418 419 /* There can be only 1 child node */420 if (WARN_ON(count > 1))421 return -EINVAL;422 423 np = of_get_next_available_child(pnode, NULL);424 if (WARN_ON(!np))425 return -ENODEV;426 427 ret = snprintf(compat, sizeof(compat), "virtio,device%x", dev->id.device);428 BUG_ON(ret >= sizeof(compat));429 430 /*431 * On powerpc/pseries virtio devices are PCI devices so PCI432 * vendor/device ids play the role of the "compatible" property.433 * Simply don't init of_node in this case.434 */435 if (!of_device_is_compatible(np, compat)) {436 ret = 0;437 goto out;438 }439 440 dev->dev.of_node = np;441 return 0;442 443out:444 of_node_put(np);445 return ret;446}447 448/**449 * register_virtio_device - register virtio device450 * @dev : virtio device to be registered451 *452 * On error, the caller must call put_device on &@dev->dev (and not kfree),453 * as another code path may have obtained a reference to @dev.454 *455 * Returns: 0 on suceess, -error on failure456 */457int register_virtio_device(struct virtio_device *dev)458{459 int err;460 461 dev->dev.bus = &virtio_bus;462 device_initialize(&dev->dev);463 464 /* Assign a unique device index and hence name. */465 err = ida_alloc(&virtio_index_ida, GFP_KERNEL);466 if (err < 0)467 goto out;468 469 dev->index = err;470 err = dev_set_name(&dev->dev, "virtio%u", dev->index);471 if (err)472 goto out_ida_remove;473 474 err = virtio_device_of_init(dev);475 if (err)476 goto out_ida_remove;477 478 spin_lock_init(&dev->config_lock);479 dev->config_core_enabled = false;480 dev->config_change_pending = false;481 482 INIT_LIST_HEAD(&dev->vqs);483 spin_lock_init(&dev->vqs_list_lock);484 485 /* We always start by resetting the device, in case a previous486 * driver messed it up. This also tests that code path a little. */487 virtio_reset_device(dev);488 489 /* Acknowledge that we've seen the device. */490 virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);491 492 virtio_debug_device_init(dev);493 494 /*495 * device_add() causes the bus infrastructure to look for a matching496 * driver.497 */498 err = device_add(&dev->dev);499 if (err)500 goto out_of_node_put;501 502 return 0;503 504out_of_node_put:505 of_node_put(dev->dev.of_node);506out_ida_remove:507 ida_free(&virtio_index_ida, dev->index);508out:509 virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);510 return err;511}512EXPORT_SYMBOL_GPL(register_virtio_device);513 514bool is_virtio_device(struct device *dev)515{516 return dev->bus == &virtio_bus;517}518EXPORT_SYMBOL_GPL(is_virtio_device);519 520void unregister_virtio_device(struct virtio_device *dev)521{522 int index = dev->index; /* save for after device release */523 524 device_unregister(&dev->dev);525 virtio_debug_device_exit(dev);526 ida_free(&virtio_index_ida, index);527}528EXPORT_SYMBOL_GPL(unregister_virtio_device);529 530#ifdef CONFIG_PM_SLEEP531int virtio_device_freeze(struct virtio_device *dev)532{533 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);534 int ret;535 536 virtio_config_core_disable(dev);537 538 dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;539 540 if (drv && drv->freeze) {541 ret = drv->freeze(dev);542 if (ret) {543 virtio_config_core_enable(dev);544 return ret;545 }546 }547 548 return 0;549}550EXPORT_SYMBOL_GPL(virtio_device_freeze);551 552int virtio_device_restore(struct virtio_device *dev)553{554 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);555 int ret;556 557 /* We always start by resetting the device, in case a previous558 * driver messed it up. */559 virtio_reset_device(dev);560 561 /* Acknowledge that we've seen the device. */562 virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);563 564 /* Maybe driver failed before freeze.565 * Restore the failed status, for debugging. */566 if (dev->failed)567 virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);568 569 if (!drv)570 return 0;571 572 /* We have a driver! */573 virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);574 575 ret = dev->config->finalize_features(dev);576 if (ret)577 goto err;578 579 ret = virtio_features_ok(dev);580 if (ret)581 goto err;582 583 if (drv->restore) {584 ret = drv->restore(dev);585 if (ret)586 goto err;587 }588 589 /* If restore didn't do it, mark device DRIVER_OK ourselves. */590 if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))591 virtio_device_ready(dev);592 593 virtio_config_core_enable(dev);594 595 return 0;596 597err:598 virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);599 return ret;600}601EXPORT_SYMBOL_GPL(virtio_device_restore);602#endif603 604static int virtio_init(void)605{606 if (bus_register(&virtio_bus) != 0)607 panic("virtio bus registration failed");608 virtio_debug_init();609 return 0;610}611 612static void __exit virtio_exit(void)613{614 virtio_debug_exit();615 bus_unregister(&virtio_bus);616 ida_destroy(&virtio_index_ida);617}618core_initcall(virtio_init);619module_exit(virtio_exit);620 621MODULE_DESCRIPTION("Virtio core interface");622MODULE_LICENSE("GPL");623