363 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/*3 * Copyright 2023 Red Hat4 */5 6#ifndef VDO_H7#define VDO_H8 9#include <linux/atomic.h>10#include <linux/blk_types.h>11#include <linux/completion.h>12#include <linux/dm-kcopyd.h>13#include <linux/list.h>14#include <linux/spinlock.h>15 16#include "admin-state.h"17#include "encodings.h"18#include "funnel-workqueue.h"19#include "packer.h"20#include "physical-zone.h"21#include "statistics.h"22#include "thread-registry.h"23#include "types.h"24 25enum notifier_state {26 /* Notifications are allowed but not in progress */27 MAY_NOTIFY,28 /* A notification is in progress */29 NOTIFYING,30 /* Notifications are not allowed */31 MAY_NOT_NOTIFY,32 /* A notification has completed */33 NOTIFIED,34};35 36/**37 * typedef vdo_read_only_notification_fn - A function to notify a listener that the VDO has gone38 * read-only.39 * @listener: The object to notify.40 * @parent: The completion to notify in order to acknowledge the notification.41 */42typedef void (*vdo_read_only_notification_fn)(void *listener, struct vdo_completion *parent);43 44/*45 * An object to be notified when the VDO enters read-only mode46 */47struct read_only_listener {48 /* The listener */49 void *listener;50 /* The method to call to notify the listener */51 vdo_read_only_notification_fn notify;52 /* A pointer to the next listener */53 struct read_only_listener *next;54};55 56struct vdo_thread {57 struct vdo *vdo;58 thread_id_t thread_id;59 struct vdo_work_queue *queue;60 /*61 * Each thread maintains its own notion of whether the VDO is read-only so that the62 * read-only state can be checked from any base thread without worrying about63 * synchronization or thread safety. This does mean that knowledge of the VDO going64 * read-only does not occur simultaneously across the VDO's threads, but that does not seem65 * to cause any problems.66 */67 bool is_read_only;68 /*69 * A list of objects waiting to be notified on this thread that the VDO has entered70 * read-only mode.71 */72 struct read_only_listener *listeners;73 struct registered_thread allocating_thread;74};75 76/* Keep struct bio statistics atomically */77struct atomic_bio_stats {78 atomic64_t read; /* Number of not REQ_WRITE bios */79 atomic64_t write; /* Number of REQ_WRITE bios */80 atomic64_t discard; /* Number of REQ_DISCARD bios */81 atomic64_t flush; /* Number of REQ_FLUSH bios */82 atomic64_t empty_flush; /* Number of REQ_PREFLUSH bios without data */83 atomic64_t fua; /* Number of REQ_FUA bios */84};85 86/* Counters are atomic since updates can arrive concurrently from arbitrary threads. */87struct atomic_statistics {88 atomic64_t bios_submitted;89 atomic64_t bios_completed;90 atomic64_t flush_out;91 atomic64_t invalid_advice_pbn_count;92 atomic64_t no_space_error_count;93 atomic64_t read_only_error_count;94 struct atomic_bio_stats bios_in;95 struct atomic_bio_stats bios_in_partial;96 struct atomic_bio_stats bios_out;97 struct atomic_bio_stats bios_out_completed;98 struct atomic_bio_stats bios_acknowledged;99 struct atomic_bio_stats bios_acknowledged_partial;100 struct atomic_bio_stats bios_meta;101 struct atomic_bio_stats bios_meta_completed;102 struct atomic_bio_stats bios_journal;103 struct atomic_bio_stats bios_journal_completed;104 struct atomic_bio_stats bios_page_cache;105 struct atomic_bio_stats bios_page_cache_completed;106};107 108struct read_only_notifier {109 /* The completion for entering read-only mode */110 struct vdo_completion completion;111 /* A completion waiting for notifications to be drained or enabled */112 struct vdo_completion *waiter;113 /* Lock to protect the next two fields */114 spinlock_t lock;115 /* The code of the error which put the VDO into read-only mode */116 int read_only_error;117 /* The current state of the notifier (values described above) */118 enum notifier_state state;119};120 121/*122 * The thread ID returned when the current thread is not a vdo thread, or can not be determined123 * (usually due to being at interrupt context).124 */125#define VDO_INVALID_THREAD_ID ((thread_id_t) -1)126 127struct thread_config {128 zone_count_t logical_zone_count;129 zone_count_t physical_zone_count;130 zone_count_t hash_zone_count;131 thread_count_t bio_thread_count;132 thread_count_t thread_count;133 thread_id_t admin_thread;134 thread_id_t journal_thread;135 thread_id_t packer_thread;136 thread_id_t dedupe_thread;137 thread_id_t bio_ack_thread;138 thread_id_t cpu_thread;139 thread_id_t *logical_threads;140 thread_id_t *physical_threads;141 thread_id_t *hash_zone_threads;142 thread_id_t *bio_threads;143};144 145struct thread_count_config;146 147struct vdo_super_block {148 /* The vio for reading and writing the super block to disk */149 struct vio vio;150 /* A buffer to hold the super block */151 u8 *buffer;152 /* Whether this super block may not be written */153 bool unwritable;154};155 156struct data_vio_pool;157 158struct vdo_administrator {159 struct vdo_completion completion;160 struct admin_state state;161 atomic_t busy;162 u32 phase;163 struct completion callback_sync;164};165 166struct vdo {167 char thread_name_prefix[MAX_VDO_WORK_QUEUE_NAME_LEN];168 struct vdo_thread *threads;169 vdo_action_fn action;170 struct vdo_completion *completion;171 struct vio_tracer *vio_tracer;172 173 /* The atomic version of the state of this vdo */174 atomic_t state;175 /* The full state of all components */176 struct vdo_component_states states;177 /*178 * A counter value to attach to thread names and log messages to identify the individual179 * device.180 */181 unsigned int instance;182 /* The read-only notifier */183 struct read_only_notifier read_only_notifier;184 /* The load-time configuration of this vdo */185 struct device_config *device_config;186 /* The thread mapping */187 struct thread_config thread_config;188 189 /* The super block */190 struct vdo_super_block super_block;191 192 /* The partitioning of the underlying storage */193 struct layout layout;194 struct layout next_layout;195 struct dm_kcopyd_client *partition_copier;196 197 /* The block map */198 struct block_map *block_map;199 200 /* The journal for block map recovery */201 struct recovery_journal *recovery_journal;202 203 /* The slab depot */204 struct slab_depot *depot;205 206 /* The compressed-block packer */207 struct packer *packer;208 /* Whether incoming data should be compressed */209 bool compressing;210 211 /* The handler for flush requests */212 struct flusher *flusher;213 214 /* The state the vdo was in when loaded (primarily for unit tests) */215 enum vdo_state load_state;216 217 /* The logical zones of this vdo */218 struct logical_zones *logical_zones;219 220 /* The physical zones of this vdo */221 struct physical_zones *physical_zones;222 223 /* The hash lock zones of this vdo */224 struct hash_zones *hash_zones;225 226 /* Bio submission manager used for sending bios to the storage device. */227 struct io_submitter *io_submitter;228 229 /* The pool of data_vios for servicing incoming bios */230 struct data_vio_pool *data_vio_pool;231 232 /* The manager for administrative operations */233 struct vdo_administrator admin;234 235 /* Flags controlling administrative operations */236 const struct admin_state_code *suspend_type;237 bool allocations_allowed;238 bool dump_on_shutdown;239 atomic_t processing_message;240 241 /*242 * Statistics243 * Atomic stats counters244 */245 struct atomic_statistics stats;246 /* Used to gather statistics without allocating memory */247 struct vdo_statistics stats_buffer;248 /* Protects the stats_buffer */249 struct mutex stats_mutex;250 251 /* A list of all device_configs referencing this vdo */252 struct list_head device_config_list;253 254 /* This VDO's list entry for the device registry */255 struct list_head registration;256 257 /* Underlying block device info. */258 u64 starting_sector_offset;259 struct volume_geometry geometry;260 261 /* N blobs of context data for LZ4 code, one per CPU thread. */262 char **compression_context;263};264 265/**266 * vdo_uses_bio_ack_queue() - Indicate whether the vdo is configured to use a separate work queue267 * for acknowledging received and processed bios.268 * @vdo: The vdo.269 *270 * Note that this directly controls the handling of write operations, but the compile-time flag271 * VDO_USE_BIO_ACK_QUEUE_FOR_READ is also checked for read operations.272 *273 * Return: Whether a bio-acknowledgement work queue is in use.274 */275static inline bool vdo_uses_bio_ack_queue(struct vdo *vdo)276{277 return vdo->device_config->thread_counts.bio_ack_threads > 0;278}279 280/**281 * typedef vdo_filter_fn - Method type for vdo matching methods.282 *283 * A filter function returns false if the vdo doesn't match.284 */285typedef bool (*vdo_filter_fn)(struct vdo *vdo, const void *context);286 287void vdo_initialize_device_registry_once(void);288struct vdo * __must_check vdo_find_matching(vdo_filter_fn filter, const void *context);289 290int __must_check vdo_make_thread(struct vdo *vdo, thread_id_t thread_id,291 const struct vdo_work_queue_type *type,292 unsigned int queue_count, void *contexts[]);293 294static inline int __must_check vdo_make_default_thread(struct vdo *vdo,295 thread_id_t thread_id)296{297 return vdo_make_thread(vdo, thread_id, NULL, 1, NULL);298}299 300int __must_check vdo_make(unsigned int instance, struct device_config *config,301 char **reason, struct vdo **vdo_ptr);302 303void vdo_destroy(struct vdo *vdo);304 305void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent);306 307struct block_device * __must_check vdo_get_backing_device(const struct vdo *vdo);308 309const char * __must_check vdo_get_device_name(const struct dm_target *target);310 311int __must_check vdo_synchronous_flush(struct vdo *vdo);312 313const struct admin_state_code * __must_check vdo_get_admin_state(const struct vdo *vdo);314 315bool vdo_set_compressing(struct vdo *vdo, bool enable);316 317bool vdo_get_compressing(struct vdo *vdo);318 319void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats);320 321thread_id_t vdo_get_callback_thread_id(void);322 323enum vdo_state __must_check vdo_get_state(const struct vdo *vdo);324 325void vdo_set_state(struct vdo *vdo, enum vdo_state state);326 327void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent);328 329int vdo_register_read_only_listener(struct vdo *vdo, void *listener,330 vdo_read_only_notification_fn notification,331 thread_id_t thread_id);332 333int vdo_enable_read_only_entry(struct vdo *vdo);334 335void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent);336 337void vdo_allow_read_only_mode_entry(struct vdo_completion *parent);338 339void vdo_enter_read_only_mode(struct vdo *vdo, int error_code);340 341bool __must_check vdo_is_read_only(struct vdo *vdo);342 343bool __must_check vdo_in_read_only_mode(const struct vdo *vdo);344 345bool __must_check vdo_in_recovery_mode(const struct vdo *vdo);346 347void vdo_enter_recovery_mode(struct vdo *vdo);348 349void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name);350 351void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone,352 const char *name);353 354void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone,355 const char *name);356 357int __must_check vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn,358 struct physical_zone **zone_ptr);359 360void vdo_dump_status(const struct vdo *vdo);361 362#endif /* VDO_H */363