671 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/*3 * Copyright 2023 Red Hat4 */5 6#ifndef DATA_VIO_H7#define DATA_VIO_H8 9#include <linux/atomic.h>10#include <linux/bio.h>11#include <linux/list.h>12 13#include "permassert.h"14 15#include "indexer.h"16 17#include "block-map.h"18#include "completion.h"19#include "constants.h"20#include "dedupe.h"21#include "encodings.h"22#include "logical-zone.h"23#include "physical-zone.h"24#include "types.h"25#include "vdo.h"26#include "vio.h"27#include "wait-queue.h"28 29/* Codes for describing the last asynchronous operation performed on a vio. */30enum async_operation_number {31 MIN_VIO_ASYNC_OPERATION_NUMBER,32 VIO_ASYNC_OP_LAUNCH = MIN_VIO_ASYNC_OPERATION_NUMBER,33 VIO_ASYNC_OP_ACKNOWLEDGE_WRITE,34 VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK,35 VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK,36 VIO_ASYNC_OP_LOCK_DUPLICATE_PBN,37 VIO_ASYNC_OP_CHECK_FOR_DUPLICATION,38 VIO_ASYNC_OP_CLEANUP,39 VIO_ASYNC_OP_COMPRESS_DATA_VIO,40 VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT,41 VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ,42 VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE,43 VIO_ASYNC_OP_HASH_DATA_VIO,44 VIO_ASYNC_OP_JOURNAL_REMAPPING,45 VIO_ASYNC_OP_ATTEMPT_PACKING,46 VIO_ASYNC_OP_PUT_MAPPED_BLOCK,47 VIO_ASYNC_OP_READ_DATA_VIO,48 VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX,49 VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS,50 VIO_ASYNC_OP_VERIFY_DUPLICATION,51 VIO_ASYNC_OP_WRITE_DATA_VIO,52 MAX_VIO_ASYNC_OPERATION_NUMBER,53} __packed;54 55struct lbn_lock {56 logical_block_number_t lbn;57 bool locked;58 struct vdo_wait_queue waiters;59 struct logical_zone *zone;60};61 62/* A position in the arboreal block map at a specific level. */63struct block_map_tree_slot {64 page_number_t page_index;65 struct block_map_slot block_map_slot;66};67 68/* Fields for using the arboreal block map. */69struct tree_lock {70 /* The current height at which this data_vio is operating */71 height_t height;72 /* The block map tree for this LBN */73 root_count_t root_index;74 /* Whether we hold a page lock */75 bool locked;76 /* The key for the lock map */77 u64 key;78 /* The queue of waiters for the page this vio is allocating or loading */79 struct vdo_wait_queue waiters;80 /* The block map tree slots for this LBN */81 struct block_map_tree_slot tree_slots[VDO_BLOCK_MAP_TREE_HEIGHT + 1];82};83 84struct zoned_pbn {85 physical_block_number_t pbn;86 enum block_mapping_state state;87 struct physical_zone *zone;88};89 90/*91 * Where a data_vio is on the compression path; advance_compression_stage() depends on the order of92 * this enum.93 */94enum data_vio_compression_stage {95 /* A data_vio which has not yet entered the compression path */96 DATA_VIO_PRE_COMPRESSOR,97 /* A data_vio which is in the compressor */98 DATA_VIO_COMPRESSING,99 /* A data_vio which is blocked in the packer */100 DATA_VIO_PACKING,101 /* A data_vio which is no longer on the compression path (and never will be) */102 DATA_VIO_POST_PACKER,103};104 105struct data_vio_compression_status {106 enum data_vio_compression_stage stage;107 bool may_not_compress;108};109 110struct compression_state {111 /*112 * The current compression status of this data_vio. This field contains a value which113 * consists of a data_vio_compression_stage and a flag indicating whether a request has114 * been made to cancel (or prevent) compression for this data_vio.115 *116 * This field should be accessed through the get_data_vio_compression_status() and117 * set_data_vio_compression_status() methods. It should not be accessed directly.118 */119 atomic_t status;120 121 /* The compressed size of this block */122 u16 size;123 124 /* The packer input or output bin slot which holds the enclosing data_vio */125 slot_number_t slot;126 127 /* The packer bin to which the enclosing data_vio has been assigned */128 struct packer_bin *bin;129 130 /* A link in the chain of data_vios which have been packed together */131 struct data_vio *next_in_batch;132 133 /* A vio which is blocked in the packer while holding a lock this vio needs. */134 struct data_vio *lock_holder;135 136 /*137 * The compressed block used to hold the compressed form of this block and that of any138 * other blocks for which this data_vio is the compressed write agent.139 */140 struct compressed_block *block;141};142 143/* Fields supporting allocation of data blocks. */144struct allocation {145 /* The physical zone in which to allocate a physical block */146 struct physical_zone *zone;147 148 /* The block allocated to this vio */149 physical_block_number_t pbn;150 151 /*152 * If non-NULL, the pooled PBN lock held on the allocated block. Must be a write lock until153 * the block has been written, after which it will become a read lock.154 */155 struct pbn_lock *lock;156 157 /* The type of write lock to obtain on the allocated block */158 enum pbn_lock_type write_lock_type;159 160 /* The zone which was the start of the current allocation cycle */161 zone_count_t first_allocation_zone;162 163 /* Whether this vio should wait for a clean slab */164 bool wait_for_clean_slab;165};166 167struct reference_updater {168 enum journal_operation operation;169 bool increment;170 struct zoned_pbn zpbn;171 struct pbn_lock *lock;172 struct vdo_waiter waiter;173};174 175/* A vio for processing user data requests. */176struct data_vio {177 /* The vdo_wait_queue entry structure */178 struct vdo_waiter waiter;179 180 /* The logical block of this request */181 struct lbn_lock logical;182 183 /* The state for traversing the block map tree */184 struct tree_lock tree_lock;185 186 /* The current partition address of this block */187 struct zoned_pbn mapped;188 189 /* The hash of this vio (if not zero) */190 struct uds_record_name record_name;191 192 /* Used for logging and debugging */193 enum async_operation_number last_async_operation;194 195 /* The operations to record in the recovery and slab journals */196 struct reference_updater increment_updater;197 struct reference_updater decrement_updater;198 199 u16 read : 1;200 u16 write : 1;201 u16 fua : 1;202 u16 is_zero : 1;203 u16 is_discard : 1;204 u16 is_partial : 1;205 u16 is_duplicate : 1;206 u16 first_reference_operation_complete : 1;207 u16 downgrade_allocation_lock : 1;208 209 struct allocation allocation;210 211 /*212 * Whether this vio has received an allocation. This field is examined from threads not in213 * the allocation zone.214 */215 bool allocation_succeeded;216 217 /* The new partition address of this block after the vio write completes */218 struct zoned_pbn new_mapped;219 220 /* The hash zone responsible for the name (NULL if is_zero_block) */221 struct hash_zone *hash_zone;222 223 /* The lock this vio holds or shares with other vios with the same data */224 struct hash_lock *hash_lock;225 226 /* All data_vios sharing a hash lock are kept in a list linking these list entries */227 struct list_head hash_lock_entry;228 229 /* The block number in the partition of the UDS deduplication advice */230 struct zoned_pbn duplicate;231 232 /*233 * The sequence number of the recovery journal block containing the increment entry for234 * this vio.235 */236 sequence_number_t recovery_sequence_number;237 238 /* The point in the recovery journal where this write last made an entry */239 struct journal_point recovery_journal_point;240 241 /* The list of vios in user initiated write requests */242 struct list_head write_entry;243 244 /* The generation number of the VDO that this vio belongs to */245 sequence_number_t flush_generation;246 247 /* The completion to use for fetching block map pages for this vio */248 struct vdo_page_completion page_completion;249 250 /* The user bio that initiated this VIO */251 struct bio *user_bio;252 253 /* partial block support */254 block_size_t offset;255 256 /*257 * The number of bytes to be discarded. For discards, this field will always be positive,258 * whereas for non-discards it will always be 0. Hence it can be used to determine whether259 * a data_vio is processing a discard, even after the user_bio has been acknowledged.260 */261 u32 remaining_discard;262 263 struct dedupe_context *dedupe_context;264 265 /* Fields beyond this point will not be reset when a pooled data_vio is reused. */266 267 struct vio vio;268 269 /* The completion for making reference count decrements */270 struct vdo_completion decrement_completion;271 272 /* All of the fields necessary for the compression path */273 struct compression_state compression;274 275 /* A block used as output during compression or uncompression */276 char *scratch_block;277 278 struct list_head pool_entry;279};280 281static inline struct data_vio *vio_as_data_vio(struct vio *vio)282{283 VDO_ASSERT_LOG_ONLY((vio->type == VIO_TYPE_DATA), "vio is a data_vio");284 return container_of(vio, struct data_vio, vio);285}286 287static inline struct data_vio *as_data_vio(struct vdo_completion *completion)288{289 return vio_as_data_vio(as_vio(completion));290}291 292static inline struct data_vio *vdo_waiter_as_data_vio(struct vdo_waiter *waiter)293{294 if (waiter == NULL)295 return NULL;296 297 return container_of(waiter, struct data_vio, waiter);298}299 300static inline struct data_vio *data_vio_from_reference_updater(struct reference_updater *updater)301{302 if (updater->increment)303 return container_of(updater, struct data_vio, increment_updater);304 305 return container_of(updater, struct data_vio, decrement_updater);306}307 308static inline bool data_vio_has_flush_generation_lock(struct data_vio *data_vio)309{310 return !list_empty(&data_vio->write_entry);311}312 313static inline struct vdo *vdo_from_data_vio(struct data_vio *data_vio)314{315 return data_vio->vio.completion.vdo;316}317 318static inline bool data_vio_has_allocation(struct data_vio *data_vio)319{320 return (data_vio->allocation.pbn != VDO_ZERO_BLOCK);321}322 323struct data_vio_compression_status __must_check324advance_data_vio_compression_stage(struct data_vio *data_vio);325struct data_vio_compression_status __must_check326get_data_vio_compression_status(struct data_vio *data_vio);327bool cancel_data_vio_compression(struct data_vio *data_vio);328 329struct data_vio_pool;330 331int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size,332 data_vio_count_t discard_limit, struct data_vio_pool **pool_ptr);333void free_data_vio_pool(struct data_vio_pool *pool);334void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio);335void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion);336void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion);337 338void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios);339data_vio_count_t get_data_vio_pool_active_discards(struct data_vio_pool *pool);340data_vio_count_t get_data_vio_pool_discard_limit(struct data_vio_pool *pool);341data_vio_count_t get_data_vio_pool_maximum_discards(struct data_vio_pool *pool);342int __must_check set_data_vio_pool_discard_limit(struct data_vio_pool *pool,343 data_vio_count_t limit);344data_vio_count_t get_data_vio_pool_active_requests(struct data_vio_pool *pool);345data_vio_count_t get_data_vio_pool_request_limit(struct data_vio_pool *pool);346data_vio_count_t get_data_vio_pool_maximum_requests(struct data_vio_pool *pool);347 348void complete_data_vio(struct vdo_completion *completion);349void handle_data_vio_error(struct vdo_completion *completion);350 351static inline void continue_data_vio(struct data_vio *data_vio)352{353 vdo_launch_completion(&data_vio->vio.completion);354}355 356/**357 * continue_data_vio_with_error() - Set an error code and then continue processing a data_vio.358 *359 * This will not mask older errors. This function can be called with a success code, but it is more360 * efficient to call continue_data_vio() if the caller knows the result was a success.361 */362static inline void continue_data_vio_with_error(struct data_vio *data_vio, int result)363{364 vdo_continue_completion(&data_vio->vio.completion, result);365}366 367const char * __must_check get_data_vio_operation_name(struct data_vio *data_vio);368 369static inline void assert_data_vio_in_hash_zone(struct data_vio *data_vio)370{371 thread_id_t expected = data_vio->hash_zone->thread_id;372 thread_id_t thread_id = vdo_get_callback_thread_id();373 /*374 * It's odd to use the LBN, but converting the record name to hex is a bit clunky for an375 * inline, and the LBN better than nothing as an identifier.376 */377 VDO_ASSERT_LOG_ONLY((expected == thread_id),378 "data_vio for logical block %llu on thread %u, should be on hash zone thread %u",379 (unsigned long long) data_vio->logical.lbn, thread_id, expected);380}381 382static inline void set_data_vio_hash_zone_callback(struct data_vio *data_vio,383 vdo_action_fn callback)384{385 vdo_set_completion_callback(&data_vio->vio.completion, callback,386 data_vio->hash_zone->thread_id);387}388 389/**390 * launch_data_vio_hash_zone_callback() - Set a callback as a hash zone operation and invoke it391 * immediately.392 */393static inline void launch_data_vio_hash_zone_callback(struct data_vio *data_vio,394 vdo_action_fn callback)395{396 set_data_vio_hash_zone_callback(data_vio, callback);397 vdo_launch_completion(&data_vio->vio.completion);398}399 400static inline void assert_data_vio_in_logical_zone(struct data_vio *data_vio)401{402 thread_id_t expected = data_vio->logical.zone->thread_id;403 thread_id_t thread_id = vdo_get_callback_thread_id();404 405 VDO_ASSERT_LOG_ONLY((expected == thread_id),406 "data_vio for logical block %llu on thread %u, should be on thread %u",407 (unsigned long long) data_vio->logical.lbn, thread_id, expected);408}409 410static inline void set_data_vio_logical_callback(struct data_vio *data_vio,411 vdo_action_fn callback)412{413 vdo_set_completion_callback(&data_vio->vio.completion, callback,414 data_vio->logical.zone->thread_id);415}416 417/**418 * launch_data_vio_logical_callback() - Set a callback as a logical block operation and invoke it419 * immediately.420 */421static inline void launch_data_vio_logical_callback(struct data_vio *data_vio,422 vdo_action_fn callback)423{424 set_data_vio_logical_callback(data_vio, callback);425 vdo_launch_completion(&data_vio->vio.completion);426}427 428static inline void assert_data_vio_in_allocated_zone(struct data_vio *data_vio)429{430 thread_id_t expected = data_vio->allocation.zone->thread_id;431 thread_id_t thread_id = vdo_get_callback_thread_id();432 433 VDO_ASSERT_LOG_ONLY((expected == thread_id),434 "struct data_vio for allocated physical block %llu on thread %u, should be on thread %u",435 (unsigned long long) data_vio->allocation.pbn, thread_id,436 expected);437}438 439static inline void set_data_vio_allocated_zone_callback(struct data_vio *data_vio,440 vdo_action_fn callback)441{442 vdo_set_completion_callback(&data_vio->vio.completion, callback,443 data_vio->allocation.zone->thread_id);444}445 446/**447 * launch_data_vio_allocated_zone_callback() - Set a callback as a physical block operation in a448 * data_vio's allocated zone and queue the data_vio and449 * invoke it immediately.450 */451static inline void launch_data_vio_allocated_zone_callback(struct data_vio *data_vio,452 vdo_action_fn callback)453{454 set_data_vio_allocated_zone_callback(data_vio, callback);455 vdo_launch_completion(&data_vio->vio.completion);456}457 458static inline void assert_data_vio_in_duplicate_zone(struct data_vio *data_vio)459{460 thread_id_t expected = data_vio->duplicate.zone->thread_id;461 thread_id_t thread_id = vdo_get_callback_thread_id();462 463 VDO_ASSERT_LOG_ONLY((expected == thread_id),464 "data_vio for duplicate physical block %llu on thread %u, should be on thread %u",465 (unsigned long long) data_vio->duplicate.pbn, thread_id,466 expected);467}468 469static inline void set_data_vio_duplicate_zone_callback(struct data_vio *data_vio,470 vdo_action_fn callback)471{472 vdo_set_completion_callback(&data_vio->vio.completion, callback,473 data_vio->duplicate.zone->thread_id);474}475 476/**477 * launch_data_vio_duplicate_zone_callback() - Set a callback as a physical block operation in a478 * data_vio's duplicate zone and queue the data_vio and479 * invoke it immediately.480 */481static inline void launch_data_vio_duplicate_zone_callback(struct data_vio *data_vio,482 vdo_action_fn callback)483{484 set_data_vio_duplicate_zone_callback(data_vio, callback);485 vdo_launch_completion(&data_vio->vio.completion);486}487 488static inline void assert_data_vio_in_mapped_zone(struct data_vio *data_vio)489{490 thread_id_t expected = data_vio->mapped.zone->thread_id;491 thread_id_t thread_id = vdo_get_callback_thread_id();492 493 VDO_ASSERT_LOG_ONLY((expected == thread_id),494 "data_vio for mapped physical block %llu on thread %u, should be on thread %u",495 (unsigned long long) data_vio->mapped.pbn, thread_id, expected);496}497 498static inline void set_data_vio_mapped_zone_callback(struct data_vio *data_vio,499 vdo_action_fn callback)500{501 vdo_set_completion_callback(&data_vio->vio.completion, callback,502 data_vio->mapped.zone->thread_id);503}504 505static inline void assert_data_vio_in_new_mapped_zone(struct data_vio *data_vio)506{507 thread_id_t expected = data_vio->new_mapped.zone->thread_id;508 thread_id_t thread_id = vdo_get_callback_thread_id();509 510 VDO_ASSERT_LOG_ONLY((expected == thread_id),511 "data_vio for new_mapped physical block %llu on thread %u, should be on thread %u",512 (unsigned long long) data_vio->new_mapped.pbn, thread_id,513 expected);514}515 516static inline void set_data_vio_new_mapped_zone_callback(struct data_vio *data_vio,517 vdo_action_fn callback)518{519 vdo_set_completion_callback(&data_vio->vio.completion, callback,520 data_vio->new_mapped.zone->thread_id);521}522 523static inline void assert_data_vio_in_journal_zone(struct data_vio *data_vio)524{525 thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread;526 thread_id_t thread_id = vdo_get_callback_thread_id();527 528 VDO_ASSERT_LOG_ONLY((journal_thread == thread_id),529 "data_vio for logical block %llu on thread %u, should be on journal thread %u",530 (unsigned long long) data_vio->logical.lbn, thread_id,531 journal_thread);532}533 534static inline void set_data_vio_journal_callback(struct data_vio *data_vio,535 vdo_action_fn callback)536{537 thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread;538 539 vdo_set_completion_callback(&data_vio->vio.completion, callback, journal_thread);540}541 542/**543 * launch_data_vio_journal_callback() - Set a callback as a journal operation and invoke it544 * immediately.545 */546static inline void launch_data_vio_journal_callback(struct data_vio *data_vio,547 vdo_action_fn callback)548{549 set_data_vio_journal_callback(data_vio, callback);550 vdo_launch_completion(&data_vio->vio.completion);551}552 553static inline void assert_data_vio_in_packer_zone(struct data_vio *data_vio)554{555 thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread;556 thread_id_t thread_id = vdo_get_callback_thread_id();557 558 VDO_ASSERT_LOG_ONLY((packer_thread == thread_id),559 "data_vio for logical block %llu on thread %u, should be on packer thread %u",560 (unsigned long long) data_vio->logical.lbn, thread_id,561 packer_thread);562}563 564static inline void set_data_vio_packer_callback(struct data_vio *data_vio,565 vdo_action_fn callback)566{567 thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread;568 569 vdo_set_completion_callback(&data_vio->vio.completion, callback, packer_thread);570}571 572/**573 * launch_data_vio_packer_callback() - Set a callback as a packer operation and invoke it574 * immediately.575 */576static inline void launch_data_vio_packer_callback(struct data_vio *data_vio,577 vdo_action_fn callback)578{579 set_data_vio_packer_callback(data_vio, callback);580 vdo_launch_completion(&data_vio->vio.completion);581}582 583static inline void assert_data_vio_on_cpu_thread(struct data_vio *data_vio)584{585 thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread;586 thread_id_t thread_id = vdo_get_callback_thread_id();587 588 VDO_ASSERT_LOG_ONLY((cpu_thread == thread_id),589 "data_vio for logical block %llu on thread %u, should be on cpu thread %u",590 (unsigned long long) data_vio->logical.lbn, thread_id,591 cpu_thread);592}593 594static inline void set_data_vio_cpu_callback(struct data_vio *data_vio,595 vdo_action_fn callback)596{597 thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread;598 599 vdo_set_completion_callback(&data_vio->vio.completion, callback, cpu_thread);600}601 602/**603 * launch_data_vio_cpu_callback() - Set a callback to run on the CPU queues and invoke it604 * immediately.605 */606static inline void launch_data_vio_cpu_callback(struct data_vio *data_vio,607 vdo_action_fn callback,608 enum vdo_completion_priority priority)609{610 set_data_vio_cpu_callback(data_vio, callback);611 vdo_launch_completion_with_priority(&data_vio->vio.completion, priority);612}613 614static inline void set_data_vio_bio_zone_callback(struct data_vio *data_vio,615 vdo_action_fn callback)616{617 vdo_set_completion_callback(&data_vio->vio.completion, callback,618 get_vio_bio_zone_thread_id(&data_vio->vio));619}620 621/**622 * launch_data_vio_bio_zone_callback() - Set a callback as a bio zone operation and invoke it623 * immediately.624 */625static inline void launch_data_vio_bio_zone_callback(struct data_vio *data_vio,626 vdo_action_fn callback)627{628 set_data_vio_bio_zone_callback(data_vio, callback);629 vdo_launch_completion_with_priority(&data_vio->vio.completion,630 BIO_Q_DATA_PRIORITY);631}632 633/**634 * launch_data_vio_on_bio_ack_queue() - If the vdo uses a bio_ack queue, set a callback to run on635 * it and invoke it immediately, otherwise, just run the636 * callback on the current thread.637 */638static inline void launch_data_vio_on_bio_ack_queue(struct data_vio *data_vio,639 vdo_action_fn callback)640{641 struct vdo_completion *completion = &data_vio->vio.completion;642 struct vdo *vdo = completion->vdo;643 644 if (!vdo_uses_bio_ack_queue(vdo)) {645 callback(completion);646 return;647 }648 649 vdo_set_completion_callback(completion, callback,650 vdo->thread_config.bio_ack_thread);651 vdo_launch_completion_with_priority(completion, BIO_ACK_Q_ACK_PRIORITY);652}653 654void data_vio_allocate_data_block(struct data_vio *data_vio,655 enum pbn_lock_type write_lock_type,656 vdo_action_fn callback, vdo_action_fn error_handler);657 658void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset);659 660int __must_check uncompress_data_vio(struct data_vio *data_vio,661 enum block_mapping_state mapping_state,662 char *buffer);663 664void update_metadata_for_data_vio_write(struct data_vio *data_vio,665 struct pbn_lock *lock);666void write_data_vio(struct data_vio *data_vio);667void launch_compress_data_vio(struct data_vio *data_vio);668void continue_data_vio_with_block_map_slot(struct vdo_completion *completion);669 670#endif /* DATA_VIO_H */671