374 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2023 Red Hat4 */5 6#include "logical-zone.h"7 8#include "logger.h"9#include "memory-alloc.h"10#include "permassert.h"11#include "string-utils.h"12 13#include "action-manager.h"14#include "admin-state.h"15#include "block-map.h"16#include "completion.h"17#include "constants.h"18#include "data-vio.h"19#include "flush.h"20#include "int-map.h"21#include "physical-zone.h"22#include "vdo.h"23 24#define ALLOCATIONS_PER_ZONE 12825 26/**27 * as_logical_zone() - Convert a generic vdo_completion to a logical_zone.28 * @completion: The completion to convert.29 *30 * Return: The completion as a logical_zone.31 */32static struct logical_zone *as_logical_zone(struct vdo_completion *completion)33{34 vdo_assert_completion_type(completion, VDO_GENERATION_FLUSHED_COMPLETION);35 return container_of(completion, struct logical_zone, completion);36}37 38/* get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */39static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number)40{41 struct logical_zones *zones = context;42 43 return zones->zones[zone_number].thread_id;44}45 46/**47 * initialize_zone() - Initialize a logical zone.48 * @zones: The logical_zones to which this zone belongs.49 * @zone_number: The logical_zone's index.50 */51static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number)52{53 int result;54 struct vdo *vdo = zones->vdo;55 struct logical_zone *zone = &zones->zones[zone_number];56 zone_count_t allocation_zone_number;57 58 result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->lbn_operations);59 if (result != VDO_SUCCESS)60 return result;61 62 if (zone_number < vdo->thread_config.logical_zone_count - 1)63 zone->next = &zones->zones[zone_number + 1];64 65 vdo_initialize_completion(&zone->completion, vdo,66 VDO_GENERATION_FLUSHED_COMPLETION);67 zone->zones = zones;68 zone->zone_number = zone_number;69 zone->thread_id = vdo->thread_config.logical_threads[zone_number];70 zone->block_map_zone = &vdo->block_map->zones[zone_number];71 INIT_LIST_HEAD(&zone->write_vios);72 vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION);73 74 allocation_zone_number = zone->thread_id % vdo->thread_config.physical_zone_count;75 zone->allocation_zone = &vdo->physical_zones->zones[allocation_zone_number];76 77 return vdo_make_default_thread(vdo, zone->thread_id);78}79 80/**81 * vdo_make_logical_zones() - Create a set of logical zones.82 * @vdo: The vdo to which the zones will belong.83 * @zones_ptr: A pointer to hold the new zones.84 *85 * Return: VDO_SUCCESS or an error code.86 */87int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr)88{89 struct logical_zones *zones;90 int result;91 zone_count_t zone;92 zone_count_t zone_count = vdo->thread_config.logical_zone_count;93 94 if (zone_count == 0)95 return VDO_SUCCESS;96 97 result = vdo_allocate_extended(struct logical_zones, zone_count,98 struct logical_zone, __func__, &zones);99 if (result != VDO_SUCCESS)100 return result;101 102 zones->vdo = vdo;103 zones->zone_count = zone_count;104 for (zone = 0; zone < zone_count; zone++) {105 result = initialize_zone(zones, zone);106 if (result != VDO_SUCCESS) {107 vdo_free_logical_zones(zones);108 return result;109 }110 }111 112 result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone,113 vdo->thread_config.admin_thread, zones, NULL,114 vdo, &zones->manager);115 if (result != VDO_SUCCESS) {116 vdo_free_logical_zones(zones);117 return result;118 }119 120 *zones_ptr = zones;121 return VDO_SUCCESS;122}123 124/**125 * vdo_free_logical_zones() - Free a set of logical zones.126 * @zones: The set of zones to free.127 */128void vdo_free_logical_zones(struct logical_zones *zones)129{130 zone_count_t index;131 132 if (zones == NULL)133 return;134 135 vdo_free(vdo_forget(zones->manager));136 137 for (index = 0; index < zones->zone_count; index++)138 vdo_int_map_free(vdo_forget(zones->zones[index].lbn_operations));139 140 vdo_free(zones);141}142 143static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what)144{145 VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id),146 "%s() called on correct thread", what);147}148 149/**150 * check_for_drain_complete() - Check whether this zone has drained.151 * @zone: The zone to check.152 */153static void check_for_drain_complete(struct logical_zone *zone)154{155 if (!vdo_is_state_draining(&zone->state) || zone->notifying ||156 !list_empty(&zone->write_vios))157 return;158 159 vdo_finish_draining(&zone->state);160}161 162/**163 * initiate_drain() - Initiate a drain.164 *165 * Implements vdo_admin_initiator_fn.166 */167static void initiate_drain(struct admin_state *state)168{169 check_for_drain_complete(container_of(state, struct logical_zone, state));170}171 172/**173 * drain_logical_zone() - Drain a logical zone.174 *175 * Implements vdo_zone_action_fn.176 */177static void drain_logical_zone(void *context, zone_count_t zone_number,178 struct vdo_completion *parent)179{180 struct logical_zones *zones = context;181 182 vdo_start_draining(&zones->zones[zone_number].state,183 vdo_get_current_manager_operation(zones->manager), parent,184 initiate_drain);185}186 187void vdo_drain_logical_zones(struct logical_zones *zones,188 const struct admin_state_code *operation,189 struct vdo_completion *parent)190{191 vdo_schedule_operation(zones->manager, operation, NULL, drain_logical_zone, NULL,192 parent);193}194 195/**196 * resume_logical_zone() - Resume a logical zone.197 *198 * Implements vdo_zone_action_fn.199 */200static void resume_logical_zone(void *context, zone_count_t zone_number,201 struct vdo_completion *parent)202{203 struct logical_zone *zone = &(((struct logical_zones *) context)->zones[zone_number]);204 205 vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state));206}207 208/**209 * vdo_resume_logical_zones() - Resume a set of logical zones.210 * @zones: The logical zones to resume.211 * @parent: The object to notify when the zones have resumed.212 */213void vdo_resume_logical_zones(struct logical_zones *zones, struct vdo_completion *parent)214{215 vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, NULL,216 resume_logical_zone, NULL, parent);217}218 219/**220 * update_oldest_active_generation() - Update the oldest active generation.221 * @zone: The zone.222 *223 * Return: true if the oldest active generation has changed.224 */225static bool update_oldest_active_generation(struct logical_zone *zone)226{227 struct data_vio *data_vio =228 list_first_entry_or_null(&zone->write_vios, struct data_vio,229 write_entry);230 sequence_number_t oldest =231 (data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation;232 233 if (oldest == zone->oldest_active_generation)234 return false;235 236 WRITE_ONCE(zone->oldest_active_generation, oldest);237 return true;238}239 240/**241 * vdo_increment_logical_zone_flush_generation() - Increment the flush generation in a logical242 * zone.243 * @zone: The logical zone.244 * @expected_generation: The expected value of the flush generation before the increment.245 */246void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone,247 sequence_number_t expected_generation)248{249 assert_on_zone_thread(zone, __func__);250 VDO_ASSERT_LOG_ONLY((zone->flush_generation == expected_generation),251 "logical zone %u flush generation %llu should be %llu before increment",252 zone->zone_number, (unsigned long long) zone->flush_generation,253 (unsigned long long) expected_generation);254 255 zone->flush_generation++;256 zone->ios_in_flush_generation = 0;257 update_oldest_active_generation(zone);258}259 260/**261 * vdo_acquire_flush_generation_lock() - Acquire the shared lock on a flush generation by a write262 * data_vio.263 * @data_vio: The data_vio.264 */265void vdo_acquire_flush_generation_lock(struct data_vio *data_vio)266{267 struct logical_zone *zone = data_vio->logical.zone;268 269 assert_on_zone_thread(zone, __func__);270 VDO_ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal");271 272 data_vio->flush_generation = zone->flush_generation;273 list_add_tail(&data_vio->write_entry, &zone->write_vios);274 zone->ios_in_flush_generation++;275}276 277static void attempt_generation_complete_notification(struct vdo_completion *completion);278 279/**280 * notify_flusher() - Notify the flush that at least one generation no longer has active VIOs.281 * @completion: The zone completion.282 *283 * This callback is registered in attempt_generation_complete_notification().284 */285static void notify_flusher(struct vdo_completion *completion)286{287 struct logical_zone *zone = as_logical_zone(completion);288 289 vdo_complete_flushes(zone->zones->vdo->flusher);290 vdo_launch_completion_callback(completion,291 attempt_generation_complete_notification,292 zone->thread_id);293}294 295/**296 * attempt_generation_complete_notification() - Notify the flusher if some generation no297 * longer has active VIOs.298 * @completion: The zone completion.299 */300static void attempt_generation_complete_notification(struct vdo_completion *completion)301{302 struct logical_zone *zone = as_logical_zone(completion);303 304 assert_on_zone_thread(zone, __func__);305 if (zone->oldest_active_generation <= zone->notification_generation) {306 zone->notifying = false;307 check_for_drain_complete(zone);308 return;309 }310 311 zone->notifying = true;312 zone->notification_generation = zone->oldest_active_generation;313 vdo_launch_completion_callback(&zone->completion, notify_flusher,314 vdo_get_flusher_thread_id(zone->zones->vdo->flusher));315}316 317/**318 * vdo_release_flush_generation_lock() - Release the shared lock on a flush generation held by a319 * write data_vio.320 * @data_vio: The data_vio whose lock is to be released.321 *322 * If there are pending flushes, and this data_vio completes the oldest generation active in this323 * zone, an attempt will be made to finish any flushes which may now be complete.324 */325void vdo_release_flush_generation_lock(struct data_vio *data_vio)326{327 struct logical_zone *zone = data_vio->logical.zone;328 329 assert_on_zone_thread(zone, __func__);330 331 if (!data_vio_has_flush_generation_lock(data_vio))332 return;333 334 list_del_init(&data_vio->write_entry);335 VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation),336 "data_vio releasing lock on generation %llu is not older than oldest active generation %llu",337 (unsigned long long) data_vio->flush_generation,338 (unsigned long long) zone->oldest_active_generation);339 340 if (!update_oldest_active_generation(zone) || zone->notifying)341 return;342 343 attempt_generation_complete_notification(&zone->completion);344}345 346struct physical_zone *vdo_get_next_allocation_zone(struct logical_zone *zone)347{348 if (zone->allocation_count == ALLOCATIONS_PER_ZONE) {349 zone->allocation_count = 0;350 zone->allocation_zone = zone->allocation_zone->next;351 }352 353 zone->allocation_count++;354 return zone->allocation_zone;355}356 357/**358 * vdo_dump_logical_zone() - Dump information about a logical zone to the log for debugging.359 * @zone: The zone to dump360 *361 * Context: the information is dumped in a thread-unsafe fashion.362 *363 */364void vdo_dump_logical_zone(const struct logical_zone *zone)365{366 vdo_log_info("logical_zone %u", zone->zone_number);367 vdo_log_info(" flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu",368 (unsigned long long) READ_ONCE(zone->flush_generation),369 (unsigned long long) READ_ONCE(zone->oldest_active_generation),370 (unsigned long long) READ_ONCE(zone->notification_generation),371 vdo_bool_to_string(READ_ONCE(zone->notifying)),372 (unsigned long long) READ_ONCE(zone->ios_in_flush_generation));373}374