416 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2023 Red Hat4 */5 6#include "io-factory.h"7 8#include <linux/atomic.h>9#include <linux/blkdev.h>10#include <linux/err.h>11#include <linux/mount.h>12 13#include "logger.h"14#include "memory-alloc.h"15#include "numeric.h"16 17/*18 * The I/O factory object manages access to index storage, which is a contiguous range of blocks on19 * a block device.20 *21 * The factory holds the open device and is responsible for closing it. The factory has methods to22 * make helper structures that can be used to access sections of the index.23 */24struct io_factory {25 struct block_device *bdev;26 atomic_t ref_count;27};28 29/* The buffered reader allows efficient I/O by reading page-sized segments into a buffer. */30struct buffered_reader {31 struct io_factory *factory;32 struct dm_bufio_client *client;33 struct dm_buffer *buffer;34 sector_t limit;35 sector_t block_number;36 u8 *start;37 u8 *end;38};39 40#define MAX_READ_AHEAD_BLOCKS 441 42/*43 * The buffered writer allows efficient I/O by buffering writes and committing page-sized segments44 * to storage.45 */46struct buffered_writer {47 struct io_factory *factory;48 struct dm_bufio_client *client;49 struct dm_buffer *buffer;50 sector_t limit;51 sector_t block_number;52 u8 *start;53 u8 *end;54 int error;55};56 57static void uds_get_io_factory(struct io_factory *factory)58{59 atomic_inc(&factory->ref_count);60}61 62int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_ptr)63{64 int result;65 struct io_factory *factory;66 67 result = vdo_allocate(1, struct io_factory, __func__, &factory);68 if (result != VDO_SUCCESS)69 return result;70 71 factory->bdev = bdev;72 atomic_set_release(&factory->ref_count, 1);73 74 *factory_ptr = factory;75 return UDS_SUCCESS;76}77 78int uds_replace_storage(struct io_factory *factory, struct block_device *bdev)79{80 factory->bdev = bdev;81 return UDS_SUCCESS;82}83 84/* Free an I/O factory once all references have been released. */85void uds_put_io_factory(struct io_factory *factory)86{87 if (atomic_add_return(-1, &factory->ref_count) <= 0)88 vdo_free(factory);89}90 91size_t uds_get_writable_size(struct io_factory *factory)92{93 return bdev_nr_bytes(factory->bdev);94}95 96/* Create a struct dm_bufio_client for an index region starting at offset. */97int uds_make_bufio(struct io_factory *factory, off_t block_offset, size_t block_size,98 unsigned int reserved_buffers, struct dm_bufio_client **client_ptr)99{100 struct dm_bufio_client *client;101 102 client = dm_bufio_client_create(factory->bdev, block_size, reserved_buffers, 0,103 NULL, NULL, 0);104 if (IS_ERR(client))105 return -PTR_ERR(client);106 107 dm_bufio_set_sector_offset(client, block_offset * SECTORS_PER_BLOCK);108 *client_ptr = client;109 return UDS_SUCCESS;110}111 112static void read_ahead(struct buffered_reader *reader, sector_t block_number)113{114 if (block_number < reader->limit) {115 sector_t read_ahead = min((sector_t) MAX_READ_AHEAD_BLOCKS,116 reader->limit - block_number);117 118 dm_bufio_prefetch(reader->client, block_number, read_ahead);119 }120}121 122void uds_free_buffered_reader(struct buffered_reader *reader)123{124 if (reader == NULL)125 return;126 127 if (reader->buffer != NULL)128 dm_bufio_release(reader->buffer);129 130 dm_bufio_client_destroy(reader->client);131 uds_put_io_factory(reader->factory);132 vdo_free(reader);133}134 135/* Create a buffered reader for an index region starting at offset. */136int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block_count,137 struct buffered_reader **reader_ptr)138{139 int result;140 struct dm_bufio_client *client = NULL;141 struct buffered_reader *reader = NULL;142 143 result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client);144 if (result != UDS_SUCCESS)145 return result;146 147 result = vdo_allocate(1, struct buffered_reader, "buffered reader", &reader);148 if (result != VDO_SUCCESS) {149 dm_bufio_client_destroy(client);150 return result;151 }152 153 *reader = (struct buffered_reader) {154 .factory = factory,155 .client = client,156 .buffer = NULL,157 .limit = block_count,158 .block_number = 0,159 .start = NULL,160 .end = NULL,161 };162 163 read_ahead(reader, 0);164 uds_get_io_factory(factory);165 *reader_ptr = reader;166 return UDS_SUCCESS;167}168 169static int position_reader(struct buffered_reader *reader, sector_t block_number,170 off_t offset)171{172 struct dm_buffer *buffer = NULL;173 void *data;174 175 if ((reader->end == NULL) || (block_number != reader->block_number)) {176 if (block_number >= reader->limit)177 return UDS_OUT_OF_RANGE;178 179 if (reader->buffer != NULL)180 dm_bufio_release(vdo_forget(reader->buffer));181 182 data = dm_bufio_read(reader->client, block_number, &buffer);183 if (IS_ERR(data))184 return -PTR_ERR(data);185 186 reader->buffer = buffer;187 reader->start = data;188 if (block_number == reader->block_number + 1)189 read_ahead(reader, block_number + 1);190 }191 192 reader->block_number = block_number;193 reader->end = reader->start + offset;194 return UDS_SUCCESS;195}196 197static size_t bytes_remaining_in_read_buffer(struct buffered_reader *reader)198{199 return (reader->end == NULL) ? 0 : reader->start + UDS_BLOCK_SIZE - reader->end;200}201 202static int reset_reader(struct buffered_reader *reader)203{204 sector_t block_number;205 206 if (bytes_remaining_in_read_buffer(reader) > 0)207 return UDS_SUCCESS;208 209 block_number = reader->block_number;210 if (reader->end != NULL)211 block_number++;212 213 return position_reader(reader, block_number, 0);214}215 216int uds_read_from_buffered_reader(struct buffered_reader *reader, u8 *data,217 size_t length)218{219 int result = UDS_SUCCESS;220 size_t chunk_size;221 222 while (length > 0) {223 result = reset_reader(reader);224 if (result != UDS_SUCCESS)225 return result;226 227 chunk_size = min(length, bytes_remaining_in_read_buffer(reader));228 memcpy(data, reader->end, chunk_size);229 length -= chunk_size;230 data += chunk_size;231 reader->end += chunk_size;232 }233 234 return UDS_SUCCESS;235}236 237/*238 * Verify that the next data on the reader matches the required value. If the value matches, the239 * matching contents are consumed. If the value does not match, the reader state is unchanged.240 */241int uds_verify_buffered_data(struct buffered_reader *reader, const u8 *value,242 size_t length)243{244 int result = UDS_SUCCESS;245 size_t chunk_size;246 sector_t start_block_number = reader->block_number;247 int start_offset = reader->end - reader->start;248 249 while (length > 0) {250 result = reset_reader(reader);251 if (result != UDS_SUCCESS) {252 result = UDS_CORRUPT_DATA;253 break;254 }255 256 chunk_size = min(length, bytes_remaining_in_read_buffer(reader));257 if (memcmp(value, reader->end, chunk_size) != 0) {258 result = UDS_CORRUPT_DATA;259 break;260 }261 262 length -= chunk_size;263 value += chunk_size;264 reader->end += chunk_size;265 }266 267 if (result != UDS_SUCCESS)268 position_reader(reader, start_block_number, start_offset);269 270 return result;271}272 273/* Create a buffered writer for an index region starting at offset. */274int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block_count,275 struct buffered_writer **writer_ptr)276{277 int result;278 struct dm_bufio_client *client = NULL;279 struct buffered_writer *writer;280 281 result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client);282 if (result != UDS_SUCCESS)283 return result;284 285 result = vdo_allocate(1, struct buffered_writer, "buffered writer", &writer);286 if (result != VDO_SUCCESS) {287 dm_bufio_client_destroy(client);288 return result;289 }290 291 *writer = (struct buffered_writer) {292 .factory = factory,293 .client = client,294 .buffer = NULL,295 .limit = block_count,296 .start = NULL,297 .end = NULL,298 .block_number = 0,299 .error = UDS_SUCCESS,300 };301 302 uds_get_io_factory(factory);303 *writer_ptr = writer;304 return UDS_SUCCESS;305}306 307static size_t get_remaining_write_space(struct buffered_writer *writer)308{309 return writer->start + UDS_BLOCK_SIZE - writer->end;310}311 312static int __must_check prepare_next_buffer(struct buffered_writer *writer)313{314 struct dm_buffer *buffer = NULL;315 void *data;316 317 if (writer->block_number >= writer->limit) {318 writer->error = UDS_OUT_OF_RANGE;319 return UDS_OUT_OF_RANGE;320 }321 322 data = dm_bufio_new(writer->client, writer->block_number, &buffer);323 if (IS_ERR(data)) {324 writer->error = -PTR_ERR(data);325 return writer->error;326 }327 328 writer->buffer = buffer;329 writer->start = data;330 writer->end = data;331 return UDS_SUCCESS;332}333 334static int flush_previous_buffer(struct buffered_writer *writer)335{336 size_t available;337 338 if (writer->buffer == NULL)339 return writer->error;340 341 if (writer->error == UDS_SUCCESS) {342 available = get_remaining_write_space(writer);343 344 if (available > 0)345 memset(writer->end, 0, available);346 347 dm_bufio_mark_buffer_dirty(writer->buffer);348 }349 350 dm_bufio_release(writer->buffer);351 writer->buffer = NULL;352 writer->start = NULL;353 writer->end = NULL;354 writer->block_number++;355 return writer->error;356}357 358void uds_free_buffered_writer(struct buffered_writer *writer)359{360 int result;361 362 if (writer == NULL)363 return;364 365 flush_previous_buffer(writer);366 result = -dm_bufio_write_dirty_buffers(writer->client);367 if (result != UDS_SUCCESS)368 vdo_log_warning_strerror(result, "%s: failed to sync storage", __func__);369 370 dm_bufio_client_destroy(writer->client);371 uds_put_io_factory(writer->factory);372 vdo_free(writer);373}374 375/*376 * Append data to the buffer, writing as needed. If no data is provided, zeros are written instead.377 * If a write error occurs, it is recorded and returned on every subsequent write attempt.378 */379int uds_write_to_buffered_writer(struct buffered_writer *writer, const u8 *data,380 size_t length)381{382 int result = writer->error;383 size_t chunk_size;384 385 while ((length > 0) && (result == UDS_SUCCESS)) {386 if (writer->buffer == NULL) {387 result = prepare_next_buffer(writer);388 continue;389 }390 391 chunk_size = min(length, get_remaining_write_space(writer));392 if (data == NULL) {393 memset(writer->end, 0, chunk_size);394 } else {395 memcpy(writer->end, data, chunk_size);396 data += chunk_size;397 }398 399 length -= chunk_size;400 writer->end += chunk_size;401 402 if (get_remaining_write_space(writer) == 0)403 result = uds_flush_buffered_writer(writer);404 }405 406 return result;407}408 409int uds_flush_buffered_writer(struct buffered_writer *writer)410{411 if (writer->error != UDS_SUCCESS)412 return writer->error;413 414 return flush_previous_buffer(writer);415}416