740 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2023 Red Hat4 */5 6#include "index-session.h"7 8#include <linux/atomic.h>9 10#include "logger.h"11#include "memory-alloc.h"12#include "time-utils.h"13 14#include "funnel-requestqueue.h"15#include "index.h"16#include "index-layout.h"17 18/*19 * The index session contains a lock (the request_mutex) which ensures that only one thread can20 * change the state of its index at a time. The state field indicates the current state of the21 * index through a set of descriptive flags. The request_mutex must be notified whenever a22 * non-transient state flag is cleared. The request_mutex is also used to count the number of23 * requests currently in progress so that they can be drained when suspending or closing the index.24 *25 * If the index session is suspended shortly after opening an index, it may have to suspend during26 * a rebuild. Depending on the size of the index, a rebuild may take a significant amount of time,27 * so UDS allows the rebuild to be paused in order to suspend the session in a timely manner. When28 * the index session is resumed, the rebuild can continue from where it left off. If the index29 * session is shut down with a suspended rebuild, the rebuild progress is abandoned and the rebuild30 * will start from the beginning the next time the index is loaded. The mutex and status fields in31 * the index_load_context are used to record the state of any interrupted rebuild.32 */33 34enum index_session_flag_bit {35 IS_FLAG_BIT_START = 8,36 /* The session has started loading an index but not completed it. */37 IS_FLAG_BIT_LOADING = IS_FLAG_BIT_START,38 /* The session has loaded an index, which can handle requests. */39 IS_FLAG_BIT_LOADED,40 /* The session's index has been permanently disabled. */41 IS_FLAG_BIT_DISABLED,42 /* The session's index is suspended. */43 IS_FLAG_BIT_SUSPENDED,44 /* The session is handling some index state change. */45 IS_FLAG_BIT_WAITING,46 /* The session's index is closing and draining requests. */47 IS_FLAG_BIT_CLOSING,48 /* The session is being destroyed and is draining requests. */49 IS_FLAG_BIT_DESTROYING,50};51 52enum index_session_flag {53 IS_FLAG_LOADED = (1 << IS_FLAG_BIT_LOADED),54 IS_FLAG_LOADING = (1 << IS_FLAG_BIT_LOADING),55 IS_FLAG_DISABLED = (1 << IS_FLAG_BIT_DISABLED),56 IS_FLAG_SUSPENDED = (1 << IS_FLAG_BIT_SUSPENDED),57 IS_FLAG_WAITING = (1 << IS_FLAG_BIT_WAITING),58 IS_FLAG_CLOSING = (1 << IS_FLAG_BIT_CLOSING),59 IS_FLAG_DESTROYING = (1 << IS_FLAG_BIT_DESTROYING),60};61 62/* Release a reference to an index session. */63static void release_index_session(struct uds_index_session *index_session)64{65 mutex_lock(&index_session->request_mutex);66 if (--index_session->request_count == 0)67 uds_broadcast_cond(&index_session->request_cond);68 mutex_unlock(&index_session->request_mutex);69}70 71/*72 * Acquire a reference to the index session for an asynchronous index request. The reference must73 * eventually be released with a corresponding call to release_index_session().74 */75static int get_index_session(struct uds_index_session *index_session)76{77 unsigned int state;78 int result = UDS_SUCCESS;79 80 mutex_lock(&index_session->request_mutex);81 index_session->request_count++;82 state = index_session->state;83 mutex_unlock(&index_session->request_mutex);84 85 if (state == IS_FLAG_LOADED) {86 return UDS_SUCCESS;87 } else if (state & IS_FLAG_DISABLED) {88 result = UDS_DISABLED;89 } else if ((state & IS_FLAG_LOADING) ||90 (state & IS_FLAG_SUSPENDED) ||91 (state & IS_FLAG_WAITING)) {92 result = -EBUSY;93 } else {94 result = UDS_NO_INDEX;95 }96 97 release_index_session(index_session);98 return result;99}100 101int uds_launch_request(struct uds_request *request)102{103 size_t internal_size;104 int result;105 106 if (request->callback == NULL) {107 vdo_log_error("missing required callback");108 return -EINVAL;109 }110 111 switch (request->type) {112 case UDS_DELETE:113 case UDS_POST:114 case UDS_QUERY:115 case UDS_QUERY_NO_UPDATE:116 case UDS_UPDATE:117 break;118 default:119 vdo_log_error("received invalid callback type");120 return -EINVAL;121 }122 123 /* Reset all internal fields before processing. */124 internal_size =125 sizeof(struct uds_request) - offsetof(struct uds_request, zone_number);126 // FIXME should be using struct_group for this instead127 memset((char *) request + sizeof(*request) - internal_size, 0, internal_size);128 129 result = get_index_session(request->session);130 if (result != UDS_SUCCESS)131 return result;132 133 request->found = false;134 request->unbatched = false;135 request->index = request->session->index;136 137 uds_enqueue_request(request, STAGE_TRIAGE);138 return UDS_SUCCESS;139}140 141static void enter_callback_stage(struct uds_request *request)142{143 if (request->status != UDS_SUCCESS) {144 /* All request errors are considered unrecoverable */145 mutex_lock(&request->session->request_mutex);146 request->session->state |= IS_FLAG_DISABLED;147 mutex_unlock(&request->session->request_mutex);148 }149 150 uds_request_queue_enqueue(request->session->callback_queue, request);151}152 153static inline void count_once(u64 *count_ptr)154{155 WRITE_ONCE(*count_ptr, READ_ONCE(*count_ptr) + 1);156}157 158static void update_session_stats(struct uds_request *request)159{160 struct session_stats *session_stats = &request->session->stats;161 162 count_once(&session_stats->requests);163 164 switch (request->type) {165 case UDS_POST:166 if (request->found)167 count_once(&session_stats->posts_found);168 else169 count_once(&session_stats->posts_not_found);170 171 if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER)172 count_once(&session_stats->posts_found_open_chapter);173 else if (request->location == UDS_LOCATION_IN_DENSE)174 count_once(&session_stats->posts_found_dense);175 else if (request->location == UDS_LOCATION_IN_SPARSE)176 count_once(&session_stats->posts_found_sparse);177 break;178 179 case UDS_UPDATE:180 if (request->found)181 count_once(&session_stats->updates_found);182 else183 count_once(&session_stats->updates_not_found);184 break;185 186 case UDS_DELETE:187 if (request->found)188 count_once(&session_stats->deletions_found);189 else190 count_once(&session_stats->deletions_not_found);191 break;192 193 case UDS_QUERY:194 case UDS_QUERY_NO_UPDATE:195 if (request->found)196 count_once(&session_stats->queries_found);197 else198 count_once(&session_stats->queries_not_found);199 break;200 201 default:202 request->status = VDO_ASSERT(false, "unknown request type: %d",203 request->type);204 }205}206 207static void handle_callbacks(struct uds_request *request)208{209 struct uds_index_session *index_session = request->session;210 211 if (request->status == UDS_SUCCESS)212 update_session_stats(request);213 214 request->status = uds_status_to_errno(request->status);215 request->callback(request);216 release_index_session(index_session);217}218 219static int __must_check make_empty_index_session(struct uds_index_session **index_session_ptr)220{221 int result;222 struct uds_index_session *session;223 224 result = vdo_allocate(1, struct uds_index_session, __func__, &session);225 if (result != VDO_SUCCESS)226 return result;227 228 mutex_init(&session->request_mutex);229 uds_init_cond(&session->request_cond);230 mutex_init(&session->load_context.mutex);231 uds_init_cond(&session->load_context.cond);232 233 result = uds_make_request_queue("callbackW", &handle_callbacks,234 &session->callback_queue);235 if (result != UDS_SUCCESS) {236 vdo_free(session);237 return result;238 }239 240 *index_session_ptr = session;241 return UDS_SUCCESS;242}243 244int uds_create_index_session(struct uds_index_session **session)245{246 if (session == NULL) {247 vdo_log_error("missing session pointer");248 return -EINVAL;249 }250 251 return uds_status_to_errno(make_empty_index_session(session));252}253 254static int __must_check start_loading_index_session(struct uds_index_session *index_session)255{256 int result;257 258 mutex_lock(&index_session->request_mutex);259 if (index_session->state & IS_FLAG_SUSPENDED) {260 vdo_log_info("Index session is suspended");261 result = -EBUSY;262 } else if (index_session->state != 0) {263 vdo_log_info("Index is already loaded");264 result = -EBUSY;265 } else {266 index_session->state |= IS_FLAG_LOADING;267 result = UDS_SUCCESS;268 }269 mutex_unlock(&index_session->request_mutex);270 return result;271}272 273static void finish_loading_index_session(struct uds_index_session *index_session,274 int result)275{276 mutex_lock(&index_session->request_mutex);277 index_session->state &= ~IS_FLAG_LOADING;278 if (result == UDS_SUCCESS)279 index_session->state |= IS_FLAG_LOADED;280 281 uds_broadcast_cond(&index_session->request_cond);282 mutex_unlock(&index_session->request_mutex);283}284 285static int initialize_index_session(struct uds_index_session *index_session,286 enum uds_open_index_type open_type)287{288 int result;289 struct uds_configuration *config;290 291 result = uds_make_configuration(&index_session->parameters, &config);292 if (result != UDS_SUCCESS) {293 vdo_log_error_strerror(result, "Failed to allocate config");294 return result;295 }296 297 memset(&index_session->stats, 0, sizeof(index_session->stats));298 result = uds_make_index(config, open_type, &index_session->load_context,299 enter_callback_stage, &index_session->index);300 if (result != UDS_SUCCESS)301 vdo_log_error_strerror(result, "Failed to make index");302 else303 uds_log_configuration(config);304 305 uds_free_configuration(config);306 return result;307}308 309static const char *get_open_type_string(enum uds_open_index_type open_type)310{311 switch (open_type) {312 case UDS_CREATE:313 return "creating index";314 case UDS_LOAD:315 return "loading or rebuilding index";316 case UDS_NO_REBUILD:317 return "loading index";318 default:319 return "unknown open method";320 }321}322 323/*324 * Open an index under the given session. This operation will fail if the325 * index session is suspended, or if there is already an open index.326 */327int uds_open_index(enum uds_open_index_type open_type,328 const struct uds_parameters *parameters,329 struct uds_index_session *session)330{331 int result;332 char name[BDEVNAME_SIZE];333 334 if (parameters == NULL) {335 vdo_log_error("missing required parameters");336 return -EINVAL;337 }338 if (parameters->bdev == NULL) {339 vdo_log_error("missing required block device");340 return -EINVAL;341 }342 if (session == NULL) {343 vdo_log_error("missing required session pointer");344 return -EINVAL;345 }346 347 result = start_loading_index_session(session);348 if (result != UDS_SUCCESS)349 return uds_status_to_errno(result);350 351 session->parameters = *parameters;352 format_dev_t(name, parameters->bdev->bd_dev);353 vdo_log_info("%s: %s", get_open_type_string(open_type), name);354 355 result = initialize_index_session(session, open_type);356 if (result != UDS_SUCCESS)357 vdo_log_error_strerror(result, "Failed %s",358 get_open_type_string(open_type));359 360 finish_loading_index_session(session, result);361 return uds_status_to_errno(result);362}363 364static void wait_for_no_requests_in_progress(struct uds_index_session *index_session)365{366 mutex_lock(&index_session->request_mutex);367 while (index_session->request_count > 0) {368 uds_wait_cond(&index_session->request_cond,369 &index_session->request_mutex);370 }371 mutex_unlock(&index_session->request_mutex);372}373 374static int __must_check save_index(struct uds_index_session *index_session)375{376 wait_for_no_requests_in_progress(index_session);377 return uds_save_index(index_session->index);378}379 380static void suspend_rebuild(struct uds_index_session *session)381{382 mutex_lock(&session->load_context.mutex);383 switch (session->load_context.status) {384 case INDEX_OPENING:385 session->load_context.status = INDEX_SUSPENDING;386 387 /* Wait until the index indicates that it is not replaying. */388 while ((session->load_context.status != INDEX_SUSPENDED) &&389 (session->load_context.status != INDEX_READY)) {390 uds_wait_cond(&session->load_context.cond,391 &session->load_context.mutex);392 }393 394 break;395 396 case INDEX_READY:397 /* Index load does not need to be suspended. */398 break;399 400 case INDEX_SUSPENDED:401 case INDEX_SUSPENDING:402 case INDEX_FREEING:403 default:404 /* These cases should not happen. */405 VDO_ASSERT_LOG_ONLY(false, "Bad load context state %u",406 session->load_context.status);407 break;408 }409 mutex_unlock(&session->load_context.mutex);410}411 412/*413 * Suspend index operation, draining all current index requests and preventing new index requests414 * from starting. Optionally saves all index data before returning.415 */416int uds_suspend_index_session(struct uds_index_session *session, bool save)417{418 int result = UDS_SUCCESS;419 bool no_work = false;420 bool rebuilding = false;421 422 /* Wait for any current index state change to complete. */423 mutex_lock(&session->request_mutex);424 while (session->state & IS_FLAG_CLOSING)425 uds_wait_cond(&session->request_cond, &session->request_mutex);426 427 if ((session->state & IS_FLAG_WAITING) || (session->state & IS_FLAG_DESTROYING)) {428 no_work = true;429 vdo_log_info("Index session is already changing state");430 result = -EBUSY;431 } else if (session->state & IS_FLAG_SUSPENDED) {432 no_work = true;433 } else if (session->state & IS_FLAG_LOADING) {434 session->state |= IS_FLAG_WAITING;435 rebuilding = true;436 } else if (session->state & IS_FLAG_LOADED) {437 session->state |= IS_FLAG_WAITING;438 } else {439 no_work = true;440 session->state |= IS_FLAG_SUSPENDED;441 uds_broadcast_cond(&session->request_cond);442 }443 mutex_unlock(&session->request_mutex);444 445 if (no_work)446 return uds_status_to_errno(result);447 448 if (rebuilding)449 suspend_rebuild(session);450 else if (save)451 result = save_index(session);452 else453 result = uds_flush_index_session(session);454 455 mutex_lock(&session->request_mutex);456 session->state &= ~IS_FLAG_WAITING;457 session->state |= IS_FLAG_SUSPENDED;458 uds_broadcast_cond(&session->request_cond);459 mutex_unlock(&session->request_mutex);460 return uds_status_to_errno(result);461}462 463static int replace_device(struct uds_index_session *session, struct block_device *bdev)464{465 int result;466 467 result = uds_replace_index_storage(session->index, bdev);468 if (result != UDS_SUCCESS)469 return result;470 471 session->parameters.bdev = bdev;472 return UDS_SUCCESS;473}474 475/*476 * Resume index operation after being suspended. If the index is suspended and the supplied block477 * device differs from the current backing store, the index will start using the new backing store.478 */479int uds_resume_index_session(struct uds_index_session *session,480 struct block_device *bdev)481{482 int result = UDS_SUCCESS;483 bool no_work = false;484 bool resume_replay = false;485 486 mutex_lock(&session->request_mutex);487 if (session->state & IS_FLAG_WAITING) {488 vdo_log_info("Index session is already changing state");489 no_work = true;490 result = -EBUSY;491 } else if (!(session->state & IS_FLAG_SUSPENDED)) {492 /* If not suspended, just succeed. */493 no_work = true;494 result = UDS_SUCCESS;495 } else {496 session->state |= IS_FLAG_WAITING;497 if (session->state & IS_FLAG_LOADING)498 resume_replay = true;499 }500 mutex_unlock(&session->request_mutex);501 502 if (no_work)503 return result;504 505 if ((session->index != NULL) && (bdev != session->parameters.bdev)) {506 result = replace_device(session, bdev);507 if (result != UDS_SUCCESS) {508 mutex_lock(&session->request_mutex);509 session->state &= ~IS_FLAG_WAITING;510 uds_broadcast_cond(&session->request_cond);511 mutex_unlock(&session->request_mutex);512 return uds_status_to_errno(result);513 }514 }515 516 if (resume_replay) {517 mutex_lock(&session->load_context.mutex);518 switch (session->load_context.status) {519 case INDEX_SUSPENDED:520 session->load_context.status = INDEX_OPENING;521 /* Notify the index to start replaying again. */522 uds_broadcast_cond(&session->load_context.cond);523 break;524 525 case INDEX_READY:526 /* There is no index rebuild to resume. */527 break;528 529 case INDEX_OPENING:530 case INDEX_SUSPENDING:531 case INDEX_FREEING:532 default:533 /* These cases should not happen; do nothing. */534 VDO_ASSERT_LOG_ONLY(false, "Bad load context state %u",535 session->load_context.status);536 break;537 }538 mutex_unlock(&session->load_context.mutex);539 }540 541 mutex_lock(&session->request_mutex);542 session->state &= ~IS_FLAG_WAITING;543 session->state &= ~IS_FLAG_SUSPENDED;544 uds_broadcast_cond(&session->request_cond);545 mutex_unlock(&session->request_mutex);546 return UDS_SUCCESS;547}548 549static int save_and_free_index(struct uds_index_session *index_session)550{551 int result = UDS_SUCCESS;552 bool suspended;553 struct uds_index *index = index_session->index;554 555 if (index == NULL)556 return UDS_SUCCESS;557 558 mutex_lock(&index_session->request_mutex);559 suspended = (index_session->state & IS_FLAG_SUSPENDED);560 mutex_unlock(&index_session->request_mutex);561 562 if (!suspended) {563 result = uds_save_index(index);564 if (result != UDS_SUCCESS)565 vdo_log_warning_strerror(result,566 "ignoring error from save_index");567 }568 uds_free_index(index);569 index_session->index = NULL;570 571 /*572 * Reset all index state that happens to be in the index573 * session, so it doesn't affect any future index.574 */575 mutex_lock(&index_session->load_context.mutex);576 index_session->load_context.status = INDEX_OPENING;577 mutex_unlock(&index_session->load_context.mutex);578 579 mutex_lock(&index_session->request_mutex);580 /* Only the suspend bit will remain relevant. */581 index_session->state &= IS_FLAG_SUSPENDED;582 mutex_unlock(&index_session->request_mutex);583 584 return result;585}586 587/* Save and close the current index. */588int uds_close_index(struct uds_index_session *index_session)589{590 int result = UDS_SUCCESS;591 592 /* Wait for any current index state change to complete. */593 mutex_lock(&index_session->request_mutex);594 while ((index_session->state & IS_FLAG_WAITING) ||595 (index_session->state & IS_FLAG_CLOSING)) {596 uds_wait_cond(&index_session->request_cond,597 &index_session->request_mutex);598 }599 600 if (index_session->state & IS_FLAG_SUSPENDED) {601 vdo_log_info("Index session is suspended");602 result = -EBUSY;603 } else if ((index_session->state & IS_FLAG_DESTROYING) ||604 !(index_session->state & IS_FLAG_LOADED)) {605 /* The index doesn't exist, hasn't finished loading, or is being destroyed. */606 result = UDS_NO_INDEX;607 } else {608 index_session->state |= IS_FLAG_CLOSING;609 }610 mutex_unlock(&index_session->request_mutex);611 if (result != UDS_SUCCESS)612 return uds_status_to_errno(result);613 614 vdo_log_debug("Closing index");615 wait_for_no_requests_in_progress(index_session);616 result = save_and_free_index(index_session);617 vdo_log_debug("Closed index");618 619 mutex_lock(&index_session->request_mutex);620 index_session->state &= ~IS_FLAG_CLOSING;621 uds_broadcast_cond(&index_session->request_cond);622 mutex_unlock(&index_session->request_mutex);623 return uds_status_to_errno(result);624}625 626/* This will save and close an open index before destroying the session. */627int uds_destroy_index_session(struct uds_index_session *index_session)628{629 int result;630 bool load_pending = false;631 632 vdo_log_debug("Destroying index session");633 634 /* Wait for any current index state change to complete. */635 mutex_lock(&index_session->request_mutex);636 while ((index_session->state & IS_FLAG_WAITING) ||637 (index_session->state & IS_FLAG_CLOSING)) {638 uds_wait_cond(&index_session->request_cond,639 &index_session->request_mutex);640 }641 642 if (index_session->state & IS_FLAG_DESTROYING) {643 mutex_unlock(&index_session->request_mutex);644 vdo_log_info("Index session is already closing");645 return -EBUSY;646 }647 648 index_session->state |= IS_FLAG_DESTROYING;649 load_pending = ((index_session->state & IS_FLAG_LOADING) &&650 (index_session->state & IS_FLAG_SUSPENDED));651 mutex_unlock(&index_session->request_mutex);652 653 if (load_pending) {654 /* Tell the index to terminate the rebuild. */655 mutex_lock(&index_session->load_context.mutex);656 if (index_session->load_context.status == INDEX_SUSPENDED) {657 index_session->load_context.status = INDEX_FREEING;658 uds_broadcast_cond(&index_session->load_context.cond);659 }660 mutex_unlock(&index_session->load_context.mutex);661 662 /* Wait until the load exits before proceeding. */663 mutex_lock(&index_session->request_mutex);664 while (index_session->state & IS_FLAG_LOADING) {665 uds_wait_cond(&index_session->request_cond,666 &index_session->request_mutex);667 }668 mutex_unlock(&index_session->request_mutex);669 }670 671 wait_for_no_requests_in_progress(index_session);672 result = save_and_free_index(index_session);673 uds_request_queue_finish(index_session->callback_queue);674 index_session->callback_queue = NULL;675 vdo_log_debug("Destroyed index session");676 vdo_free(index_session);677 return uds_status_to_errno(result);678}679 680/* Wait until all callbacks for index operations are complete. */681int uds_flush_index_session(struct uds_index_session *index_session)682{683 wait_for_no_requests_in_progress(index_session);684 uds_wait_for_idle_index(index_session->index);685 return UDS_SUCCESS;686}687 688/* Statistics collection is intended to be thread-safe. */689static void collect_stats(const struct uds_index_session *index_session,690 struct uds_index_stats *stats)691{692 const struct session_stats *session_stats = &index_session->stats;693 694 stats->current_time = ktime_to_seconds(current_time_ns(CLOCK_REALTIME));695 stats->posts_found = READ_ONCE(session_stats->posts_found);696 stats->in_memory_posts_found = READ_ONCE(session_stats->posts_found_open_chapter);697 stats->dense_posts_found = READ_ONCE(session_stats->posts_found_dense);698 stats->sparse_posts_found = READ_ONCE(session_stats->posts_found_sparse);699 stats->posts_not_found = READ_ONCE(session_stats->posts_not_found);700 stats->updates_found = READ_ONCE(session_stats->updates_found);701 stats->updates_not_found = READ_ONCE(session_stats->updates_not_found);702 stats->deletions_found = READ_ONCE(session_stats->deletions_found);703 stats->deletions_not_found = READ_ONCE(session_stats->deletions_not_found);704 stats->queries_found = READ_ONCE(session_stats->queries_found);705 stats->queries_not_found = READ_ONCE(session_stats->queries_not_found);706 stats->requests = READ_ONCE(session_stats->requests);707}708 709int uds_get_index_session_stats(struct uds_index_session *index_session,710 struct uds_index_stats *stats)711{712 if (stats == NULL) {713 vdo_log_error("received a NULL index stats pointer");714 return -EINVAL;715 }716 717 collect_stats(index_session, stats);718 if (index_session->index != NULL) {719 uds_get_index_stats(index_session->index, stats);720 } else {721 stats->entries_indexed = 0;722 stats->memory_used = 0;723 stats->collisions = 0;724 stats->entries_discarded = 0;725 }726 727 return UDS_SUCCESS;728}729 730void uds_wait_cond(struct cond_var *cv, struct mutex *mutex)731{732 DEFINE_WAIT(__wait);733 734 prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE);735 mutex_unlock(mutex);736 schedule();737 finish_wait(&cv->wait_queue, &__wait);738 mutex_lock(mutex);739}740