486 lines · c
1// SPDX-License-Identifier: GPL-2.02#include "bcachefs.h"3#include "btree_iter.h"4#include "error.h"5#include "journal.h"6#include "recovery_passes.h"7#include "super.h"8#include "thread_with_file.h"9 10#define FSCK_ERR_RATELIMIT_NR 1011 12bool bch2_inconsistent_error(struct bch_fs *c)13{14 set_bit(BCH_FS_error, &c->flags);15 16 switch (c->opts.errors) {17 case BCH_ON_ERROR_continue:18 return false;19 case BCH_ON_ERROR_fix_safe:20 case BCH_ON_ERROR_ro:21 if (bch2_fs_emergency_read_only(c))22 bch_err(c, "inconsistency detected - emergency read only at journal seq %llu",23 journal_cur_seq(&c->journal));24 return true;25 case BCH_ON_ERROR_panic:26 panic(bch2_fmt(c, "panic after error"));27 return true;28 default:29 BUG();30 }31}32 33int bch2_topology_error(struct bch_fs *c)34{35 set_bit(BCH_FS_topology_error, &c->flags);36 if (!test_bit(BCH_FS_fsck_running, &c->flags)) {37 bch2_inconsistent_error(c);38 return -BCH_ERR_btree_need_topology_repair;39 } else {40 return bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology) ?:41 -BCH_ERR_btree_node_read_validate_error;42 }43}44 45void bch2_fatal_error(struct bch_fs *c)46{47 if (bch2_fs_emergency_read_only(c))48 bch_err(c, "fatal error - emergency read only");49}50 51void bch2_io_error_work(struct work_struct *work)52{53 struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work);54 struct bch_fs *c = ca->fs;55 bool dev;56 57 down_write(&c->state_lock);58 dev = bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_ro,59 BCH_FORCE_IF_DEGRADED);60 if (dev61 ? __bch2_dev_set_state(c, ca, BCH_MEMBER_STATE_ro,62 BCH_FORCE_IF_DEGRADED)63 : bch2_fs_emergency_read_only(c))64 bch_err(ca,65 "too many IO errors, setting %s RO",66 dev ? "device" : "filesystem");67 up_write(&c->state_lock);68}69 70void bch2_io_error(struct bch_dev *ca, enum bch_member_error_type type)71{72 atomic64_inc(&ca->errors[type]);73 //queue_work(system_long_wq, &ca->io_error_work);74}75 76enum ask_yn {77 YN_NO,78 YN_YES,79 YN_ALLNO,80 YN_ALLYES,81};82 83static enum ask_yn parse_yn_response(char *buf)84{85 buf = strim(buf);86 87 if (strlen(buf) == 1)88 switch (buf[0]) {89 case 'n':90 return YN_NO;91 case 'y':92 return YN_YES;93 case 'N':94 return YN_ALLNO;95 case 'Y':96 return YN_ALLYES;97 }98 return -1;99}100 101#ifdef __KERNEL__102static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)103{104 struct stdio_redirect *stdio = c->stdio;105 106 if (c->stdio_filter && c->stdio_filter != current)107 stdio = NULL;108 109 if (!stdio)110 return YN_NO;111 112 if (trans)113 bch2_trans_unlock(trans);114 115 unsigned long unlock_long_at = trans ? jiffies + HZ * 2 : 0;116 darray_char line = {};117 int ret;118 119 do {120 unsigned long t;121 bch2_print(c, " (y,n, or Y,N for all errors of this type) ");122rewait:123 t = unlock_long_at124 ? max_t(long, unlock_long_at - jiffies, 0)125 : MAX_SCHEDULE_TIMEOUT;126 127 int r = bch2_stdio_redirect_readline_timeout(stdio, &line, t);128 if (r == -ETIME) {129 bch2_trans_unlock_long(trans);130 unlock_long_at = 0;131 goto rewait;132 }133 134 if (r < 0) {135 ret = YN_NO;136 break;137 }138 139 darray_last(line) = '\0';140 } while ((ret = parse_yn_response(line.data)) < 0);141 142 darray_exit(&line);143 return ret;144}145#else146 147#include "tools-util.h"148 149static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)150{151 char *buf = NULL;152 size_t buflen = 0;153 int ret;154 155 do {156 fputs(" (y,n, or Y,N for all errors of this type) ", stdout);157 fflush(stdout);158 159 if (getline(&buf, &buflen, stdin) < 0)160 die("error reading from standard input");161 } while ((ret = parse_yn_response(buf)) < 0);162 163 free(buf);164 return ret;165}166 167#endif168 169static struct fsck_err_state *fsck_err_get(struct bch_fs *c, const char *fmt)170{171 struct fsck_err_state *s;172 173 if (!test_bit(BCH_FS_fsck_running, &c->flags))174 return NULL;175 176 list_for_each_entry(s, &c->fsck_error_msgs, list)177 if (s->fmt == fmt) {178 /*179 * move it to the head of the list: repeated fsck errors180 * are common181 */182 list_move(&s->list, &c->fsck_error_msgs);183 return s;184 }185 186 s = kzalloc(sizeof(*s), GFP_NOFS);187 if (!s) {188 if (!c->fsck_alloc_msgs_err)189 bch_err(c, "kmalloc err, cannot ratelimit fsck errs");190 c->fsck_alloc_msgs_err = true;191 return NULL;192 }193 194 INIT_LIST_HEAD(&s->list);195 s->fmt = fmt;196 list_add(&s->list, &c->fsck_error_msgs);197 return s;198}199 200/* s/fix?/fixing/ s/recreate?/recreating/ */201static void prt_actioning(struct printbuf *out, const char *action)202{203 unsigned len = strlen(action);204 205 BUG_ON(action[len - 1] != '?');206 --len;207 208 if (action[len - 1] == 'e')209 --len;210 211 prt_bytes(out, action, len);212 prt_str(out, "ing");213}214 215static const u8 fsck_flags_extra[] = {216#define x(t, n, flags) [BCH_FSCK_ERR_##t] = flags,217 BCH_SB_ERRS()218#undef x219};220 221int __bch2_fsck_err(struct bch_fs *c,222 struct btree_trans *trans,223 enum bch_fsck_flags flags,224 enum bch_sb_error_id err,225 const char *fmt, ...)226{227 struct fsck_err_state *s = NULL;228 va_list args;229 bool print = true, suppressing = false, inconsistent = false;230 struct printbuf buf = PRINTBUF, *out = &buf;231 int ret = -BCH_ERR_fsck_ignore;232 const char *action_orig = "fix?", *action = action_orig;233 234 might_sleep();235 236 if (!WARN_ON(err >= ARRAY_SIZE(fsck_flags_extra)))237 flags |= fsck_flags_extra[err];238 239 if (!c)240 c = trans->c;241 242 /*243 * Ugly: if there's a transaction in the current task it has to be244 * passed in to unlock if we prompt for user input.245 *246 * But, plumbing a transaction and transaction restarts into247 * bkey_validate() is problematic.248 *249 * So:250 * - make all bkey errors AUTOFIX, they're simple anyways (we just251 * delete the key)252 * - and we don't need to warn if we're not prompting253 */254 WARN_ON((flags & FSCK_CAN_FIX) &&255 !(flags & FSCK_AUTOFIX) &&256 !trans &&257 bch2_current_has_btree_trans(c));258 259 if ((flags & FSCK_CAN_FIX) &&260 test_bit(err, c->sb.errors_silent))261 return -BCH_ERR_fsck_fix;262 263 bch2_sb_error_count(c, err);264 265 va_start(args, fmt);266 prt_vprintf(out, fmt, args);267 va_end(args);268 269 /* Custom fix/continue/recreate/etc.? */270 if (out->buf[out->pos - 1] == '?') {271 const char *p = strrchr(out->buf, ',');272 if (p) {273 out->pos = p - out->buf;274 action = kstrdup(p + 2, GFP_KERNEL);275 if (!action) {276 ret = -ENOMEM;277 goto err;278 }279 }280 }281 282 mutex_lock(&c->fsck_error_msgs_lock);283 s = fsck_err_get(c, fmt);284 if (s) {285 /*286 * We may be called multiple times for the same error on287 * transaction restart - this memoizes instead of asking the user288 * multiple times for the same error:289 */290 if (s->last_msg && !strcmp(buf.buf, s->last_msg)) {291 ret = s->ret;292 mutex_unlock(&c->fsck_error_msgs_lock);293 goto err;294 }295 296 kfree(s->last_msg);297 s->last_msg = kstrdup(buf.buf, GFP_KERNEL);298 if (!s->last_msg) {299 mutex_unlock(&c->fsck_error_msgs_lock);300 ret = -ENOMEM;301 goto err;302 }303 304 if (c->opts.ratelimit_errors &&305 !(flags & FSCK_NO_RATELIMIT) &&306 s->nr >= FSCK_ERR_RATELIMIT_NR) {307 if (s->nr == FSCK_ERR_RATELIMIT_NR)308 suppressing = true;309 else310 print = false;311 }312 313 s->nr++;314 }315 316#ifdef BCACHEFS_LOG_PREFIX317 if (!strncmp(fmt, "bcachefs:", 9))318 prt_printf(out, bch2_log_msg(c, ""));319#endif320 321 if ((flags & FSCK_CAN_FIX) &&322 (flags & FSCK_AUTOFIX) &&323 (c->opts.errors == BCH_ON_ERROR_continue ||324 c->opts.errors == BCH_ON_ERROR_fix_safe)) {325 prt_str(out, ", ");326 prt_actioning(out, action);327 ret = -BCH_ERR_fsck_fix;328 } else if (!test_bit(BCH_FS_fsck_running, &c->flags)) {329 if (c->opts.errors != BCH_ON_ERROR_continue ||330 !(flags & (FSCK_CAN_FIX|FSCK_CAN_IGNORE))) {331 prt_str(out, ", shutting down");332 inconsistent = true;333 ret = -BCH_ERR_fsck_errors_not_fixed;334 } else if (flags & FSCK_CAN_FIX) {335 prt_str(out, ", ");336 prt_actioning(out, action);337 ret = -BCH_ERR_fsck_fix;338 } else {339 prt_str(out, ", continuing");340 ret = -BCH_ERR_fsck_ignore;341 }342 } else if (c->opts.fix_errors == FSCK_FIX_exit) {343 prt_str(out, ", exiting");344 ret = -BCH_ERR_fsck_errors_not_fixed;345 } else if (flags & FSCK_CAN_FIX) {346 int fix = s && s->fix347 ? s->fix348 : c->opts.fix_errors;349 350 if (fix == FSCK_FIX_ask) {351 prt_str(out, ", ");352 prt_str(out, action);353 354 if (bch2_fs_stdio_redirect(c))355 bch2_print(c, "%s", out->buf);356 else357 bch2_print_string_as_lines(KERN_ERR, out->buf);358 print = false;359 360 int ask = bch2_fsck_ask_yn(c, trans);361 362 if (trans) {363 ret = bch2_trans_relock(trans);364 if (ret) {365 mutex_unlock(&c->fsck_error_msgs_lock);366 goto err;367 }368 }369 370 if (ask >= YN_ALLNO && s)371 s->fix = ask == YN_ALLNO372 ? FSCK_FIX_no373 : FSCK_FIX_yes;374 375 ret = ask & 1376 ? -BCH_ERR_fsck_fix377 : -BCH_ERR_fsck_ignore;378 } else if (fix == FSCK_FIX_yes ||379 (c->opts.nochanges &&380 !(flags & FSCK_CAN_IGNORE))) {381 prt_str(out, ", ");382 prt_actioning(out, action);383 ret = -BCH_ERR_fsck_fix;384 } else {385 prt_str(out, ", not ");386 prt_actioning(out, action);387 }388 } else if (flags & FSCK_NEED_FSCK) {389 prt_str(out, " (run fsck to correct)");390 } else {391 prt_str(out, " (repair unimplemented)");392 }393 394 if (ret == -BCH_ERR_fsck_ignore &&395 (c->opts.fix_errors == FSCK_FIX_exit ||396 !(flags & FSCK_CAN_IGNORE)))397 ret = -BCH_ERR_fsck_errors_not_fixed;398 399 bool exiting =400 test_bit(BCH_FS_fsck_running, &c->flags) &&401 (ret != -BCH_ERR_fsck_fix &&402 ret != -BCH_ERR_fsck_ignore);403 404 if (exiting)405 print = true;406 407 if (print) {408 if (bch2_fs_stdio_redirect(c))409 bch2_print(c, "%s\n", out->buf);410 else411 bch2_print_string_as_lines(KERN_ERR, out->buf);412 }413 414 if (exiting)415 bch_err(c, "Unable to continue, halting");416 else if (suppressing)417 bch_err(c, "Ratelimiting new instances of previous error");418 419 if (s)420 s->ret = ret;421 422 mutex_unlock(&c->fsck_error_msgs_lock);423 424 if (inconsistent)425 bch2_inconsistent_error(c);426 427 if (ret == -BCH_ERR_fsck_fix) {428 set_bit(BCH_FS_errors_fixed, &c->flags);429 } else {430 set_bit(BCH_FS_errors_not_fixed, &c->flags);431 set_bit(BCH_FS_error, &c->flags);432 }433err:434 if (action != action_orig)435 kfree(action);436 printbuf_exit(&buf);437 return ret;438}439 440int __bch2_bkey_fsck_err(struct bch_fs *c,441 struct bkey_s_c k,442 enum bch_validate_flags validate_flags,443 enum bch_sb_error_id err,444 const char *fmt, ...)445{446 if (validate_flags & BCH_VALIDATE_silent)447 return -BCH_ERR_fsck_delete_bkey;448 449 unsigned fsck_flags = 0;450 if (!(validate_flags & (BCH_VALIDATE_write|BCH_VALIDATE_commit)))451 fsck_flags |= FSCK_AUTOFIX|FSCK_CAN_FIX;452 453 struct printbuf buf = PRINTBUF;454 va_list args;455 456 prt_str(&buf, "invalid bkey ");457 bch2_bkey_val_to_text(&buf, c, k);458 prt_str(&buf, "\n ");459 va_start(args, fmt);460 prt_vprintf(&buf, fmt, args);461 va_end(args);462 prt_str(&buf, ": delete?");463 464 int ret = __bch2_fsck_err(c, NULL, fsck_flags, err, "%s", buf.buf);465 printbuf_exit(&buf);466 return ret;467}468 469void bch2_flush_fsck_errs(struct bch_fs *c)470{471 struct fsck_err_state *s, *n;472 473 mutex_lock(&c->fsck_error_msgs_lock);474 475 list_for_each_entry_safe(s, n, &c->fsck_error_msgs, list) {476 if (s->ratelimited && s->last_msg)477 bch_err(c, "Saw %llu errors like:\n %s", s->nr, s->last_msg);478 479 list_del(&s->list);480 kfree(s->last_msg);481 kfree(s);482 }483 484 mutex_unlock(&c->fsck_error_msgs_lock);485}486