diff --git a/fs/bcachefs.h b/fs/bcachefs.h index 95f27b8e4a..363dfe992a 100644 --- a/fs/bcachefs.h +++ b/fs/bcachefs.h @@ -702,6 +702,7 @@ struct journal_seq_blacklist_table { x(node_rewrite) \ x(stripe_create) \ x(stripe_delete) \ + x(stripe_scrub) \ x(reflink) \ x(fallocate) \ x(fsync) \ diff --git a/fs/data/ec/create.c b/fs/data/ec/create.c index 653e5ec04e..7e0efeded1 100644 --- a/fs/data/ec/create.c +++ b/fs/data/ec/create.c @@ -172,6 +172,147 @@ void bch2_do_stripe_deletes(struct bch_fs *c) enumerated_ref_put(&c->writes, BCH_WRITE_REF_stripe_delete); } +/* stripe scrub, queued by the read path */ + +#define STRIPE_SCRUB_INTERVAL (60 * HZ) + +/* + * A read found a bad block in stripe @idx, one of @blocks: scrub them, which + * repairs the stripe. A stripe scrubbed in the last STRIPE_SCRUB_INTERVAL isn't + * queued again: a repair that narrows the stripe leaves the bad block in place + * until reconcile has evacuated it, and reads keep failing until then. When the + * queue is full, the next full scrub finds it. + */ +bool bch2_ec_stripe_scrub_queue(struct bch_fs *c, u64 idx, u32 blocks) +{ + struct bch_fs_ec *ec = &c->ec; + struct ec_stripe_scrub *slot = NULL; + + if (!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_stripe_scrub)) + return false; + + scoped_guard(spinlock, &ec->stripe_scrub_lock) { + for (struct ec_stripe_scrub *s = ec->stripe_scrub; + s < ec->stripe_scrub + ARRAY_SIZE(ec->stripe_scrub); + s++) { + bool expired = s->done_at && + time_after_eq(jiffies, s->done_at + STRIPE_SCRUB_INTERVAL); + + if (s->blocks && s->idx == idx && !expired) { + if (!s->running && !s->done_at) + s->blocks |= blocks; + slot = NULL; + break; + } + + if (!slot && (!s->blocks || expired)) + slot = s; + } + + if (slot) + *slot = (struct ec_stripe_scrub) { .idx = idx, .blocks = blocks }; + } + + if (slot) + queue_work(c->write_ref_wq, &ec->stripe_scrub_work); + else + enumerated_ref_put(&c->writes, BCH_WRITE_REF_stripe_scrub); + return slot != NULL; +} + +static int ec_stripe_scrub_get(struct bch_fs *c, u64 idx, struct bkey_buf *sk) +{ + CLASS(btree_trans, trans)(c); + CLASS(btree_iter, iter)(trans, BTREE_ID_stripes, POS(0, idx), 0); + struct bkey_s_c k; + + try(lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_slot(&iter)))); + bch2_bkey_buf_reassemble(sk, k); + return 0; +} + +static int ec_stripe_scrub_pred(struct btree_trans *trans, void *arg, + enum btree_id btree, struct bkey_s_c k, + struct bch_inode_opts *io_opts, + struct data_update_opts *data_opts) +{ + return false; +} + +/* + * A backpointer walk over a block's last sector finds its stripe backpointer + * and, with verify_stripes, checks and repairs the block as scrub does. + */ +static int ec_stripe_scrub(struct bch_fs *c, u64 idx, u32 blocks) +{ + struct bkey_buf sk __cleanup(bch2_bkey_buf_exit); + bch2_bkey_buf_init(&sk); + + try(ec_stripe_scrub_get(c, idx, &sk)); + if (sk.k->k.type != KEY_TYPE_stripe) + return 0; + + const struct bch_stripe *v = &bkey_i_to_stripe(sk.k)->v; + struct bch_move_stats stats; + bch2_move_stats_init(&stats, "stripe_scrub"); + + for (unsigned i = 0; i < v->nr_blocks; i++) + if ((blocks & BIT(i)) && v->ptrs[i].dev != BCH_SB_MEMBER_INVALID) { + u64 end = v->ptrs[i].offset + le16_to_cpu(v->sectors); + + try(bch2_move_data_phys(c, v->ptrs[i].dev, end - 1, end, + BIT(BCH_DATA_user)|BIT(BCH_DATA_parity), + NULL, &stats, + writepoint_hashed((unsigned long) current), + false, true, ec_stripe_scrub_pred, NULL)); + } + + /* The parity doesn't match the data, the stripe changed, or a block couldn't be read: */ + if (!atomic64_read(&stats.sectors_error_corrected) && + !atomic64_read(&stats.sectors_error_uncorrected)) { + CLASS(bch_log_msg_ratelimited, msg)(c); + prt_printf(&msg.m, "stripe %llu: queued for scrub by a read, but no block it could read fails the stripe's checksums\n", + idx); + bch2_bkey_val_to_text(&msg.m, c, bkey_i_to_s_c(sk.k)); + } + return 0; +} + +void bch2_ec_stripe_scrub_work(struct work_struct *work) +{ + struct bch_fs *c = container_of(work, struct bch_fs, ec.stripe_scrub_work); + struct bch_fs_ec *ec = &c->ec; + + while (true) { + struct ec_stripe_scrub *s = NULL; + + scoped_guard(spinlock, &ec->stripe_scrub_lock) + for (unsigned i = 0; i < ARRAY_SIZE(ec->stripe_scrub); i++) + if (ec->stripe_scrub[i].blocks && + !ec->stripe_scrub[i].running && + !ec->stripe_scrub[i].done_at) { + s = &ec->stripe_scrub[i]; + s->running = true; + break; + } + if (!s) + break; + + /* Going read-only: drop what's left. A running entry is only ours to change. */ + int ret = test_bit(BCH_FS_going_ro, &c->flags) + ? 0 + : ec_stripe_scrub(c, s->idx, s->blocks); + if (ret && !bch2_err_matches(ret, EROFS)) + bch_err_msg(c, ret, "scrubbing stripe %llu", s->idx); + + scoped_guard(spinlock, &ec->stripe_scrub_lock) { + s->running = false; + s->done_at = jiffies ?: 1; + } + enumerated_ref_put(&c->writes, BCH_WRITE_REF_stripe_scrub); + } +} + /* stripe creation */ static int ec_stripe_key_update(struct btree_trans *trans, @@ -2009,16 +2150,10 @@ static CLOSURE_CALLBACK(ec_old_stripe_fold) return; } - /* - * Data is only declared lost if it really can't be read: if anything we - * did read is bad, or the rebuild doesn't check out, read the skipped - * blocks too and start over. - */ + /* Skipped blocks are read after all before any data is declared lost: */ u32 skipped = s->old_stripe_skipped; if (skipped) { - struct ec_stripe_buf *buf = &s->old_stripe; - - if (bch2_stripe_buf_blocks_good(buf, (BIT(buf->key.v.nr_blocks) - 1) & ~skipped)) { + if (bch2_stripe_buf_blocks_good(&s->old_stripe, ~skipped)) { ec_old_stripe_read_done(s); if (!s->old_stripe_err) { closure_return(cl); @@ -2027,14 +2162,10 @@ static CLOSURE_CALLBACK(ec_old_stripe_fold) } s->old_stripe_skipped = 0; - s->old_stripe_err = 0; s->old_stripe_lost_blocks = 0; - memset(buf->err[STRIPE_BUF_POST_RECOV], 0, sizeof(buf->err[STRIPE_BUF_POST_RECOV])); - for (unsigned i = 0; i < buf->key.v.nr_blocks; i++) - if (skipped & BIT(i)) { - buf->err[STRIPE_BUF_PRE_RECOV][i] = 0; - bch2_ec_block_io(s->c, buf, REQ_OP_READ, i); - } + for (unsigned i = 0; i < s->old_stripe.key.v.nr_blocks; i++) + if (skipped & BIT(i)) + bch2_ec_block_io(s->c, &s->old_stripe, REQ_OP_READ, i); continue_at(cl, ec_old_stripe_fold, system_dfl_wq); return; } @@ -2058,20 +2189,13 @@ static void ec_old_stripe_read(struct bch_fs *c, struct ec_stripe_new *s) closure_init(&s->cl, NULL); closure_init(&s->old_stripe.io, &s->cl); - /* - * A carried block on a device much slower than the rest of the stripe - - * typically a failing drive being evacuated - is rebuilt from the other - * blocks instead, so read all of those now: - */ + /* A carried block on a device much slower than the rest is rebuilt instead: */ u32 required = ec_old_stripe_required(s); s->old_stripe_skipped = bch2_ec_read_around_skip(c, &s->old_stripe.key.v, required); s->old_stripe_read_all = s->old_stripe_skipped != 0; - for (unsigned i = 0; i < s->old_stripe.key.v.nr_blocks; i++) - if (s->old_stripe_skipped & BIT(i)) - s->old_stripe.err[STRIPE_BUF_PRE_RECOV][i] = -BCH_ERR_stripe_read_skipped; - else if (s->old_stripe_read_all || (required & BIT(i))) - bch2_ec_block_io(c, &s->old_stripe, REQ_OP_READ, i); + bch2_stripe_buf_read(c, &s->old_stripe, + s->old_stripe_skipped ? ~s->old_stripe_skipped : required); /* * Not stripe_create_wq: create waits on the fold, so running them on @@ -2822,9 +2946,7 @@ static void stripe_repair_rebuild_abort(struct bch_fs *c, struct ec_stripe_new * /* * Read @s into @buf as ec_old_stripe_fold() would: the blocks in @required, - * then the rest if one of those is bad (@read_all). A block in @required on a - * device much slower than the rest of the stripe is rebuilt from the others - * instead, and read after all if that doesn't check out. The caller has @s open. + * then the rest if one of those is bad (@read_all). The caller has @s open. */ static int stripe_repair_read(struct btree_trans *trans, struct bkey_s_c_stripe s, u32 required, struct ec_stripe_buf *buf, bool *read_all) @@ -2847,40 +2969,24 @@ static int stripe_repair_read(struct btree_trans *trans, struct bkey_s_c_stripe bch2_trans_unlock_long(trans); - u32 all = BIT(buf->key.v.nr_blocks) - 1; u32 skipped = bch2_ec_read_around_skip(c, &buf->key.v, required); - unsigned long blocks = skipped ? all & ~skipped : required; + u32 read = skipped ? ~skipped : required; unsigned i; - for_each_set_bit(i, &blocks, BCH_BKEY_PTRS_MAX) - bch2_ec_block_io(c, buf, REQ_OP_READ, i); - closure_sync(&buf->io); - - if (skipped) { - blocks = skipped; - for_each_set_bit(i, &blocks, BCH_BKEY_PTRS_MAX) - buf->err[STRIPE_BUF_PRE_RECOV][i] = -BCH_ERR_stripe_read_skipped; - - if (bch2_stripe_buf_blocks_good(buf, all & ~skipped) && - !bch2_stripe_buf_validate_msg(c, buf, true, required)) { - *read_all = false; - return 0; - } - /* Data is only declared lost if it really can't be read: */ - memset(buf->err[STRIPE_BUF_POST_RECOV], 0, sizeof(buf->err[STRIPE_BUF_POST_RECOV])); - for_each_set_bit(i, &blocks, BCH_BKEY_PTRS_MAX) { - buf->err[STRIPE_BUF_PRE_RECOV][i] = 0; - bch2_ec_block_io(c, buf, REQ_OP_READ, i); - } - closure_sync(&buf->io); + bch2_stripe_buf_read(c, buf, read); + closure_sync(&buf->io); - *read_all = !bch2_stripe_buf_blocks_good(buf, required); + if (skipped && + bch2_stripe_buf_blocks_good(buf, read) && + !bch2_stripe_buf_validate_msg(c, buf, true, required)) { + *read_all = false; return 0; } - *read_all = !bch2_stripe_buf_blocks_good(buf, required); + /* Skipped blocks are read after all before any data is declared lost: */ + *read_all = skipped || !bch2_stripe_buf_blocks_good(buf, required); if (*read_all) { - blocks = ~required & (BIT(buf->key.v.nr_blocks) - 1); + unsigned long blocks = ~read & (BIT(buf->key.v.nr_blocks) - 1); for_each_set_bit(i, &blocks, BCH_BKEY_PTRS_MAX) bch2_ec_block_io(c, buf, REQ_OP_READ, i); closure_sync(&buf->io); diff --git a/fs/data/ec/create.h b/fs/data/ec/create.h index f262367f25..c0d7446e3d 100644 --- a/fs/data/ec/create.h +++ b/fs/data/ec/create.h @@ -79,8 +79,7 @@ struct ec_stripe_new { */ int old_stripe_err; u32 old_stripe_lost_blocks; - /* Carried blocks on slow devices, rebuilt from the others */ - u32 old_stripe_skipped; + u32 old_stripe_skipped; /* slow, rebuilt from the others */ struct bch_devs_mask devs; enum bch_watermark watermark; @@ -244,6 +243,9 @@ static inline void ec_stripe_new_put(struct bch_fs *c, struct ec_stripe_new *s, void bch2_ec_stripe_delete_work(struct work_struct *); +bool bch2_ec_stripe_scrub_queue(struct bch_fs *, u64, u32); +void bch2_ec_stripe_scrub_work(struct work_struct *); + void bch2_new_stripes_to_text(struct printbuf *, struct bch_fs *); struct moving_context; diff --git a/fs/data/ec/init.c b/fs/data/ec/init.c index 9d005b5704..de1b7635bb 100644 --- a/fs/data/ec/init.c +++ b/fs/data/ec/init.c @@ -403,6 +403,9 @@ void bch2_fs_ec_init_early(struct bch_fs *c) init_waitqueue_head(&c->ec.stripe_new_wait); INIT_WORK(&c->ec.stripe_delete_work, bch2_ec_stripe_delete_work); + + spin_lock_init(&c->ec.stripe_scrub_lock); + INIT_WORK(&c->ec.stripe_scrub_work, bch2_ec_stripe_scrub_work); } int bch2_fs_ec_init(struct bch_fs *c) diff --git a/fs/data/ec/io.c b/fs/data/ec/io.c index 17c34c0333..87fe5bb202 100644 --- a/fs/data/ec/io.c +++ b/fs/data/ec/io.c @@ -7,6 +7,7 @@ #include "btree/iter.h" #include "data/checksum.h" +#include "data/ec/create.h" #include "data/ec/io.h" #include "data/ec/trigger.h" #include "data/extents.h" @@ -191,17 +192,17 @@ void bch2_ec_stripe_buf_move(struct ec_stripe_buf *dst, struct ec_stripe_buf *sr } /* Over ec_stripe_buf_limit, waits on @cl if given, and otherwise goes over it */ -int __bch2_ec_stripe_buf_init(struct bch_fs *c, - struct ec_stripe_buf *buf, - unsigned offset, unsigned size, - struct closure *cl, enum ec_stripe_buf_flags flags) +int bch2_ec_stripe_buf_init(struct bch_fs *c, + struct ec_stripe_buf *buf, + unsigned offset, unsigned size, + struct closure *cl) { unsigned csum_granularity = 1U << buf->key.v.csum_granularity_bits; unsigned end = offset + size; BUG_ON(end > le16_to_cpu(buf->key.v.sectors)); - if (!(flags & EC_STRIPE_BUF_unaligned)) { + if (!buf->unaligned) { offset = round_down(offset, csum_granularity); end = min_t(unsigned, le16_to_cpu(buf->key.v.sectors), round_up(end, csum_granularity)); @@ -224,7 +225,6 @@ int __bch2_ec_stripe_buf_init(struct bch_fs *c, buf->c = c; buf->offset = offset; buf->size = end - offset; - buf->unaligned = flags & EC_STRIPE_BUF_unaligned; for (unsigned i = 0; i < buf->key.v.nr_blocks; i++) { buf->data[i] = kvmalloc(buf->size << 9, GFP_KERNEL); @@ -547,6 +547,7 @@ static int bch2_stripe_buf_validate(struct bch_fs *c, struct ec_stripe_buf *buf, { closure_sync(&buf->io); + memset(buf->err[STRIPE_BUF_POST_RECOV], 0, sizeof(buf->err[STRIPE_BUF_POST_RECOV])); bch2_ec_validate_checksums(c, buf, false, STRIPE_BUF_PRE_RECOV); if (!ec_nr_failed(buf, STRIPE_BUF_PRE_RECOV)) return 0; @@ -684,9 +685,20 @@ static void ec_block_endio(struct bio *bio) void bch2_ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf, blk_opf_t opf, unsigned idx) { + buf->err[STRIPE_BUF_PRE_RECOV][idx] = 0; bch2_ec_block_io_range(c, buf, opf, idx, buf->offset, buf->size); } +/* Blocks not read are erasures, rebuilt from the rest when they're needed */ +void bch2_stripe_buf_read(struct bch_fs *c, struct ec_stripe_buf *buf, u32 blocks) +{ + for (unsigned i = 0; i < buf->key.v.nr_blocks; i++) + if (blocks & BIT(i)) + bch2_ec_block_io(c, buf, REQ_OP_READ, i); + else + buf->err[STRIPE_BUF_PRE_RECOV][i] = -BCH_ERR_stripe_read_skipped; +} + void bch2_ec_block_io_range(struct bch_fs *c, struct ec_stripe_buf *buf, blk_opf_t opf, unsigned idx, unsigned sector_offset, unsigned sectors) @@ -770,249 +782,185 @@ static int get_stripe_key_trans(struct btree_trans *trans, u64 idx, #define EC_READ_AROUND_MIN_GAIN_NS (1000ULL * 1000) -static inline u64 ec_dev_read_latency(struct bch_dev *ca) +/* What a read-around has to beat: its slowest device's latency @l, scaled */ +static inline u64 ec_read_around_cost(u64 l, unsigned penalty) { - return atomic64_read(&ca->cur_latency[READ]); + return div_u64(l * penalty, 100) + EC_READ_AROUND_MIN_GAIN_NS; } -/* - * The latency a read-around has to beat: that of its slowest device, @l, scaled - * for reading k blocks instead of one, plus the least gain worth having. - */ -static inline u64 ec_read_around_cost(u64 l, unsigned penalty) +/* On a tie the highest index is the slower: Q is left out before P before data */ +static unsigned ec_slowest_block(const struct bch_stripe *v, u32 mask, const u64 *lat) { - return div_u64(l * penalty, 100) + EC_READ_AROUND_MIN_GAIN_NS; + unsigned slowest = __ffs(mask); + + for (unsigned i = slowest + 1; i < v->nr_blocks; i++) + if ((mask & BIT(i)) && lat[i] >= lat[slowest]) + slowest = i; + return slowest; } /* - * The blocks a read-around of @block reads: the k fastest of the stripe's other - * blocks. With two parity blocks there are k + 1 to choose from, and the - * slowest is left out; on a tie, the highest index, so Q before P before data - - * rebuilding a data block and Q is an XOR and a syndrome, no Galois field - * recovery. - * - * A device with no read latency sample yet doesn't qualify: what it costs is - * unknown. Returns 0 if fewer than k blocks qualify; otherwise the mask, and in - * @l_r the latency of the slowest device in it. + * The blocks of @v slower by @cost than @read, the nr_data fastest that can be + * read now (not in @skip, online, not stale, with a latency sample), which a + * read around any of them reads; @read is 0 if there aren't nr_data. */ -static u32 ec_read_around_blocks(struct bch_fs *c, const struct bch_stripe *v, - unsigned block, struct bch_io_failures *failed, - u64 *l_r) +static u32 ec_read_around_slow(struct bch_fs *c, const struct bch_stripe *v, + u32 skip, unsigned penalty, u32 *read, u64 *cost) { unsigned nr_data = v->nr_blocks - v->nr_redundant; u64 lat[BCH_BKEY_PTRS_MAX] = {}; - u32 mask = 0; + u32 usable = 0, slow = 0; scoped_guard(rcu) for (unsigned i = 0; i < v->nr_blocks; i++) { - if (i == block) - continue; - struct bch_dev *ca = bch2_dev_rcu_noerror(c, v->ptrs[i].dev); - if (!ca || + if ((skip & BIT(i)) || + !ca || !bch2_dev_is_online(ca) || - dev_ptr_stale_rcu(ca, &v->ptrs[i]) || - (failed && bch2_dev_io_failed(bch2_dev_io_failures(failed, ca->dev_idx)))) + dev_ptr_stale_rcu(ca, &v->ptrs[i])) continue; - lat[i] = ec_dev_read_latency(ca); + lat[i] = atomic64_read(&ca->cur_latency[READ]); if (lat[i]) - mask |= BIT(i); + usable |= BIT(i); } - while (hweight32(mask) > nr_data) { - unsigned slowest = __ffs(mask); - - for (unsigned i = slowest + 1; i < v->nr_blocks; i++) - if ((mask & BIT(i)) && lat[i] >= lat[slowest]) - slowest = i; - mask &= ~BIT(slowest); - } + u32 mask = usable; + while (hweight32(mask) > nr_data) + mask &= ~BIT(ec_slowest_block(v, mask, lat)); - if (hweight32(mask) < nr_data) + *read = hweight32(mask) == nr_data ? mask : 0; + if (!*read) return 0; - *l_r = 0; + *cost = ec_read_around_cost(lat[ec_slowest_block(v, mask, lat)], penalty); + for (unsigned i = 0; i < v->nr_blocks; i++) - if (mask & BIT(i)) - *l_r = max(*l_r, lat[i]); - return mask; + if ((usable & ~mask & BIT(i)) && lat[i] > *cost) + slow |= BIT(i); + return slow; } /* - * Blocks of @required on devices much slower than the rest, which a read of - * the whole stripe can rebuild from the others: the read-around rule without - * the coin toss, for stripe repair and reuse, which read in bulk. At most - * nr_redundant, less blocks already lost to offline devices, and only when - * every device read in their place has a latency sample. + * For stripe repair and reuse, reading @required of @v: the blocks to rebuild + * instead when one of those is far slower. No coin toss: background work. */ u32 bch2_ec_read_around_skip(struct bch_fs *c, const struct bch_stripe *v, u32 required) { unsigned penalty = c->opts.ec_read_around_penalty; - u64 lat[BCH_BKEY_PTRS_MAX] = {}; - u32 offline = 0, skip = 0; + u32 read; + u64 cost; - if (!penalty) + if (!penalty || + !(ec_read_around_slow(c, v, 0, penalty, &read, &cost) & required)) return 0; - scoped_guard(rcu) - for (unsigned i = 0; i < v->nr_blocks; i++) { - struct bch_dev *ca = bch2_dev_rcu_noerror(c, v->ptrs[i].dev); - - if (ca && bch2_dev_is_online(ca)) - lat[i] = ec_dev_read_latency(ca); - else - offline |= BIT(i); - } - - for (unsigned budget = v->nr_redundant - min(hweight32(offline), v->nr_redundant); - budget; - --budget) { - u32 candidates = required & ~skip & ~offline; - if (!candidates) - break; - - unsigned slowest = __ffs(candidates); - for (unsigned i = slowest + 1; i < v->nr_blocks; i++) - if ((candidates & BIT(i)) && lat[i] > lat[slowest]) - slowest = i; - - u64 l_r = 0; - for (unsigned i = 0; i < v->nr_blocks; i++) { - if ((skip | offline | BIT(slowest)) & BIT(i)) - continue; - if (!lat[i]) - return skip; - l_r = max(l_r, lat[i]); - } - - if (lat[slowest] <= ec_read_around_cost(l_r, penalty)) - break; - skip |= BIT(slowest); - } - - return skip; + return ~read & (BIT(v->nr_blocks) - 1); } /* - * Should a read of @pick, which bch2_bkey_pick_read_device() chose to read - * directly, go around its device instead? - * - * A reconstruct reads k blocks instead of one, so only when that is clearly - * faster. With d the device's read latency and r the cost of the reconstruct - * (its slowest device's latency, scaled by ec_read_around_penalty, plus 1 ms): - * never when d <= r - so never on a healthy pool - and otherwise with - * probability 1 - (r/d)^2. The slow device keeps getting some reads, so its - * latency estimate recovers when it does. - * - * Not for reads that must come from the device (scrub), or retries: a retry - * after an error has its own reconstruct path, and the retry carrying a - * read-around decision (BCH_READ_ec_read_around) doesn't roll again. - * - * Reconcile reads each device's data on that device's own thread, in LBA - * order (soft_require_read_device). Those read around it only when it's far - * slower: by 16x, the factor ptr_better() uses to keep replicated reads there - * when the other replica is non-rotational. - * - * Returns 0 or a transaction restart. + * Should a read of @pick, picked to read directly, go around its device? With d + * its latency and r the reconstruct's cost: never when d <= r, otherwise with + * probability 1 - (r/d)^2, so the slow device keeps some reads and its latency + * estimate recovers when it does. Reconcile's reads of a device on its own + * thread (soft_require_read_device) go around it only at 16x, as ptr_better() + * keeps replicated reads there. The decision is carried to the retry that does + * the read in @flags. Returns 0 or a transaction restart. */ -int bch2_ec_read_around_pick(struct btree_trans *trans, - struct extent_ptr_decoded *pick, - struct bch_io_failures *failed, - enum bch_read_flags flags, int preferred_dev) +int bch2_ec_read_around_pick(struct btree_trans *trans, struct extent_ptr_decoded *pick, + struct bch_io_failures *failed, enum bch_read_flags *flags, + int preferred_dev) { struct bch_fs *c = trans->c; unsigned penalty = c->opts.ec_read_around_penalty; - bool decided = flags & BCH_READ_ec_read_around; if (!penalty || - (flags & BCH_READ_hard_require_read_device) || - ((flags & BCH_READ_in_retry) && !decided)) + pick->mode != BCH_READ_MODE_direct || + (*flags & BCH_READ_hard_require_read_device)) return 0; - if ((flags & BCH_READ_soft_require_read_device) && + /* Only the retry carrying a read-around decision reads around, and once */ + if (*flags & BCH_READ_in_retry) { + if ((*flags & BCH_READ_ec_read_around) && + !bch2_read_mode_tried(bch2_dev_io_failures(failed, pick->ptr.dev), + BCH_READ_MODE_ec_read_around)) + pick->mode = BCH_READ_MODE_ec_read_around; + return 0; + } + + if ((*flags & BCH_READ_soft_require_read_device) && pick->ptr.dev == preferred_dev) penalty *= 16; - if (failed && - bch2_read_mode_tried(bch2_dev_io_failures(failed, pick->ptr.dev), - BCH_READ_MODE_ec)) - return 0; - - /* - * Finding the stripe's devices takes a stripes btree lookup, so first - * check against every online device: r can't be lower than the fastest. - */ + /* Before the stripes btree lookup: r can't be lower than the fastest device */ u64 l_d = 0, l_min = U64_MAX; scoped_guard(rcu) { struct bch_dev *ca = bch2_dev_rcu_noerror(c, pick->ptr.dev); if (!ca) return 0; - l_d = ec_dev_read_latency(ca); + l_d = atomic64_read(&ca->cur_latency[READ]); if (l_d <= EC_READ_AROUND_MIN_GAIN_NS) return 0; for_each_online_member_rcu(c, peer) { - u64 l = ec_dev_read_latency(peer); + u64 l = atomic64_read(&peer->cur_latency[READ]); if (peer != ca && l) l_min = min(l_min, l); } } - if (l_min == U64_MAX || - l_d <= ec_read_around_cost(l_min, penalty)) + if (l_min == U64_MAX || l_d <= ec_read_around_cost(l_min, penalty)) return 0; - if (!decided) { - /* Data updates get here unlocked, by bch2_data_update_init(): */ - try(bch2_trans_relock(trans)); + /* Data updates get here unlocked, by bch2_data_update_init(): */ + try(bch2_trans_relock(trans)); - CLASS(btree_iter, iter)(trans, BTREE_ID_stripes, POS(0, pick->ec.idx), BTREE_ITER_slots); - struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter); - int ret = bkey_err(k); - if (ret) - return bch2_err_matches(ret, BCH_ERR_transaction_restart) ? ret : 0; + CLASS(btree_iter, iter)(trans, BTREE_ID_stripes, POS(0, pick->ec.idx), BTREE_ITER_slots); + struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter); + int ret = bkey_err(k); + if (ret) + return bch2_err_matches(ret, BCH_ERR_transaction_restart) ? ret : 0; - if (k.k->type != KEY_TYPE_stripe) - return 0; + if (k.k->type != KEY_TYPE_stripe) + return 0; - const struct bch_stripe *v = bkey_s_c_to_stripe(k).v; - if (!bch2_ptr_matches_stripe(v, *pick)) - return 0; + const struct bch_stripe *v = bkey_s_c_to_stripe(k).v; + if (!bch2_ptr_matches_stripe(v, *pick)) + return 0; - u64 l_r; - if (!ec_read_around_blocks(c, v, pick->ec.block, failed, &l_r)) - return 0; + u32 read; + u64 r; + if (!(ec_read_around_slow(c, v, 0, penalty, &read, &r) & BIT(pick->ec.block))) + return 0; - /* - * Only if its stripe buffers fit under the limit now - not - * worth going over it for. Racy: a read-around that loses the - * race goes over a little. - */ - long bytes = ((long) pick->crc.compressed_size << 9) * v->nr_blocks; - long used = READ_ONCE(c->ec.stripe_buf_bytes); - if (used && used + bytes > bch2_ec_stripe_buf_limit(c)) - return 0; + /* + * Only if its stripe buffers fit under the limit now - not worth going + * over it for. Racy: a read-around that loses the race goes over a + * little. + */ + long bytes = ((long) pick->crc.compressed_size << 9) * v->nr_blocks; + long used = READ_ONCE(c->ec.stripe_buf_bytes); + if (used && used + bytes > bch2_ec_stripe_buf_limit(c)) + return 0; - /* In ~us, so the squares fit: */ - u64 d = min_t(u64, l_d >> 10, U32_MAX); - u64 r = min_t(u64, ec_read_around_cost(l_r, penalty) >> 10, U32_MAX); + /* In ~us, so the squares fit: */ + u64 d = min_t(u64, l_d >> 10, U32_MAX); + r = min_t(u64, r >> 10, U32_MAX); - if (d <= r || - bch2_get_random_u64_below(d * d) < r * r) - return 0; - } + if (d <= r || bch2_get_random_u64_below(d * d) < r * r) + return 0; - pick->mode = BCH_READ_MODE_ec; + event_inc_trace(c, data_read_ec_read_around, buf, bch2_bkey_val_to_text(&buf, c, k)); + pick->mode = BCH_READ_MODE_ec_read_around; + *flags |= BCH_READ_ec_read_around; return 0; } int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio, struct bkey_s_c orig_k, - struct bch_io_failures *failed) + struct printbuf *msg) { - struct printbuf *msg = &failed->ec_msg; /* * We need the original extent to read to still be locked when we check * for non-spurious stale stripe pointers @@ -1080,41 +1028,29 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio, bch2_run_explicit_recovery_pass(c, &msg.m, BCH_RECOVERY_PASS_check_allocations, 0); } + /* + * The block is rebuilt without reading it. A read-around reads the k + * fastest, and only the extent's range of each when the extent's + * checksum checks the result. + */ unsigned block = rbio->pick.ec.block; - bool skipped_failed_block = false; - u32 read_mask = EC_BLOCKS_ALL; - enum ec_stripe_buf_flags buf_flags = 0; - if (bch2_ec_read_optional(failed, &rbio->pick, rbio->flags)) { - u64 l_r; - read_mask = ec_read_around_blocks(c, &buf->key.v, block, failed, &l_r); + bool read_around = rbio->pick.mode == BCH_READ_MODE_ec_read_around; + u32 read_mask = EC_BLOCKS_ALL & ~BIT(block); + if (read_around) { + u64 cost; + ec_read_around_slow(c, &buf->key.v, BIT(block), + c->opts.ec_read_around_penalty, &read_mask, &cost); /* Raced with a device going offline: not worth a message */ if (!read_mask) return bch_err_throw(c, stripe_reconstruct_insufficient_blocks); - /* - * When the extent is checksummed, its checksum checks the - * result, as it would a direct read, so read just the extent's - * range of each block, without rounding to checksum granules. - */ - if (rbio->pick.crc.csum_type) - buf_flags |= EC_STRIPE_BUF_unaligned; - } else if (bch2_read_mode_tried(bch2_dev_io_failures(failed, rbio->pick.ptr.dev), - BCH_READ_MODE_direct)) { - /* - * The block just failed to read, so rebuild it from the - * others: on a drive retrying a bad sector, reading it again - * costs as long again. If the others can't rebuild it, it's - * read after all, below. - */ - read_mask &= ~BIT(block); - skipped_failed_block = true; + buf->unaligned = rbio->pick.crc.csum_type != 0; } /* Don't hold btree locks for stripe buffer allocations, or IO */ bch2_trans_unlock(trans); - ret = __bch2_ec_stripe_buf_init(c, buf, offset, bio_sectors(&rbio->bio), NULL, - buf_flags); + ret = bch2_ec_stripe_buf_init(c, buf, offset, bio_sectors(&rbio->bio), NULL); if (ret) { prt_printf(msg, "error allocating stripe data buffers\n"); bch2_bkey_val_to_text(msg, c, bkey_i_to_s_c(&buf->key.k_i)); @@ -1122,27 +1058,16 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio, return bch_err_throw(c, stripe_reconstruct_enomem); } - /* Blocks not read are erasures, rebuilt with the target: */ - for (unsigned i = 0; i < buf->key.v.nr_blocks; i++) - if (read_mask & BIT(i)) - bch2_ec_block_io(c, buf, REQ_OP_READ, i); - else - buf->err[STRIPE_BUF_PRE_RECOV][i] = -BCH_ERR_stripe_read_skipped; - + bch2_stripe_buf_read(c, buf, read_mask); ret = bch2_stripe_buf_validate(c, buf, false, EC_BLOCKS_ALL); /* - * The rest of the stripe couldn't rebuild the block we failed to read: - * another block is bad too. Our block's error may have been transient, - * so read it after all before the data is given up on. + * Read it after all before giving up: its error may have been transient. + * A read-around falls back to the device instead. */ - if (ret && - ret != -BCH_ERR_stripe_reconstruct_stale_race && - skipped_failed_block) { - memset(buf->err[STRIPE_BUF_POST_RECOV], 0, sizeof(buf->err[STRIPE_BUF_POST_RECOV])); - buf->err[STRIPE_BUF_PRE_RECOV][block] = 0; + if (ret && !read_around && + ret != -BCH_ERR_stripe_reconstruct_stale_race) { bch2_ec_block_io(c, buf, REQ_OP_READ, block); - ret = bch2_stripe_buf_validate(c, buf, false, EC_BLOCKS_ALL); } @@ -1155,6 +1080,14 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio, ? bch_err_throw(c, data_read_ptr_stale_retry) : bch_err_throw(c, data_read_ptr_stale_race); + /* Blocks that don't match the stripe's checksums stay bad until scrubbed: */ + u32 bad = 0; + for (unsigned i = 0; i < buf->key.v.nr_blocks; i++) + if (buf->err[STRIPE_BUF_PRE_RECOV][i] == -BCH_ERR_stripe_read_csum_err) + bad |= BIT(i); + if (bad) + bch2_ec_stripe_scrub_queue(c, rbio->pick.ec.idx, bad); + if (!ret) memcpy_to_bio(&rbio->bio, rbio->bio.bi_iter, buf->data[block] + ((offset - buf->offset) << 9)); diff --git a/fs/data/ec/io.h b/fs/data/ec/io.h index 306fb3ca27..092d84618e 100644 --- a/fs/data/ec/io.h +++ b/fs/data/ec/io.h @@ -26,7 +26,7 @@ struct ec_stripe_buf { /* might not be buffering the entire stripe: */ unsigned offset; unsigned size; - /* Not aligned to the checksum granularity, so the checksums aren't checked */ + /* Set before init: exactly the range given, not checked by the stripe's checksums */ bool unaligned; s16 err[2][BCH_BKEY_PTRS_MAX]; void *data[BCH_BKEY_PTRS_MAX]; @@ -84,20 +84,8 @@ static inline unsigned long bch2_ec_stripe_buf_limit(struct bch_fs *c) return (totalram_pages() << PAGE_SHIFT) / 100 * c->opts.ec_stripe_buf_limit; } -enum ec_stripe_buf_flags { - /* Exactly the range given, without rounding to the checksum granularity */ - EC_STRIPE_BUF_unaligned = BIT(0), -}; - -int __bch2_ec_stripe_buf_init(struct bch_fs *, struct ec_stripe_buf *, unsigned, unsigned, - struct closure *, enum ec_stripe_buf_flags); - -static inline int bch2_ec_stripe_buf_init(struct bch_fs *c, struct ec_stripe_buf *buf, - unsigned offset, unsigned size, - struct closure *cl) -{ - return __bch2_ec_stripe_buf_init(c, buf, offset, size, cl, 0); -} +int bch2_ec_stripe_buf_init(struct bch_fs *, struct ec_stripe_buf *, unsigned, unsigned, + struct closure *); DEFINE_FREE(ec_stripe_buf_free, struct ec_stripe_buf *, if (_T) { bch2_ec_stripe_buf_exit(_T); kfree(_T); }); @@ -111,14 +99,15 @@ int bch2_stripe_buf_validate_msg(struct bch_fs *, struct ec_stripe_buf *, bool, void bch2_ec_block_io(struct bch_fs *, struct ec_stripe_buf *, blk_opf_t, unsigned); void bch2_ec_block_io_range(struct bch_fs *, struct ec_stripe_buf *, blk_opf_t, unsigned, unsigned, unsigned); +void bch2_stripe_buf_read(struct bch_fs *, struct ec_stripe_buf *, u32); u32 bch2_ec_read_around_skip(struct bch_fs *, const struct bch_stripe *, u32); int bch2_ec_read_around_pick(struct btree_trans *, struct extent_ptr_decoded *, - struct bch_io_failures *, enum bch_read_flags, int); + struct bch_io_failures *, enum bch_read_flags *, int); struct bch_read_bio; int bch2_ec_read_extent(struct btree_trans *, struct bch_read_bio *, - struct bkey_s_c, struct bch_io_failures *); + struct bkey_s_c, struct printbuf *); #endif /* _BCACHEFS_DATA_EC_IO_H */ diff --git a/fs/data/ec/types.h b/fs/data/ec/types.h index ead241382d..8893774c3f 100644 --- a/fs/data/ec/types.h +++ b/fs/data/ec/types.h @@ -57,6 +57,16 @@ struct bch_fs_ec { struct work_struct stripe_delete_work; + /* Stripes a read found a bad block in, for bch2_ec_stripe_scrub_work(): */ + spinlock_t stripe_scrub_lock; + struct ec_stripe_scrub { + u64 idx; + u32 blocks; /* 0 for a free slot */ + bool running; + unsigned long done_at; /* jiffies, 0 until scrubbed */ + } stripe_scrub[16]; + struct work_struct stripe_scrub_work; + struct bio_set block_bioset; GENRADIX(struct gc_stripe) gc_stripes; diff --git a/fs/data/extents.c b/fs/data/extents.c index 29f023d5b5..6818ef7c8a 100644 --- a/fs/data/extents.c +++ b/fs/data/extents.c @@ -139,8 +139,7 @@ void bch2_mark_io_failure(struct bch_io_failures *failed, struct bch_dev_io_failures *f = bch2_dev_io_failures_mut(failed, p->ptr.dev); f->tried |= BIT(p->mode); - - if (p->mode == BCH_READ_MODE_ec) + if (p->mode != BCH_READ_MODE_direct) f->ec_errcode = err; else if (err == -BCH_ERR_data_read_retry_csum_err) f->csum_nr++; @@ -274,11 +273,9 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, struct bch_dev_io_failures *f = unlikely(failed) ? bch2_dev_io_failures(failed, p.ptr.dev) : NULL; - - p.has_ec &= !bch2_read_mode_tried(f, BCH_READ_MODE_ec); - if (unlikely(f)) { p.crc_retry_nr = f->csum_nr; + p.has_ec &= !bch2_read_mode_tried(f, BCH_READ_MODE_ec); if (ca) { have_io_errors |= f->errcode != 0; diff --git a/fs/data/extents.h b/fs/data/extents.h index ac587bbe12..700099a4b1 100644 --- a/fs/data/extents.h +++ b/fs/data/extents.h @@ -436,20 +436,6 @@ static inline bool bch2_read_mode_tried(const struct bch_dev_io_failures *f, return f && (f->tried & BIT(mode)); } -/* - * An ec read tried first, with a direct read still to come: it may give up, - * and if it fails the device hasn't failed. - */ -static inline bool bch2_ec_read_optional(struct bch_io_failures *failed, - const struct extent_ptr_decoded *p, - enum bch_read_flags flags) -{ - return p->mode == BCH_READ_MODE_ec && - (flags & BCH_READ_ec_read_around) && - !bch2_read_mode_tried(failed ? bch2_dev_io_failures(failed, p->ptr.dev) : NULL, - BCH_READ_MODE_direct); -} - void bch2_mark_io_failure(struct bch_io_failures *, struct extent_ptr_decoded *, int); void bch2_mark_dev_io_failure(struct bch_io_failures *, unsigned, int); int bch2_bkey_pick_read_device(struct bch_fs *, struct bkey_s_c, diff --git a/fs/data/extents_types.h b/fs/data/extents_types.h index 59c2e66ed5..386f066f9d 100644 --- a/fs/data/extents_types.h +++ b/fs/data/extents_types.h @@ -20,13 +20,13 @@ struct bch_extent_crc_unpacked { }; /* - * Read the device, or rebuild from the rest of its stripe. Each is tried at - * most once per device, in an order that depends on the read - ec first for a - * read-around - so what's been tried is tracked: bch_dev_io_failures.tried + * How a pointer is read: each is tried at most once per device, in an order + * that depends on the read, so what's been tried is tracked. */ #define BCH_READ_MODES() \ x(direct) \ - x(ec) + x(ec) \ + x(ec_read_around) enum bch_read_mode { #define x(n) BCH_READ_MODE_##n, diff --git a/fs/data/read.c b/fs/data/read.c index 57f089d9c6..fe13fca34b 100644 --- a/fs/data/read.c +++ b/fs/data/read.c @@ -33,6 +33,7 @@ #include "data/checksum.h" #include "data/compress.h" +#include "data/ec/create.h" #include "data/ec/io.h" #include "data/io_misc.h" #include "data/read.h" @@ -158,11 +159,7 @@ static bool bch2_target_congested(struct bch_fs *c, u16 target) static inline bool have_io_error(struct bch_io_failures *failed) { - if (failed) - darray_for_each(*failed, f) - if (bch2_dev_io_failed(f)) - return true; - return false; + return failed && darray_find_p(*failed, f, bch2_dev_io_failed(f)); } static inline struct data_update *rbio_data_update(struct bch_read_bio *rbio) @@ -280,7 +277,7 @@ static struct bch_read_bio *__promote_alloc(struct btree_trans *trans, { struct bch_fs *c = trans->c; - int ret = !failed + int ret = !have_io_error(failed) ? should_promote(c, k, orig->opts, flags) : 0; if (ret) @@ -376,11 +373,8 @@ static struct bch_read_bio *promote_alloc(struct btree_trans *trans, bool self_healing = have_io_error(failed); - /* - * We're in the retry path, but we don't know what to repair yet. Only - * the read-around retry may promote: it isn't an error retry. - */ - if (failed && !self_healing && !(flags & BCH_READ_may_promote)) + /* A reconstruct is punted to the retry, which does the promote: */ + if (pick->mode != BCH_READ_MODE_direct && !(flags & BCH_READ_in_retry)) return NULL; /* @@ -411,8 +405,7 @@ static struct bch_read_bio *promote_alloc(struct btree_trans *trans, k.k->type == KEY_TYPE_reflink_v ? BTREE_ID_reflink : BTREE_ID_extents, - k, pos, pick, flags, sectors, orig, - self_healing ? failed : NULL); + k, pos, pick, flags, sectors, orig, failed); int ret = PTR_ERR_OR_ZERO(promote); if (unlikely(ret)) { /* @@ -745,20 +738,12 @@ static void bch2_rbio_retry(struct work_struct *work) .inum = rbio->read_pos.inode, }; struct bpos read_pos = rbio->read_pos; - int orig_ret = rbio->ret; - /* Not an error: the read-around decision, punted here to do the reconstruct */ - bool read_around = !orig_ret && (flags & BCH_READ_ec_read_around); CLASS(bch_io_failures, failed)(); flags &= ~BCH_READ_hard_require_read_device; - if (read_around) { - event_inc_trace(c, data_read_ec_read_around, buf, - bch2_read_bio_to_text_atomic(&buf, rbio)); - } else { - event_inc_trace(c, data_read_retry, buf, - bch2_read_bio_to_text_atomic(&buf, rbio)); - } + event_inc_trace(c, data_read_retry, buf, + bch2_read_bio_to_text_atomic(&buf, rbio)); { CLASS(btree_trans, trans)(c); @@ -785,7 +770,8 @@ static void bch2_rbio_retry(struct work_struct *work) rbio = bch2_rbio_free(rbio); flags |= BCH_READ_in_retry; - if (!read_around) + /* A read-around is punted here without an error, and may promote */ + if (!(flags & BCH_READ_ec_read_around)) flags &= ~BCH_READ_may_promote; flags &= ~BCH_READ_last_fragment; flags |= BCH_READ_must_clone; @@ -802,33 +788,7 @@ static void bch2_rbio_retry(struct work_struct *work) bch2_err_matches(ret, BCH_ERR_data_read_ptr_stale_race))) ret = 0; - /* - * The read-around failed and the direct read after it didn't: - * nothing was lost, and the device is fine. Say what was found, - * unless it was only a race. - */ - bool read_around_failed = !ret && !have_io_error(&failed) && - (flags & BCH_READ_ec_read_around) && failed.nr; - - if (read_around_failed) { - event_inc_trace(c, data_read_ec_read_around_fail, buf, - bch2_read_bio_to_text_atomic(&buf, rbio)); - - if (failed.ec_msg.pos) { - CLASS(bch_log_msg_level, msg)(c, LOGLEVEL_notice); - bch2_read_err_msg_trans(trans, &msg.m, rbio, read_pos); - prt_newline(&msg.m); - - if (!bkey_deleted(&sk.k->k)) { - bch2_bkey_val_to_text(&msg.m, c, bkey_i_to_s_c(sk.k)); - prt_newline(&msg.m); - } - - bch2_io_failures_to_text(&msg.m, c, &failed); - prt_str(&msg.m, "read device directly"); - msg.m.suppress = bch2_ratelimit(c); - } - } else if (have_io_error(&failed) || failed.ec_msg.pos || ret) { + if (have_io_error(&failed) || failed.ec_msg.pos || ret) { struct printbuf *out; CLASS(bch_log_msg, msg)(c); @@ -872,12 +832,8 @@ static void bch2_rbio_retry(struct work_struct *work) * (a data update read may be an indirect extent): */ if (have_io_error(&failed) || ret) { - /* After a read-around, orig_ret is 0: the failures say what was recovered from */ - bool csum = data_read_err_is_csum(orig_ret) || - (read_around && - (bch2_io_failures_to_err_mask(&failed) & BCH_READ_ERR_checksum)); enum bch_sb_error_id e = !ret - ? (csum + ? (bch2_io_failures_to_err_mask(&failed) & BCH_READ_ERR_checksum ? BCH_FSCK_ERR_data_read_csum_err_recovered : BCH_FSCK_ERR_data_read_io_err_recovered) : (data_read_err_is_csum(ret) @@ -931,7 +887,15 @@ static int bch2_rbio_error(struct bch_read_bio *rbio, int ret) rbio->ret = ret; /* A failed read-around isn't an error: the read goes to the device next */ - if (!bch2_ec_read_optional(rbio->failed, &rbio->pick, rbio->flags)) + if (rbio->pick.mode == BCH_READ_MODE_ec_read_around) { + event_inc_trace(rbio->c, data_read_ec_read_around_fail, buf, + bch2_read_bio_to_text_atomic(&buf, rbio)); + + /* Its data failed the extent checksum: a block it read is bad, and nothing says which */ + if (data_read_err_is_csum(ret)) + bch2_ec_stripe_scrub_queue(rbio->c, rbio->pick.ec.idx, + ~BIT(rbio->pick.ec.block)); + } else bch2_rbio_parent(rbio)->saw_error = true; if (!(rbio->flags & BCH_READ_in_retry)) { @@ -1628,24 +1592,8 @@ int __bch2_read_extent(struct btree_trans *trans, return read_extent_pick_err(trans, orig, read_pos, data_btree, k, flags, ret); ret = 0; - /* - * A read-around is punted to the retry, which does the promote: the - * rbio allocated here is freed unused. - */ - enum bch_read_flags punt_promote = 0; - - if (pick.has_ec && pick.mode == BCH_READ_MODE_direct) { - try(bch2_ec_read_around_pick(trans, &pick, failed, flags, dev)); - - if (pick.mode == BCH_READ_MODE_ec) { - flags |= BCH_READ_ec_read_around; - - if (!(flags & BCH_READ_in_retry)) { - punt_promote = flags & BCH_READ_may_promote; - flags &= ~BCH_READ_may_promote; - } - } - } + if (pick.has_ec) + try(bch2_ec_read_around_pick(trans, &pick, failed, &flags, dev)); if (bch2_csum_type_is_encryption(pick.crc.csum_type) && unlikely(!c->chacha20_key_set)) @@ -1768,20 +1716,19 @@ int __bch2_read_extent(struct btree_trans *trans, trans->notrace_relock_fail = true; } else { if (!(flags & BCH_READ_in_retry)) { - rbio->flags |= punt_promote; bch2_rbio_punt(rbio, bch2_rbio_retry, RBIO_CONTEXT_UNBOUND, system_dfl_wq); return 0; } /* Attempting reconstruct read: */ - ret = bch2_ec_read_extent(trans, rbio, k, failed); + ret = bch2_ec_read_extent(trans, rbio, k, &failed->ec_msg); if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) { bch2_rbio_free(rbio); return ret; } if (ret) { /* Whatever went wrong, a read-around falls back to a direct read: */ - if (bch2_ec_read_optional(failed, &rbio->pick, rbio->flags)) + if (rbio->pick.mode == BCH_READ_MODE_ec_read_around) ret = bch_err_throw(c, data_read_retry_ec_read_around); bch2_rbio_error(rbio, ret); goto out; diff --git a/fs/opts.h b/fs/opts.h index fadb85d1f8..46ef595e22 100644 --- a/fs/opts.h +++ b/fs/opts.h @@ -655,10 +655,9 @@ enum fsck_err_opts { OPT_FS|OPT_MOUNT|OPT_RUNTIME, \ OPT_UINT(0, 10000), \ BCH2_NO_SB_OPT, 150, \ - "%", "Reconstruct erasure coded reads from the rest of\n"\ - " the stripe when the device holding the data is slower\n"\ - " than this percentage of the reconstruct's latency (its\n"\ - " slowest device's) plus 1ms; 0 disables") \ + "%", "Reconstruct erasure coded reads around a device\n"\ + " slower than this percentage of the reconstruct's latency\n"\ + " plus 1ms; 0 disables") \ x(ec_stripe_buf_limit, u8, \ OPT_FS|OPT_MOUNT|OPT_RUNTIME, \ OPT_UINT(1, 25), \