diff --git a/tests/fs/bcachefs/ec-read-around.ktest b/tests/fs/bcachefs/ec-read-around.ktest new file mode 100644 index 00000000..2f34bd64 --- /dev/null +++ b/tests/fs/bcachefs/ec-read-around.ktest @@ -0,0 +1,1137 @@ +#!/usr/bin/env bash +# +# Reads of erasure-coded data, and a device slower than the rest of its stripe. +# +# test_ec_read_slow_device: one member behind dm-delay; once its latency is +# known, reads of its data are reconstructed from the rest of the stripe. +# The tests with a delayed member warm its latency estimate first; the two +# *_learns_failing_device tests start from a cold one. +# test_ec_read_healthy: no slow member, one reader and then random readers in +# parallel: no read-arounds. +# test_ec_read_around_disabled: ec_read_around_penalty=0: reads wait for the +# slow member, as before. +# test_ec_read_around_pq: 4+2 stripes, one member slow and another slower: a +# read-around of either leaves out the other. +# test_ec_read_around_fallback: read-arounds that find a bad parity block read +# the member instead; nothing is recorded against the file. +# test_ec_read_around_fallback_reconstruct: the read-around and then the direct +# read fail; the full reconstruct still recovers the data. +# test_ec_read_around_small_extents: read-arounds of 4k extents read just 4k +# from each block. +# test_ec_read_around_nocsum: data_checksum=none: read-arounds still work, +# checked against the stripe's own checksums. +# test_ec_repair_reads_around_slow_device: evacuating a member that is much +# slower than the rest: stripe repair rebuilds its blocks from the others, +# reading almost nothing from it. +# test_ec_reconcile_reads_around_slow_device: reconcile rewriting every extent +# (background_compression) reads around a member that is far slower. +# test_ec_reconcile_learns_failing_device: the same, from a cold estimate, +# with a member that takes 7 s per read: reconcile's own reads of it bring +# the estimate up. +# test_ec_evacuate_learns_failing_device: evacuating such a member, from a +# cold estimate: stripe repair's reads of it bring the estimate up. +# test_ec_reconstruct_skips_failed_block: after a read error, the reconstruct +# doesn't read the failed block again. +# test_ec_read_around_promotes: with a promote target, read-arounds promote as +# direct reads do, so a second read of the file comes from the promote target. +# test_ec_read_around_bad_block_no_slow_wait: 4+2 stripes whose slow member's +# block is good and a parity block is bad: the read-around fails, and reading +# the one block it left out is enough to rebuild around the bad one, so a read +# whose read-around fails doesn't wait on the slow member. +# test_ec_repair_bad_block_no_slow_read: evacuating the slow member of 4+2 +# stripes with a bad parity block: repair reads the one other block it left +# out and rebuilds around the bad one, reading nothing from the slow member. + +# shellcheck source=tests/fs/bcachefs/bcachefs-test-libs.sh +. "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/bcachefs-test-libs.sh" + +require-kernel-config BCACHEFS_ERASURE_CODING +require-kernel-config DM_DELAY + +config-scratch-devs 1G +config-scratch-devs 1G +config-scratch-devs 1G +config-scratch-devs 1G +config-scratch-devs 1G +config-scratch-devs 1G + +ec_ra_bucket_kb=512 +ec_ra_file_mb=96 +# Btree and journal reads may still hit the device; stripe data must not. +ec_ra_max_dev_read_kb=4096 +# Tests warm a delayed member's latency estimate first (ec_ra_prewarm). Even +# then, a read of its data goes to it with probability (r/d)^2: rarely. +ec_ra_coin_reads=1 +ec_ra_out=/tmp/ec_ra.out + +ec_ra_sectors_read() +{ + awk '{ print $3 }' "/sys/block/$(basename "$(readlink -f "$1")")/stat" +} + +# Each member's read latency estimate, as bcachefs sees it +ec_ra_latencies() +{ + local f + for f in /sys/fs/bcachefs/"$(ec_ra_uuid)"/dev-*/io_latency_read; do + echo " $(basename "$(dirname "$f")") read latency: $(cat "$f")" + done +} + +ec_ra_uuid() +{ + local entry + for entry in /sys/fs/bcachefs/*; do + [[ ${entry##*/} =~ ^[0-9a-f-]{36}$ ]] && echo "${entry##*/}" && return + done +} + +# 0 for a counter this version doesn't have, so the tests run on it too +ec_ra_counter() +{ + local f + f=/sys/fs/bcachefs/$(ec_ra_uuid)/counters/$1 + if [[ -e $f ]]; then + awk '/since mount/ { print $NF }' "$f" + else + echo 0 + fi +} + +ec_ra_stripe_keys() +{ + cat "/sys/kernel/debug/bcachefs/$(ec_ra_uuid)/btrees/stripes/keys" +} + +# Stripes with live data and a bucket on member , from the mounted +# filesystem. Empty stripes don't count: nothing reads them. Block lines print +# as " :: gen #". +ec_ra_stripes_on() +{ + ec_ra_stripe_keys | awk -v dev="$1" ' + function flush() { if (on && live) n++; on = live = 0 } + /type stripe/ { flush() } + { + for (i = 1; i < NF; i++) + if ($i ~ /^[0-9]+:[0-9]+:[0-9]+$/ && $(i + 1) == "gen") { + split($i, p, ":") + split($(i + 2), c, "#") + on += p[1] == dev + live += c[2] > 0 + } + } + END { flush(); print n + 0 }' +} + +# Stripes with a live data block on member , one per line: +# " : ..." for each block, data blocks first. +ec_ra_stripes_with_data_on() +{ + ec_ra_stripe_keys | awk -v dev="$1" -v bsz=$((ec_ra_bucket_kb * 2)) ' + function flush() { if (live) print nd line; live = 0; line = ""; idx = 0 } + /type stripe/ { flush() } + match($0, /blocks [0-9]+:[0-9]+/) { + split(substr($0, RSTART + 7, RLENGTH - 7), b, ":") + nd = b[1] + } + { + for (i = 1; i < NF; i++) + if ($i ~ /^[0-9]+:[0-9]+:[0-9]+$/ && $(i + 1) == "gen") { + split($i, p, ":") + split($(i + 2), c, "#") + if (p[1] == dev && idx < nd && c[2] > 0) + live = 1 + line = line " " p[1] ":" p[2] * bsz + p[3] + idx++ + } + } + END { flush() }' +} + +ec_ra_write_file() +{ + dd if=/dev/urandom of=/mnt/data bs=1M count=$ec_ra_file_mb conv=fsync status=none +} + +# 4k written every 8k: extents that can't merge +ec_ra_write_small_extents() +{ + local cmds=() i + for (( i = 0; i < ec_ra_file_mb * 128; i++ )); do + cmds+=(-c "pwrite -q -S $(( i & 255 )) $(( i * 8192 )) 4k") + done + xfs_io -f "${cmds[@]}" -c fsync /mnt/data +} + +# ec_ra_setup : stripes of ec_ra_data_blocks + (ec_ra_replicas - 1) +# blocks (default 3+1), formatted with ec_ra_format_opts; one file of +# ec_ra_file_mb written by ec_ra_write (default ec_ra_write_file), stripes +# built. Leaves /mnt mounted and sets ec_ra_md5. +ec_ra_setup() +{ + # ec_ra_labels, if set, holds each device's label, in order + local fmt=() i + for (( i = 1; i <= $#; i++ )); do + [[ -n ${ec_ra_labels[i - 1]:-} ]] && fmt+=(--label="${ec_ra_labels[i - 1]}") + fmt+=("${!i}") + done + # shellcheck disable=SC2086 + run_quiet "" bcachefs format -f \ + --erasure_code \ + --replicas="${ec_ra_replicas:-2}" \ + --ec_max_data_blocks="${ec_ra_data_blocks:-3}" \ + --bucket_size=${ec_ra_bucket_kb}k \ + ${ec_ra_format_opts:-} \ + "${fmt[@]}" + mount -t bcachefs "$(join_by : "$@")" /mnt + + "${ec_ra_write:-ec_ra_write_file}" + ec_ra_md5=$(md5sum < /mnt/data) + # Stripes are built by reconcile after the write: + timeout 120 bcachefs reconcile wait /mnt || true + + local stripes + stripes=$(ec_ra_stripes_on 0) + echo "stripes with a bucket on device 0: $stripes" + if (( stripes == 0 )); then + echo "ERROR: no stripes on device 0 - nothing to read around, this run tests nothing" + exit 1 + fi +} + +# ec_ra_reads -- : print per-device KiB read since the +# counts in , and set ec_ra_dev_kb[]. +ec_ra_reads() +{ + local before=() i=0 dev kb + while [[ $1 != -- ]]; do before+=("$1"); shift; done + shift + ec_ra_dev_kb=() + for dev in "$@"; do + kb=$(( ($(ec_ra_sectors_read "$dev") - before[i]) / 2 )) + echo " device $i ($(basename "$dev")): read $kb KiB" + ec_ra_dev_kb[i]=$kb + i=$((i + 1)) + done +} + +ec_ra_snapshot() +{ + local dev + for dev in "$@"; do + ec_ra_sectors_read "$dev" + done +} + +ec_ra_dm_table() +{ + local name=$1 table=$2 + dmsetup reload "$name" --table "$table" + dmsetup suspend --nolockfs "$name" + dmsetup resume "$name" +} + +ec_ra_dm_create() +{ + dmsetup create "$1" --table "0 $(blockdev --getsz "$2") linear $2 0" +} + +# ec_ra_delay : delay reads of dm device , over , +# by ; 0 for no delay. +ec_ra_delay() +{ + local sz table + sz=$(blockdev --getsz "$2") + table="0 $sz linear $2 0" + (( $3 )) && table="0 $sz delay $2 0 $3 $2 0 0 $2 0 0" + ec_ra_dm_table "$1" "$table" +} + +# ec_ra_read_loop: the file in 1M O_DIRECT reads, into $ec_ra_out; sets +# ec_ra_ms[] to how long each took. +ec_ra_read_loop() +{ + local i start + ec_ra_ms=() + : > $ec_ra_out + for (( i = 0; i < ec_ra_file_mb; i++ )); do + start=$(date +%s%N) + dd if=/mnt/data bs=1M skip=$i count=1 iflag=direct status=none >> $ec_ra_out + ec_ra_ms[i]=$(( ($(date +%s%N) - start) / 1000000 )) + done +} + +# ec_ra_over : how many reads of the last ec_ra_read_loop took over +ec_ra_over() +{ + local ms n=0 + for ms in "${ec_ra_ms[@]}"; do + (( ms > $1 )) && n=$((n + 1)) + done + echo $n +} + +ec_ra_max_ms() +{ + printf '%s\n' "${ec_ra_ms[@]}" | sort -n | tail -n1 +} + +ec_ra_check_read() +{ + if [[ $(md5sum < $ec_ra_out) != "$ec_ra_md5" ]]; then + echo "ERROR: O_DIRECT reads returned the wrong data" + exit 1 + fi + rm -f $ec_ra_out +} + +ec_ra_check_data() +{ + echo 3 > /proc/sys/vm/drop_caches + if [[ $(md5sum < /mnt/data) != "$ec_ra_md5" ]]; then + echo "ERROR: data changed" + exit 1 + fi +} + +# ec_ra_corrupt_block :: the stripe bucket at of member +ec_ra_corrupt_block() +{ + dd if=/dev/urandom of="${devs[${1%%:*}]}" bs=512 seek="${1#*:}" \ + count=$((ec_ra_bucket_kb * 2)) oflag=direct conv=notrunc status=none +} + +# ec_ra_slow_setup [ ]: members 0 and 1 over +# dm devices ecslow0 and ecslow1, for delaying; the filesystem made and read +# once while every member is fast, so the btree is cached and every member has +# a read latency sample. Sets devs[]. +ec_ra_slow_setup() +{ + local nr=$1 + modprobe dm-delay + ec_ra_dm_create ecslow0 "${ktest_scratch_dev[0]}" + ec_ra_dm_create ecslow1 "${ktest_scratch_dev[1]}" + trap 'umount /mnt 2>/dev/null; dmsetup remove --retry ecslow0 >/dev/null 2>&1; dmsetup remove --retry ecslow1 >/dev/null 2>&1; kill_descendants $BASHPID' EXIT + + devs=(/dev/mapper/ecslow0 /dev/mapper/ecslow1 "${ktest_scratch_dev[@]:2:$((nr - 2))}") + ec_ra_data_blocks=${2:-3} ec_ra_replicas=${3:-2} ec_ra_setup "${devs[@]}" + + # O_DIRECT: the file is still in the page cache from writing it + dd if=/mnt/data of=/dev/null bs=1M iflag=direct status=none +} + +# ec_ra_delay_dev +ec_ra_delay_dev() +{ + ec_ra_delay "ecslow$1" "${ktest_scratch_dev[$1]}" "$2" +} + +# ec_ra_prewarm :...: random O_DIRECT reads of the file with +# read-around off, until each 's read latency estimate is at least half +# its delay. The estimate moves 1/32 of the way to each sample, so a test +# that started from a cold one would mostly measure the learning. +ec_ra_prewarm() +{ + local sysfs penalty round m lat cold=1 + sysfs=/sys/fs/bcachefs/$(ec_ra_uuid) + penalty=$(cat "$sysfs/options/ec_read_around_penalty") + echo 0 > "$sysfs/options/ec_read_around_penalty" + for (( round = 0; round < 10 && cold; round++ )); do + fio --name=ec_ra_prewarm --filename=/mnt/data --readonly \ + --rw=randread --bs=64k --direct=1 --ioengine=libaio \ + --iodepth=8 --numjobs=4 --time_based --runtime=3 \ + --output-format=terse > /dev/null + cold=0 + for m in "$@"; do + lat=$(cat "$sysfs/dev-${m%%:*}/io_latency_read") + (( lat * 2 >= ${m#*:} * 1000000 )) || cold=1 + done + done + echo "$penalty" > "$sysfs/options/ec_read_around_penalty" + echo "read latency estimates, prewarmed for $*:" + ec_ra_latencies + if (( cold )); then + echo "ERROR: a delayed member's estimate is still under half its delay" + exit 1 + fi +} + +ec_ra_slow_teardown() +{ + ec_ra_delay_dev 0 0 + ec_ra_delay_dev 1 0 + umount /mnt + bcachefs fsck -ny "${devs[@]}" + dmsetup remove --retry ecslow0 + dmsetup remove --retry ecslow1 + trap 'kill_descendants $BASHPID' EXIT +} + +test_ec_read_slow_device() +{ + set_watchdog 600 + + # 500 ms stands in for the ~7 s a drive commonly spends in error recovery + # (SCT ERC), to keep the test short. + local delay_ms=500 slow_ms=200 devs + ec_ra_slow_setup 5 + + ec_ra_delay_dev 0 $delay_ms + ec_ra_prewarm 0:$delay_ms + + local before around slow + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + around=$(ec_ra_counter data_read_ec_read_around) + ec_ra_read_loop + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + slow=$(ec_ra_over $slow_ms) + + echo "device 0 reads delayed ${delay_ms} ms: $slow of $ec_ra_file_mb 1M O_DIRECT reads took over ${slow_ms} ms (max $(ec_ra_max_ms) ms); $around read-arounds" + ec_ra_reads "${before[@]}" -- "${devs[@]}" + ec_ra_check_read + + # Buffered reads, with the delay still on: + local buffered + buffered=$(ec_ra_counter data_read_ec_read_around) + ec_ra_check_data + buffered=$(( $(ec_ra_counter data_read_ec_read_around) - buffered )) + echo "buffered read of the file: $buffered read-arounds" + + ec_ra_slow_teardown + + if (( slow > ec_ra_coin_reads )); then + echo "ERROR: $slow reads waited on the delayed device although each stripe's" + echo " other buckets could supply the data" + exit 1 + fi + if (( ec_ra_dev_kb[0] > ec_ra_max_dev_read_kb )); then + echo "ERROR: read ${ec_ra_dev_kb[0]} KiB from the delayed device" + exit 1 + fi + if (( around == 0 || buffered == 0 )); then + echo "ERROR: reads of the delayed device's data didn't read around it" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_read_healthy() +{ + set_watchdog 600 + + local devs=("${ktest_scratch_dev[@]:0:5}") + ec_ra_setup "${devs[@]}" + cat /mnt/data > /dev/null + + local around + around=$(ec_ra_counter data_read_ec_read_around) + ec_ra_read_loop + ec_ra_check_read + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + echo "one reader: $around read-arounds" + if (( around != 0 )); then + echo "ERROR: read around a device on a healthy pool" + exit 1 + fi + + # Random reads, 4 x 16 in flight: queueing, uneven at any moment. + local kb ios before + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + around=$(ec_ra_counter data_read_ec_read_around) + kb=$(fio --name=ec_ra --filename=/mnt/data --readonly --rw=randread \ + --bs=64k --direct=1 --ioengine=libaio --iodepth=16 --numjobs=4 \ + --time_based --runtime=30 --group_reporting \ + --output-format=terse | awk -F';' '{ print $6 }') + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + ios=$(( kb / 64 )) + echo "random readers: $around read-arounds in $ios reads" + ec_ra_reads "${before[@]}" -- "${devs[@]}" + + umount /mnt + bcachefs fsck -ny "${devs[@]}" + + if (( ios == 0 )); then + echo "ERROR: fio read nothing - this run tests nothing" + exit 1 + fi + if (( around * 1000 > ios )); then + echo "ERROR: read around devices on a healthy pool in more than 0.1% of reads" + exit 1 + fi + bcachefs_test_end_checks "${devs[0]}" +} + +test_ec_read_around_disabled() +{ + set_watchdog 600 + + local delay_ms=500 slow_ms=200 devs + ec_ra_slow_setup 5 + + echo 0 > "/sys/fs/bcachefs/$(ec_ra_uuid)/options/ec_read_around_penalty" + ec_ra_delay_dev 0 $delay_ms + + local around slow + around=$(ec_ra_counter data_read_ec_read_around) + ec_ra_read_loop + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + slow=$(ec_ra_over $slow_ms) + echo "ec_read_around_penalty=0, device 0 reads delayed ${delay_ms} ms: $slow of $ec_ra_file_mb reads took over ${slow_ms} ms; $around read-arounds" + ec_ra_check_read + + ec_ra_slow_teardown + + if (( around != 0 )); then + echo "ERROR: read around a device with ec_read_around_penalty=0" + exit 1 + fi + if (( slow < 10 )); then + echo "ERROR: only $slow reads waited on the delayed device - this run tests nothing" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_read_around_pq() +{ + set_watchdog 900 + + local devs + ec_ra_slow_setup 6 4 3 + + # In stripes with data on device 0, which block device 1 holds decides + # which recovery a read-around that leaves both out does: + echo "device 1's block in stripes with data on device 0:" + ec_ra_stripes_with_data_on 0 | awk '{ + for (i = 2; i <= NF; i++) { + split($i, d, ":") + if (d[1] == 1) { + b = i - 2 + print b < $1 ? "data" : b == $1 ? "P" : "Q" + } + } + }' | sort | uniq -c + + ec_ra_delay_dev 0 500 + ec_ra_delay_dev 1 50 + ec_ra_prewarm 0:500 1:50 + + local before around slow medium + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + around=$(ec_ra_counter data_read_ec_read_around) + ec_ra_read_loop + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + slow=$(ec_ra_over 200) + medium=$(ec_ra_over 40) + + echo "device 0 delayed 500 ms, device 1 50 ms: of $ec_ra_file_mb 1M O_DIRECT reads $slow took over 200 ms, $medium over 40 ms (max $(ec_ra_max_ms) ms); $around read-arounds" + ec_ra_reads "${before[@]}" -- "${devs[@]}" + ec_ra_check_read + + ec_ra_slow_teardown + + # A read-around that didn't leave the slower of the two out would make + # every read of device 0's data wait. + if (( slow > ec_ra_coin_reads || medium > 4 * ec_ra_coin_reads )); then + echo "ERROR: reads waited on a delayed device although each stripe's other" + echo " buckets could supply the data" + exit 1 + fi + if (( ec_ra_dev_kb[0] > ec_ra_max_dev_read_kb || + ec_ra_dev_kb[1] > ec_ra_max_dev_read_kb )); then + echo "ERROR: read-arounds read a delayed device: with two parity blocks," + echo " one of the stripe's other blocks can be left out" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_read_around_fallback() +{ + set_watchdog 600 + + local devs + ec_ra_slow_setup 5 + + # A bad parity block in a few stripes with data on device 0. Only a + # reconstruct reads parity. + local line f nr=0 + while read -r line; do + read -r -a f <<< "$line" + ec_ra_corrupt_block "${f[$((f[0] + 1))]}" + nr=$((nr + 1)) + (( nr == 3 )) && break + done < <(ec_ra_stripes_with_data_on 0) + echo "corrupted the parity block of $nr stripes with data on device 0" + + # Read-around is off while prewarming, so nothing reads parity yet. + ec_ra_delay_dev 0 500 + ec_ra_prewarm 0:500 + + local around fail + around=$(ec_ra_counter data_read_ec_read_around) + fail=$(ec_ra_counter data_read_ec_read_around_fail) + ec_ra_read_loop + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + fail=$(( $(ec_ra_counter data_read_ec_read_around_fail) - fail )) + echo "$around read-arounds, $fail failed; $(ec_ra_over 200) reads took over 200 ms" + dmesg | grep -m3 "read device directly" || true + ec_ra_check_read + + local damage + damage=$(bcachefs damage /mnt/data) + + ec_ra_slow_teardown + + if (( fail == 0 )); then + echo "ERROR: no read-around failed - this run tests nothing" + exit 1 + fi + if [[ -n $damage ]]; then + echo "ERROR: damage recorded against the file, which was read without error:" + echo "$damage" + exit 1 + fi + # A scrub of the stripe (with the stripe scrub patch) counts the parity + # blocks' checksum errors against their devices; nothing else may count + # any: the file was read without error. + ktest_expect_device_errors=1 + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_read_around_fallback_reconstruct() +{ + set_watchdog 900 + + local devs + ec_ra_slow_setup 6 4 3 + + # Devices 0 and 1 slower than the rest, so a read-around of device 1 + # leaves out both and reads every other block; device 0 is slower, so it + # doesn't read device 0's block either. + ec_ra_delay_dev 0 500 + ec_ra_delay_dev 1 20 + ec_ra_prewarm 0:500 1:20 + + # In one stripe with data on device 1, device 1's block and another data + # block are bad. The read-around of device 1 reads the bad one and fails, + # the direct read gets device 1's bad block, and the full reconstruct has + # two bad blocks and two parity blocks. + local f i dev1 other + read -r -a f < <(ec_ra_stripes_with_data_on 1) + for (( i = 1; i <= f[0]; i++ )); do + [[ ${f[i]%%:*} == 1 ]] && dev1=${f[i]} + [[ ${f[i]%%:*} != [01] ]] && other=${f[i]} + done + ec_ra_corrupt_block "$dev1" + ec_ra_corrupt_block "$other" + echo "corrupted data blocks $dev1 and $other of one stripe" + + local fail + fail=$(ec_ra_counter data_read_ec_read_around_fail) + ec_ra_read_loop + fail=$(( $(ec_ra_counter data_read_ec_read_around_fail) - fail )) + echo "$fail read-arounds failed; $(ec_ra_over 200) reads took over 200 ms" + ec_ra_check_read + + ec_ra_slow_teardown + + if (( fail == 0 )); then + echo "ERROR: no read-around failed - this run tests nothing" + exit 1 + fi + # A read recovered from the bad blocks is counted in the superblock; dmesg + # missed the message in one run. + local errs + errs=$(bcachefs show-super -f errors,errors_v2 "${ktest_scratch_dev[2]}" 2>&1 || + bcachefs show-super -f errors "${ktest_scratch_dev[2]}" 2>&1 || true) + echo "superblock errors:" + echo "$errs" | sed -nE '/^errors(_v2)? /,$p' + if ! grep -q data_read_csum_err_recovered <<< "$errs"; then + echo "ERROR: no read recovered from the bad blocks - this run tests nothing" + exit 1 + fi + ktest_expect_device_errors=1 + ktest_expect_sb_errors="data_read_csum_err_recovered" + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_read_around_small_extents() +{ + set_watchdog 600 + + local devs ec_ra_file_mb=16 + ec_ra_write=ec_ra_write_small_extents ec_ra_slow_setup 5 + + ec_ra_delay_dev 0 500 + ec_ra_prewarm 0:500 + + local before around kb=0 i + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + around=$(ec_ra_counter data_read_ec_read_around) + ec_ra_read_loop + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + echo "4k extents every 8k, device 0 delayed 500 ms: $around read-arounds; $(ec_ra_over 200) of $ec_ra_file_mb reads took over 200 ms" + ec_ra_reads "${before[@]}" -- "${devs[@]}" + ec_ra_check_read + + ec_ra_slow_teardown + + for (( i = 1; i < ${#devs[@]}; i++ )); do + kb=$((kb + ec_ra_dev_kb[i])) + done + # The file holds ec_ra_file_mb / 2 of data. Read directly, that's what + # the devices read; a read-around of a 4k extent reads 4k from each of 3 + # other blocks. Rounded to the stripe's 256k checksum granules, the + # read-arounds alone would read hundreds of MiB. + if (( around < 100 )); then + echo "ERROR: only $around read-arounds - this run tests nothing" + exit 1 + fi + if (( kb > ec_ra_file_mb * 1024 * 3 / 2 )); then + echo "ERROR: the other devices read $kb KiB for $(( ec_ra_file_mb * 512 )) KiB of data" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_read_around_nocsum() +{ + set_watchdog 600 + + local devs + ec_ra_format_opts=--data_checksum=none ec_ra_slow_setup 5 + + ec_ra_delay_dev 0 500 + ec_ra_prewarm 0:500 + + local around slow + around=$(ec_ra_counter data_read_ec_read_around) + ec_ra_read_loop + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + slow=$(ec_ra_over 200) + echo "data_checksum=none, device 0 delayed 500 ms: $slow of $ec_ra_file_mb reads took over 200 ms; $around read-arounds" + ec_ra_check_read + + ec_ra_slow_teardown + + if (( slow > ec_ra_coin_reads || around == 0 )); then + echo "ERROR: reads of unchecksummed data waited on the delayed device" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_repair_reads_around_slow_device() +{ + set_watchdog 600 + + local devs + ec_ra_slow_setup 5 + + ec_ra_delay_dev 0 500 + ec_ra_prewarm 0:500 + + local stripes left waited=0 before + stripes=$(ec_ra_stripes_on 0) + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + bcachefs device set-state -f evacuating "${devs[0]}" + while left=$(ec_ra_stripes_on 0); (( left > 0 && waited < 300 )); do + sleep 1 + waited=$((waited + 1)) + done + echo "evacuating device 0 (delayed 500 ms), which had a bucket in $stripes stripes ($((stripes * ec_ra_bucket_kb)) KiB), took ${waited}s:" + ec_ra_reads "${before[@]}" -- "${devs[@]}" + + if (( left != 0 )); then + bcachefs reconcile status /mnt + echo "ERROR: $left stripes with data still on device 0 after ${waited}s" + exit 1 + fi + ec_ra_check_data + + ec_ra_slow_teardown + + if (( ec_ra_dev_kb[0] > ec_ra_max_dev_read_kb )); then + echo "ERROR: stripe repair read ${ec_ra_dev_kb[0]} KiB from the slow device;" + echo " each stripe's other blocks could rebuild its block" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_reconcile_reads_around_slow_device() +{ + set_watchdog 900 + + local devs + ec_ra_slow_setup 5 + echo "device 0 rotational, so reconcile reads it from its own thread: $(cat /sys/block/"$(basename "$(readlink -f "${ktest_scratch_dev[0]}")")"/queue/rotational)" + + ec_ra_delay_dev 0 500 + ec_ra_prewarm 0:500 + + local stripes before around start + stripes=$(ec_ra_stripes_on 0) + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + around=$(ec_ra_counter data_read_ec_read_around) + start=$(date +%s) + echo "before the rewrite:" + ec_ra_latencies + # What the read-around rule sees while reconcile runs: + (while sleep 0.5; do echo "during:"; ec_ra_latencies; done) & + local sampler=$! + # Every extent now needs rewriting: + echo lz4 > "/sys/fs/bcachefs/$(ec_ra_uuid)/options/background_compression" + if ! timeout 600 bcachefs reconcile wait /mnt; then + kill $sampler || true + bcachefs reconcile status /mnt + echo "ERROR: reconcile didn't finish in 600s" + exit 1 + fi + kill $sampler || true + wait $sampler 2>/dev/null || true + around=$(( $(ec_ra_counter data_read_ec_read_around) - around )) + echo "rewriting the file, device 0 delayed 500 ms, took $(( $(date +%s) - start ))s: $around read-arounds" + echo "after:" + ec_ra_latencies + ec_ra_reads "${before[@]}" -- "${devs[@]}" + ec_ra_check_data + + ec_ra_slow_teardown + + # Device 0 holds a bucket of $stripes stripes; rewriting reads its data, + # stripes * bucket, from it unless it reads around it. + if (( ec_ra_dev_kb[0] * 4 > stripes * ec_ra_bucket_kb )); then + echo "ERROR: reconcile read ${ec_ra_dev_kb[0]} KiB from the slow device, which" + echo " holds $(( stripes * ec_ra_bucket_kb )) KiB of stripe data" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +# Every read 7 s, the time a drive commonly spends in error recovery (SCT ERC). +# From a cold estimate, whatever is in flight before the first sample lands +# reads the member directly; how much that is depends on reconcile's and +# stripe repair's parallelism, so the test only reports it. +ec_ra_failing_ms=7000 +# One 7 s sample takes a cold estimate to 7 s / 32; this asks for several. +ec_ra_learned_ns=1000000000 + +test_ec_reconcile_learns_failing_device() +{ + set_watchdog 900 + + local devs sysfs + ec_ra_slow_setup 5 + sysfs=/sys/fs/bcachefs/$(ec_ra_uuid) + ec_ra_delay_dev 0 $ec_ra_failing_ms + echo "device 0 delayed ${ec_ra_failing_ms} ms, read latency estimates cold:" + ec_ra_latencies + + local stripes before start lat + stripes=$(ec_ra_stripes_on 0) + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + start=$(date +%s) + echo lz4 > "$sysfs/options/background_compression" + if ! timeout 600 bcachefs reconcile wait /mnt; then + bcachefs reconcile status /mnt + echo "ERROR: reconcile didn't finish the rewrite in 600s" + exit 1 + fi + lat=$(cat "$sysfs/dev-0/io_latency_read") + echo "rewriting the file took $(( $(date +%s) - start ))s; device 0 had a bucket in $stripes stripes ($(( stripes * ec_ra_bucket_kb )) KiB):" + ec_ra_latencies + ec_ra_reads "${before[@]}" -- "${devs[@]}" + ec_ra_check_data + + ec_ra_slow_teardown + + if (( lat < ec_ra_learned_ns )); then + echo "ERROR: device 0's read latency estimate is $lat ns after the rewrite" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_evacuate_learns_failing_device() +{ + set_watchdog 900 + + local devs + ec_ra_slow_setup 5 + ec_ra_delay_dev 0 $ec_ra_failing_ms + echo "device 0 delayed ${ec_ra_failing_ms} ms, read latency estimates cold:" + ec_ra_latencies + + local stripes left waited=0 before lat + stripes=$(ec_ra_stripes_on 0) + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + bcachefs device set-state -f evacuating "${devs[0]}" + while left=$(ec_ra_stripes_on 0); (( left > 0 && waited < 600 )); do + sleep 1 + waited=$((waited + 1)) + done + lat=$(cat "/sys/fs/bcachefs/$(ec_ra_uuid)/dev-0/io_latency_read") + echo "evacuating device 0, which had a bucket in $stripes stripes ($((stripes * ec_ra_bucket_kb)) KiB), took ${waited}s:" + ec_ra_latencies + ec_ra_reads "${before[@]}" -- "${devs[@]}" + + if (( left != 0 )); then + bcachefs reconcile status /mnt + echo "ERROR: $left stripes with data still on device 0 after ${waited}s" + exit 1 + fi + ec_ra_check_data + + ec_ra_slow_teardown + + if (( lat < ec_ra_learned_ns )); then + echo "ERROR: device 0's read latency estimate is $lat ns after evacuating it" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_reconstruct_skips_failed_block() +{ + ktest_expect_sb_errors="data_read_io_err_recovered|blk_sts_ioerr" + set_watchdog 600 + + local devs + ec_ra_slow_setup 5 + + # Read errors from device 0's block of one stripe: + local f i sector raw=${ktest_scratch_dev[0]} bsz=$((ec_ra_bucket_kb * 2)) sz + read -r -a f < <(ec_ra_stripes_with_data_on 0) + for (( i = 1; i <= f[0]; i++ )); do + [[ ${f[i]%%:*} == 0 ]] && sector=${f[i]#*:} + done + sz=$(blockdev --getsz "$raw") + printf '%s\n' "0 $sector linear $raw 0" \ + "$sector $bsz error" \ + "$((sector + bsz)) $((sz - sector - bsz)) linear $raw $((sector + bsz))" | + dmsetup reload ecslow0 + dmsetup suspend --nolockfs ecslow0 + dmsetup resume ecslow0 + + # Extents in that block: each direct read of one fails once. + local extents io before after + extents=$(grep -cE " 0:$((sector / bsz)):[0-9]+ gen" \ + "/sys/kernel/debug/bcachefs/$(ec_ra_uuid)/btrees/extents/keys" || true) + io=/sys/fs/bcachefs/$(ec_ra_uuid)/dev-0/io_errors + before=$(awk '$1 == "read:" { print $2; exit }' "$io") + ec_ra_read_loop + ec_ra_check_read + after=$(awk '$1 == "read:" { print $2; exit }' "$io") + echo "read errors on device 0: $((after - before)), for $extents extents in its bad block" + + ec_ra_slow_teardown + + if (( extents == 0 || after == before )); then + echo "ERROR: no reads hit the bad block - this run tests nothing" + exit 1 + fi + if (( after - before > extents )); then + echo "ERROR: the reconstruct after each read error read the failed block again" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_read_around_promotes() +{ + set_watchdog 600 + + # Five members in group hdd, the first two behind dm-delay, and a promote + # target that holds no stripes. + local devs + local -a ec_ra_labels=(hdd.hdd0 hdd.hdd1 hdd.hdd2 hdd.hdd3 hdd.hdd4 ssd.ssd0) + ec_ra_format_opts="--foreground_target=hdd --background_target=hdd" ec_ra_slow_setup 6 + + ec_ra_delay_dev 0 500 + ec_ra_prewarm 0:500 + + # Only now: the setup's reads and the prewarm would promote everything. + local sysfs ssd + sysfs=/sys/fs/bcachefs/$(ec_ra_uuid) + ssd=$(basename "$(readlink -f "${devs[5]}")") + echo ssd > "$sysfs/options/promote_target" + + local around1 fail1 + around1=$(ec_ra_counter data_read_ec_read_around) + fail1=$(ec_ra_counter data_read_ec_read_around_fail) + ec_ra_read_loop + around1=$(( $(ec_ra_counter data_read_ec_read_around) - around1 )) + fail1=$(( $(ec_ra_counter data_read_ec_read_around_fail) - fail1 )) + ec_ra_check_read + + # Promotes are written asynchronously: wait for writes to the promote + # target to stop. + local w prev=-1 still=0 t + for (( t = 0; t < 60 && still < 3; t++ )); do + w=$(awk '{ print $7 }' "/sys/block/$ssd/stat") + if (( w == prev )); then still=$((still + 1)); else still=0; fi + prev=$w + sleep 1 + done + echo "first pass: $around1 read-arounds ($fail1 failed); promoted:" + grep -h "since mount" "$sysfs/counters/data_read_promote" || true + + local before around2 slow2 + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + around2=$(ec_ra_counter data_read_ec_read_around) + ec_ra_read_loop + around2=$(( $(ec_ra_counter data_read_ec_read_around) - around2 )) + slow2=$(ec_ra_over 200) + echo "second pass: $around2 read-arounds; $slow2 of $ec_ra_file_mb reads took over 200 ms" + ec_ra_reads "${before[@]}" -- "${devs[@]}" + ec_ra_check_read + + ec_ra_slow_teardown + + if (( around1 == 0 )); then + echo "ERROR: no read-arounds in the first pass - this run tests nothing" + exit 1 + fi + if (( around2 > 0 )); then + echo "ERROR: the second pass read around again: read-arounds weren't promoted" + exit 1 + fi + if (( ec_ra_dev_kb[5] < ec_ra_file_mb * 1024 / 2 )); then + echo "ERROR: the second pass read only ${ec_ra_dev_kb[5]} KiB from the promote target" + exit 1 + fi + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + + +test_ec_read_around_bad_block_no_slow_wait() +{ + set_watchdog 900 + + local devs + ec_ra_slow_setup 6 4 3 + + # Device 1 slower than the rest, so a read-around of device 0 leaves its + # block out: with P+Q, that block and the four read are one more than a + # rebuild needs. + ec_ra_delay_dev 0 500 + ec_ra_delay_dev 1 20 + ec_ra_prewarm 0:500 1:20 + + # A parity block on a fast member is bad in a few stripes with data on + # device 0: the read-around reads it and fails. Parity, so that no read of + # the file's own data fails and goes to the full reconstruct, which reads + # every block. + local line f i nr=0 + while read -r line && (( nr < 3 )); do + read -r -a f <<< "$line" + for (( i = f[0] + 1; i < ${#f[@]}; i++ )); do + if [[ ${f[i]%%:*} != [01] ]]; then + ec_ra_corrupt_block "${f[i]}" + nr=$((nr + 1)) + break + fi + done + done < <(ec_ra_stripes_with_data_on 0) + echo "corrupted a parity block of $nr stripes with data on device 0" + + local widen fail slow + widen=$(ec_ra_counter data_read_ec_read_around_widen) + fail=$(ec_ra_counter data_read_ec_read_around_fail) + ec_ra_read_loop + widen=$(( $(ec_ra_counter data_read_ec_read_around_widen) - widen )) + fail=$(( $(ec_ra_counter data_read_ec_read_around_fail) - fail )) + slow=$(ec_ra_over 200) + echo "$widen read-arounds read the block they left out, $fail failed;" + echo "$slow reads took over 200 ms, the slowest $(ec_ra_max_ms) ms" + ec_ra_check_read + + ec_ra_slow_teardown + + # The read path goes around the slow member with probability 1 - (r/d)^2, + # so a read may still go to it directly: allow for those. + if (( slow > ec_ra_coin_reads )); then + echo "ERROR: $slow reads waited on the slow member; a read-around that read a" + echo " bad block could have read the block it left out instead" + exit 1 + fi + if (( widen == 0 )); then + echo "ERROR: no read-around read the block it left out - this run tests nothing" + exit 1 + fi + ktest_expect_device_errors=1 + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +test_ec_repair_bad_block_no_slow_read() +{ + set_watchdog 900 + + local devs + ec_ra_slow_setup 6 4 3 + + # Repair reads around the two slowest members of each stripe, device 0 and + # usually device 1: with a bad block among the four it reads, the faster + # of those two is enough to rebuild device 0's. + ec_ra_delay_dev 0 500 + ec_ra_delay_dev 1 20 + ec_ra_prewarm 0:500 1:20 + + # After the prewarm, so that no read finds the bad blocks first. Enough + # stripes that reading device 0's block of each would show. Parity, as a + # bad data block's data is moved, and the move's read of it goes to the + # full reconstruct, which reads every block. + local line f i nr=0 + while read -r line && (( nr < 16 )); do + read -r -a f <<< "$line" + for (( i = f[0] + 1; i < ${#f[@]}; i++ )); do + if [[ ${f[i]%%:*} != [01] ]]; then + ec_ra_corrupt_block "${f[i]}" + nr=$((nr + 1)) + break + fi + done + done < <(ec_ra_stripes_with_data_on 0) + echo "corrupted a parity block of $nr stripes with data on device 0" + + local left waited=0 before + mapfile -t before < <(ec_ra_snapshot "${devs[@]}") + bcachefs device set-state -f evacuating "${devs[0]}" + while left=$(ec_ra_stripes_on 0); (( left > 0 && waited < 600 )); do + sleep 1 + waited=$((waited + 1)) + done + echo "evacuating device 0 took ${waited}s:" + ec_ra_reads "${before[@]}" -- "${devs[@]}" + + if (( left != 0 )); then + bcachefs reconcile status /mnt + echo "ERROR: $left stripes with data still on device 0 after ${waited}s" + exit 1 + fi + ec_ra_check_data + + ec_ra_slow_teardown + + # Repair has no coin toss (bch2_ec_read_around_skip()): it never reads the + # slow member's blocks, so the limit is only for btree and journal reads. + if (( ec_ra_dev_kb[0] > ec_ra_max_dev_read_kb )); then + echo "ERROR: stripe repair read ${ec_ra_dev_kb[0]} KiB from the slow device;" + echo " one more fast block and the four read could rebuild each one" + exit 1 + fi + local errs + errs=$(bcachefs show-super -f errors,errors_v2 "${ktest_scratch_dev[2]}" 2>&1 || + bcachefs show-super -f errors "${ktest_scratch_dev[2]}" 2>&1 || true) + if ! grep -q stripe_read_csum_err <<< "$errs"; then + echo "ERROR: stripe repair found no bad block - this run tests nothing" + exit 1 + fi + ktest_expect_device_errors=1 + ktest_expect_sb_errors="stripe_read_csum_err" + bcachefs_test_end_checks "${ktest_scratch_dev[2]}" +} + +main "$@"