bcachefs: regression tests for tools#1125

tests/fs/bcachefs/reconcile-restart.ktest

@@ -32,6 +32,8 @@
 
 . $(dirname $(readlink -e ${BASH_SOURCE[0]}))/bcachefs-test-libs.sh
 
+config-scratch-devs 4G
+config-scratch-devs 4G
 config-scratch-devs 4G
 
 # "The thread is running" and "we couldn't tell" are different answers, and the
@@ -206,6 +208,79 @@ test_reconcile_survives_disable_during_scan()
     bcachefs_test_end_checks "$dev"
 }
 
+# pending reconcile work in bytes, from fs usage
+reconcile_pending_bytes()
+{
+    bcachefs fs usage -f rebalance_work /mnt |
+	awk '$1 == "pending:" { print $2; exit }'
+}
+
+# A device add queues a pending scan so the extents parked for lack of
+# devices are retried. If the pass is interrupted before the pending phase
+# the cookie must still be there for the next pass; it was deleted in the
+# scan phase, the first of the pass, so the work was lost.
+test_reconcile_pending_cookie_kept()
+{
+    set_watchdog 600
+    local devs=("${ktest_scratch_dev[0]}" "${ktest_scratch_dev[1]}")
+
+    run_quiet "" bcachefs format -f --replicas=2 "${devs[@]}"
+    mount -t bcachefs "$(join_by : "${devs[@]}")" /mnt
+    dd if=/dev/urandom of=/mnt/data bs=1M count=1024 oflag=direct status=none
+
+    # three replicas on two devices: every extent parks on the pending list
+    echo 3 > /sys/fs/bcachefs/*/options/data_replicas
+    timeout 300 bcachefs reconcile wait /mnt
+    local pending=$(reconcile_pending_bytes)
+    echo "pending after data_replicas=3: $pending"
+    if ((pending == 0)); then
+	echo "ERROR: no extents parked on the pending list"
+	return 1
+    fi
+
+    # Queue the device add's pending scan with reconcile stopped, so the pass
+    # starts when we start watching it.
+    echo 0 > /sys/fs/bcachefs/*/options/reconcile_enabled
+    bcachefs device add -f /mnt ${ktest_scratch_dev[2]}
+    echo 1 > /sys/fs/bcachefs/*/options/reconcile_enabled
+
+    # The scan phase comes first; stop the pass once it's past it and still
+    # working, before the pending phases complete.
+    local i status
+    for i in $(seq 1200); do
+	status=$(cat /sys/fs/bcachefs/*/reconcile_status)
+	grep "^processing" <<<"$status" >/dev/null && break
+	if grep "^waiting" <<<"$status" >/dev/null && ((i > 40)); then
+	    echo "$status"
+	    echo "ERROR: the pass finished before the test could interrupt it"
+	    return 1
+	fi
+	sleep 0.05
+    done
+    echo "$status"
+    echo 0 > /sys/fs/bcachefs/*/options/reconcile_enabled
+    sleep 1
+
+    local cookies=$(bcachefs list -b reconcile_scan /mnt)
+    echo "$cookies"
+    if ! grep "cookie 0:2:" <<<"$cookies" >/dev/null; then
+	echo "ERROR: the pending scan cookie was deleted before the pending phase ran"
+	return 1
+    fi
+
+    echo 1 > /sys/fs/bcachefs/*/options/reconcile_enabled
+    timeout 300 bcachefs reconcile wait /mnt
+    pending=$(reconcile_pending_bytes)
+    echo "pending after the retry: $pending"
+    if ((pending != 0)); then
+	echo "ERROR: pending work wasn't retried after the interrupted pass"
+	return 1
+    fi
+
+    umount /mnt
+    bcachefs_test_end_checks ${devs[0]}
+}
+
 # Suspend and hibernate failed with "Freezing of tasks failed ... bch-reconcile"
 # (bcachefs #700, tools #972). The freezer stops freezable workqueues before it
 # freezes kernel threads, and move completions run on freezable workqueues - so

tests/fs/bcachefs/replication.ktest

@@ -1440,6 +1440,32 @@ test_rereplicate2()
     umount /mnt
 }
 
+# With a metadata_replicas change pending (its scan not yet run) the btree
+# ptrs are allowed to mismatch: the metadata cookie says so. The check read
+# the cookie at the node key's inode field instead and reported them.
+test_reconcile_btree_ptr_metadata_cookie()
+{
+    set_watchdog 120
+    local devs=("${ktest_scratch_dev[0]}" "${ktest_scratch_dev[1]}")
+
+    run_quiet "" bcachefs format -f --metadata_replicas=1 "${devs[@]}"
+    mount -t bcachefs -o reconcile_enabled=0 "$(join_by : "${devs[@]}")" /mnt
+    for i in $(seq 500); do echo $i > /mnt/f$i; done
+    sync
+
+    echo 2 > /sys/fs/bcachefs/*/options/metadata_replicas
+    umount /mnt
+
+    local out rc=0
+    out=$(bcachefs fsck -n "${devs[@]}" 2>&1) || rc=$?
+    echo "fsck -n exited $rc"
+    if grep "incorrect/missing reconcile opts" <<<"$out" >/dev/null; then
+	grep -A6 "incorrect/missing reconcile opts" <<<"$out" | head -40
+	echo "ERROR: btree ptrs reported while the metadata scan is pending"
+	return 1
+    fi
+}
+
 # Bumping data_replicas on a single file (set-file-option, not the fs-wide
 # sysfs knob) must re-replicate that file's existing extents: it queues an
 # inum reconcile scan that re-derives need_rb=data_replicas per extent and
@@ -3783,5 +3809,83 @@ test_scrub_corrupt_ec_unreferenced()
     bcachefs_test_end_checks ${ktest_scratch_dev[$vdev]}
 }
 
+# The leaf nodes of a btree that have a replica on <dev_idx>, one line each:
+# btree_leaf_lines <btree> <dev_idx> <devs>...
+btree_leaf_lines()
+{
+    local btree=$1 dev_idx=$2
+    shift 2
+    # a key's value continues on indented lines: join them
+    bcachefs list -m nodes -b $btree "$@" |
+	awk '{ gsub(/[ \t]+/, " ") }
+	     /^u64s/ { if (line) print line; line = $0; next }
+	     { line = line " " $0 }
+	     END { if (line) print line }' |
+	grep "btree_ptr" | grep -E "ptr: ([^ ]+ )?$dev_idx:[0-9]+:[0-9]+"
+}
+
+# A replica that holds another node's data (an older node left by a lost
+# write) has valid checksums and a wrong seq. Scrub must rewrite the node;
+# it compared nothing but the checksums and called the replica good.
+test_btree_scrub_stale_replica()
+{
+    ktest_expect_device_errors=1
+    set_watchdog 300
+    local devs=("${ktest_scratch_dev[0]}" "${ktest_scratch_dev[1]}")
+
+    run_quiet "" bcachefs format -f		\
+	--metadata_replicas=2			\
+	--bucket_size=${BUCKET_SIZE_KB}k	\
+	--btree_node_size=64k			\
+	"${devs[@]}"
+    mount -t bcachefs "$(join_by : "${devs[@]}")" /mnt
+
+    for i in $(seq 2000); do echo $i > /mnt/f$i; done
+    sync
+    umount /mnt
+
+    local before=$(btree_leaf_lines inodes 1 "${devs[@]}")
+    local nr=$(echo "$before" | wc -l)
+    echo "inodes btree: $nr nodes with a dev 1 replica"
+    if ((nr < 2)); then
+	echo "ERROR: need two inodes leaves, got $nr"
+	return 1
+    fi
+
+    local n_line=$(echo "$before" | sed -n 1p)
+    local m_line=$(echo "$before" | sed -n 2p)
+    local n_sector=$(echo "$n_line" | bcachefs_ptr_sectors $BUCKET_SIZE_SECTORS 1)
+    local m_sector=$(echo "$m_line" | bcachefs_ptr_sectors $BUCKET_SIZE_SECTORS 1)
+    echo "node N: $n_line"
+    echo "node M: $m_line"
+
+    # Cache every inodes leaf first, so the kernel never reads the stale
+    # replica itself: only scrub reads it, from disk. noatime, so nothing
+    # but a rewrite changes N.
+    mount -t bcachefs -o noatime "$(join_by : "${devs[@]}")" /mnt
+    ls -l /mnt > /dev/null
+
+    # M's replica over N's
+    dd if=${devs[1]} of=${devs[1]} bs=512 skip=$m_sector seek=$n_sector	\
+	count=$((64 * 2)) iflag=direct oflag=direct conv=notrunc status=none
+
+    local rc=0
+    bcachefs scrub /mnt || rc=$?
+    echo "scrub exit code: $rc"
+    umount /mnt
+
+    local n_pos=$(echo "$n_line" | awk '{ for (i = 1; i < NF; i++) if ($i == "btree_ptr_v2") print $(i + 1) }')
+    local n_after=$(btree_leaf_lines inodes 1 "${devs[@]}" |
+	awk -v pos="$n_pos" '{ for (i = 1; i < NF; i++) if ($i == "btree_ptr_v2" && $(i + 1) == pos) print }')
+    local n_sector_after=$(echo "$n_after" | bcachefs_ptr_sectors $BUCKET_SIZE_SECTORS 1)
+    echo "N's dev 1 replica: sector $n_sector, after scrub ${n_sector_after:-none}"
+    if [[ $n_sector_after == "$n_sector" ]]; then
+	echo "ERROR: scrub left the stale replica of node N in place"
+	return 1
+    fi
+
+    bcachefs fsck -ny "${devs[@]}"
+}
+
 
 main "$@"