tests/generic/778

#! /bin/bash
# SPDX-License-Identifier: GPL-2.0
# Copyright (c) 2025 IBM Corporation. All Rights Reserved.
#
# FS QA Test No. 778
#
# Test multi block atomic writes with sudden FS shutdowns to ensure
# the FS is not tearing the write operation
. ./common/preamble
. ./common/atomicwrites
_begin_fstest auto atomicwrites

_require_scratch_write_atomic_multi_fsblock
_require_atomic_write_test_commands
_require_scratch_shutdown
_require_xfs_io_command "truncate"

_scratch_mkfs >> $seqres.full 2>&1
_scratch_mount >> $seqres.full

testfile=$SCRATCH_MNT/testfile
touch $testfile

awloop_runfile=$tmp.awloop_running
awloop_killfile=$tmp.awloop_kill

awu_max=$(_get_atomic_write_unit_max $testfile)
blksz=$(_get_block_size $SCRATCH_MNT)
echo "Awu max: $awu_max" >> $seqres.full

num_blocks=$((awu_max / blksz))
# keep initial value high for dry run. This will be
# tweaked in dry_run() based on device write speed.
filesize=$(( 10 * 1024 * 1024 * 1024 ))

_cleanup() {
	kill_awloop
}

atomic_write_loop() {
	local off=0
	local size=$awu_max

	rm -f $awloop_killfile
	touch $awloop_runfile

	for ((i=0; i<$((filesize / $size )); i++)); do
		# Due to sudden shutdown this can produce errors so just
		# redirect them to seqres.full
		$XFS_IO_PROG -c "open -fsd $testfile" -c "pwrite -S 0x61 -DA -V1 -b $size $off $size" >> /dev/null 2>>$seqres.full
		if [ ! -w "$testfile" ] || [ -e "$awloop_killfile" ]; then
			break
		fi
		echo "Written to offset: $((off + size))" >> $tmp.aw
		off=$((off + size))
	done

	rm -f $awloop_runfile
}

# Use sentinel files to control the loop execution because we don't know the
# pid of the xfs_io process and so we can't wait for it directly.  A bare
# wait command won't wait for a D-state xfs_io process so we can't do that
# either.  We can't use killall because check-parallel, and we can't pkill
# because the pid namespacing code was removed withotu fixing check-parallel.
kill_awloop() {
	test -e $awloop_runfile || return

	touch $awloop_killfile

	for ((i=0;i<300;i++)); do
		test -e $awloop_runfile || break
		sleep 0.1
	done
}

start_atomic_write_and_shutdown() {
	atomic_write_loop &
	local max_loops=100

	local i=0
	# Wait for at least first write to be recorded or too much time passes
	while [ ! -f "$tmp.aw" -a $i -le $max_loops ]; do
		i=$((i + 1))
		sleep 0.2
	done

	cat $tmp.aw >> $seqres.full

	if [[ $i -gt $max_loops ]]
	then
		_notrun "atomic write process took too long to start"
	fi

	echo >> $seqres.full
	echo "# Shutting down filesystem while write is running" >> $seqres.full
	_scratch_shutdown

	kill_awloop
}

# This test has the following flow:
# 1. Start doing sequential atomic writes in background, upto $filesize
# 2. Sleep for 0.2s and shutdown the FS
# 3. kill the atomic write process
# 4. verify the writes were not torn
#
# We ideally want the shutdown to happen while an atomic write is ongoing
# but this gets tricky since faster devices can actually finish the whole
# atomic write loop before sleep 0.2s completes, resulting in the shutdown
# happening after the write loop which is not what we want. A simple solution
# to this is to increase $filesize so step 1 takes long enough but a big
# $filesize leads to create_mixed_mappings() taking very long, which is not
# ideal.
#
# Hence, use the dry_run function to figure out the rough device speed and set
# $filesize accordingly.
dry_run() {
	echo >> $seqres.full
	echo "# Estimating ideal filesize..." >> $seqres.full

	start_atomic_write_and_shutdown

	bytes_written=$(tail -n 1 $tmp.aw | cut -d" " -f4)
	echo "# Bytes written in 0.2s: $bytes_written" >> $seqres.full

	filesize=$((bytes_written * 3))
	echo "# Setting \$filesize=$filesize" >> $seqres.full

	rm $tmp.aw
	sleep 0.5

	_scratch_cycle_mount

}

create_mixed_mappings() {
	local file=$1
	local size_bytes=$2

	echo "# Filling file $file with alternate mappings till size $size_bytes" >> $seqres.full
	#Fill the file with alternate written and unwritten blocks
	local off=0
	local operations=("W" "U")

	test $size_bytes -eq 0 && return

	# fallocate the whole file once because preallocating single blocks
	# with individual xfs_io invocations is really slow and the allocator
	# usually gives out consecutive blocks anyway
	$XFS_IO_PROG -f -c "falloc 0 $size_bytes" $file

	local cmds=()
	for ((i=0; i<$((size_bytes / blksz )); i++)); do
		if (( i % 2 == 0 )); then
			cmds+=(-c "pwrite -b $blksz $off $blksz")
		fi

		# batch the write commands into larger xfs_io invocations to
		# amortize the fork overhead
		if [ "${#cmds[@]}" -ge 128 ]; then
			$XFS_IO_PROG "${cmds[@]}" "$file" >> /dev/null
			cmds=()
		fi

		off=$((off + blksz))
	done

	if [ "${#cmds[@]}" -gt 0 ]; then
		$XFS_IO_PROG "${cmds[@]}" "$file" >> /dev/null
		cmds=()
	fi

	sync $file
}

populate_expected_data() {
	# create a dummy file with expected old data for different cases
	create_mixed_mappings $testfile.exp_old_mixed $awu_max
	expected_data_old_mixed=$(od -An -t x1 -j 0 -N $awu_max $testfile.exp_old_mixed)

	$XFS_IO_PROG -fc "falloc 0 $awu_max" $testfile.exp_old_zeroes >> $seqres.full
	expected_data_old_zeroes=$(od -An -t x1 -j 0 -N $awu_max $testfile.exp_old_zeroes)

	$XFS_IO_PROG -fc "pwrite -b $awu_max 0 $awu_max" $testfile.exp_old_mapped >> $seqres.full
	expected_data_old_mapped=$(od -An -t x1 -j 0 -N $awu_max $testfile.exp_old_mapped)

	# create a dummy file with expected new data
	$XFS_IO_PROG -fc "pwrite -S 0x61 -b $awu_max 0 $awu_max" $testfile.exp_new >> $seqres.full
	expected_data_new=$(od -An -t x1 -j 0 -N $awu_max $testfile.exp_new)
}

verify_data_blocks() {
	local verify_start=$1
	local verify_end=$2
	local expected_data_old="$3"
	local expected_data_new="$4"

	echo >> $seqres.full
	echo "# Checking for torn write from $verify_start to $verify_end" >> $seqres.full

	# After an atomic write, for every chunk we ensure that the underlying
	# data is either the old data or new data as writes shouldn't get torn.
	local off=$verify_start
	while [[ "$off" -lt "$verify_end" ]]
	do
		#actual_data=$(xxd -s $off -l $awu_max -p $testfile)
		actual_data=$(od -An -t x1 -j $off -N $awu_max $testfile)
		if [[ "$actual_data" != "$expected_data_new" ]] && [[ "$actual_data" != "$expected_data_old" ]]
		then
			echo "Checksum match failed at off: $off size: $awu_max"
			echo "Expected contents: (Either of the 2 below):"
			echo
			echo "Old: "
			echo "$expected_data_old"
			echo
			echo "New: "
			echo "$expected_data_new"
			echo
			echo "Actual contents: "
			echo "$actual_data"

			_fail
		fi
		echo -n "Check at offset $off succeeded! " >> $seqres.full
		if [[ "$actual_data" == "$expected_data_new" ]]
		then
			echo "matched new" >> $seqres.full
		elif [[ "$actual_data" == "$expected_data_old" ]]
		then
			echo "matched old" >> $seqres.full
		fi
		off=$(( off + awu_max ))
	done
}

# test torn write for file by shutting down in between atomic writes
test_torn_write() {
	echo >> $seqres.full
	echo "# Writing atomically to file in background" >> $seqres.full

	start_atomic_write_and_shutdown

	last_offset=$(tail -n 1 $tmp.aw | cut -d" " -f4)
	if [[ -z $last_offset ]]
	then
		last_offset=0
	fi

	echo >> $seqres.full
	echo "# Last offset of atomic write: $last_offset" >> $seqres.full

	rm $tmp.aw
	sleep 0.5

	_scratch_cycle_mount

	# we want to verify all blocks around which the shutdown happened
	verify_start=$(( last_offset - (awu_max * 5)))
	if [[ $verify_start < 0 ]]
	then
		verify_start=0
	fi

	verify_end=$(( last_offset + (awu_max * 5)))
	if [[ "$verify_end" -gt "$filesize" ]]
	then
		verify_end=$filesize
	fi
}

# test torn write for file with written and unwritten mappings
test_torn_write_mixed() {
	$XFS_IO_PROG -fc "truncate 0" $testfile >> $seqres.full

	echo >> $seqres.full
	echo "# Creating testfile with mixed mappings" >> $seqres.full
	create_mixed_mappings $testfile $filesize

	test_torn_write

	verify_data_blocks $verify_start $verify_end "$expected_data_old_mixed" "$expected_data_new"
}

# test torn write for file with completely written mappings
test_torn_write_written() {
	$XFS_IO_PROG -fc "truncate 0" $testfile >> $seqres.full

	echo >> $seqres.full
	echo "# Creating testfile with fully written mapping" >> $seqres.full
	$XFS_IO_PROG -c "pwrite -b $filesize 0 $filesize" $testfile >> $seqres.full
	sync $testfile

	test_torn_write

	verify_data_blocks $verify_start $verify_end "$expected_data_old_mapped" "$expected_data_new"
}

# test torn write for file with completely unwritten mappings
test_torn_write_unwritten() {
	$XFS_IO_PROG -fc "truncate 0" $testfile >> $seqres.full

	echo >> $seqres.full
	echo "# Creating testfile with fully unwritten mappings" >> $seqres.full
	$XFS_IO_PROG -c "falloc 0 $filesize" $testfile >> $seqres.full
	sync $testfile

	test_torn_write

	verify_data_blocks $verify_start $verify_end "$expected_data_old_zeroes" "$expected_data_new"
}

# test torn write for file with no mappings
test_torn_write_hole() {
	$XFS_IO_PROG -fc "truncate 0" $testfile >> $seqres.full

	echo >> $seqres.full
	echo "# Creating testfile with no mappings" >> $seqres.full
	$XFS_IO_PROG -c "truncate $filesize" $testfile >> $seqres.full
	sync $testfile

	test_torn_write

	verify_data_blocks $verify_start $verify_end "$expected_data_old_zeroes" "$expected_data_new"
}

test_append_torn_write() {
	$XFS_IO_PROG -c "truncate 0" $testfile >> $seqres.full

	echo >> $seqres.full
	echo "# Performing append atomic writes over file in background" >> $seqres.full

	start_atomic_write_and_shutdown

	local last_offset=$(tail -n 1 $tmp.aw | cut -d" " -f4)
	if [[ -z $last_offset ]]
	then
		last_offset=0
	fi

	echo >> $seqres.full
	echo "# Last offset of atomic write: $last_offset" >> $seqres.full
	rm $tmp.aw
	sleep 0.5

	_scratch_cycle_mount
	local filesize=$(_get_filesize $testfile)
	echo >> $seqres.full
	echo "# Filesize after shutdown: $filesize" >> $seqres.full

	# To confirm that the write went atomically, we check:
	# 1. The last block should be a multiple of awu_max
	# 2. The last block should be the completely new data

	if (( $filesize % $awu_max ))
	then
		echo "Filesize after shutdown ($filesize) not a multiple of atomic write unit ($awu_max)"
	fi

	verify_start=$(( filesize - (awu_max * 5)))
	if [[ $verify_start < 0 ]]
	then
		verify_start=0
	fi

	local verify_end=$filesize

	# Here the blocks should always match new data hence, for simplicity of
	# code, just corrupt the $expected_data_old buffer so it never matches
	local expected_data_old="POISON"
	verify_data_blocks $verify_start $verify_end "$expected_data_old" "$expected_data_new"
}

$XFS_IO_PROG -fc "truncate 0" $testfile >> $seqres.full

dry_run

echo >> $seqres.full
echo "# Populating expected data buffers" >> $seqres.full
populate_expected_data

# Loop to shake out any races due to shutdown
iter=0
while _soak_loop_running $TIME_FACTOR; do
	echo >> $seqres.full
	echo "------ Iteration $iter ------" >> $seqres.full

	echo >> $seqres.full
	echo "# Starting torn write test for atomic writes over mixed mapping" >> $seqres.full
	test_torn_write_mixed

	echo >> $seqres.full
	echo "# Starting torn write test for atomic writes over fully written mapping" >> $seqres.full
	test_torn_write_written

	echo >> $seqres.full
	echo "# Starting torn write test for atomic writes over fully unwritten mapping" >> $seqres.full
	test_torn_write_unwritten

	echo >> $seqres.full
	echo "# Starting torn write test for atomic writes over holes" >> $seqres.full
	test_torn_write_hole

	echo >> $seqres.full
	echo "# Starting shutdown torn write test for append atomic writes" >> $seqres.full
	test_append_torn_write

	iter=$((iter + 1))
done

echo "Silence is golden"
status=0
exit