common/fuzzy - pub/scm/linux/kernel/git/djwong/xfstests-dev - Git at Google

 ##/bin/bash
 # SPDX-License-Identifier: GPL-2.0+
 # Copyright (c) 2017 Oracle.  All Rights Reserved.
 #
 # Routines for fuzzing and scrubbing a filesystem.

 # Modify various files after a fuzzing operation
 _scratch_fuzz_modify() {
 	echo "+++ stressing filesystem"
 	mkdir -p $SCRATCH_MNT/data
 	_xfs_force_bdev data $SCRATCH_MNT/data
 	$FSSTRESS_PROG -n $((TIME_FACTOR * 10000)) -p $((LOAD_FACTOR * 4)) -d $SCRATCH_MNT/data

 	is_rt="$($XFS_INFO_PROG "$SCRATCH_MNT" | grep -c 'rtextents=[1-9]')"
 	if [ $is_rt -gt 0 ]; then
 		mkdir -p $SCRATCH_MNT/rt
 		_xfs_force_bdev realtime $SCRATCH_MNT/data
 		$FSSTRESS_PROG -n $((TIME_FACTOR * 10000)) -p $((LOAD_FACTOR * 4)) -d $SCRATCH_MNT/rt
 	else
 		echo "+++ xfs realtime not configured"
 	fi
 }

 # Try to access files after fuzzing
 _scratch_fuzz_test() {
 	echo "+++ ls -laR" >> $seqres.full
 	ls -laR "${SCRATCH_MNT}/test.1/" >/dev/null 2>&1

 	echo "+++ cat files" >> $seqres.full
 	(find "${SCRATCH_MNT}/test.1/" -type f -size -1048576k -print0 | xargs -0 cat) >/dev/null 2>&1
 }

 # Do we have an online scrub program?
 _require_scrub() {
 	case "${FSTYP}" in
 	"xfs")
 		test -x "$XFS_SCRUB_PROG" || _notrun "xfs_scrub not found"
 		;;
 	*)
 		_notrun "No online scrub program for ${FSTYP}."
 		;;
 	esac
 }

 # Scrub the scratch filesystem metadata (online)
 _scratch_scrub() {
 	case "${FSTYP}" in
 	"xfs")
 		$XFS_SCRUB_PROG -d -T -v "$@" $SCRATCH_MNT
 		;;
 	*)
 		_fail "No online scrub program for ${FSTYP}."
 		;;
 	esac
 }

 # Expand indexed keys (i.e. arrays) into a long format so that we can filter
 # the array indices individually, and pass regular keys right through.
 #
 # For example, "u3.bmx[0-1] = [foo,bar]" is exploded into:
 # u3.bmx[0] = [foo,bar]
 # u3.bmx[1] = [foo,bar]
 #
 # Note that we restrict array indices to [0-9] to reduce fuzz runtime.  The
 # minimum and maximum array indices can be changed by setting the variables
 # SCRATCH_XFS_{MIN,MAX}_ARRAY_IDX.
 #
 # Also filter padding fields.
 __explode_xfs_db_fields() {
 	local min_idx="${SCRATCH_XFS_MIN_ARRAY_IDX}"
 	local max_idx="${SCRATCH_XFS_MAX_ARRAY_IDX}"

 	test -z "${min_idx}" && min_idx=0
 	test -z "${max_idx}" && max_idx=9
 	test "${max_idx}" = "none" && max_idx=99999

 	grep ' = ' | \
 	sed -e 's/^\([.a-zA-Z0-9_]*\)\[\([0-9]*\)-\([0-9]*\)\]\(.*\) = \(.*\)$/\1[%d]\4 \2 \3 = \5/g' \
 	    -e 's/^\([.a-zA-Z0-9_]*\)\[\([0-9]*\)\]\(.*\) = \(.*\)$/\1[%d]\3 \2 \2 = \4/g' | \
 	while read name col1 col2 rest; do
 		if [[ "${name}" == *pad* ]]; then
 			continue
 		fi

 		if [ "${col1}" = "=" ]; then
 			echo "${name} ${col1} ${col2} ${rest}"
 			continue
 		fi

 		test "${min_idx}" -gt "${col1}" && col1="${min_idx}"
 		test "${max_idx}" -lt "${col2}" && col2="${max_idx}"

 		seq "${col1}" "${col2}" | while read idx; do
 			printf "${name} %s\n" "${idx}" "${rest}"
 		done
 	done
 }

 # Filter the xfs_db print command's field debug information
 # into field name and type.
 __filter_xfs_db_print_fields() {
 	filter="$1"
 	if [ -z "${filter}" ] || [ "${filter}" = "nofilter" ]; then
 		filter='^'
 	fi
 	__explode_xfs_db_fields | while read key equals value; do
 		fuzzkey="$(echo "${key}")"
 		if [ -z "${fuzzkey}" ]; then
 			continue
 		elif [[ "${value}" == "["* ]]; then
 			echo "${value}" | sed -e 's/^.//g' -e 's/.$//g' -e 's/,/\n/g' | while read subfield; do
 				echo "${fuzzkey}.${subfield}"
 			done
 		else
 			echo "${fuzzkey}"
 		fi
 	done | egrep "${filter}" | egrep -v '(^|\.)(sec|nsec|lsn)$'
 }

 # Navigate to some part of the filesystem and print the field info.
 # The first argument is an egrep filter for the fields
 # The rest of the arguments are xfs_db commands to locate the metadata.
 _scratch_xfs_list_metadata_fields() {
 	filter="$1"
 	shift
 	if [ -n "${SCRATCH_XFS_LIST_METADATA_FIELDS}" ]; then
 		echo "${SCRATCH_XFS_LIST_METADATA_FIELDS}" | tr '[ ,]' '[\n\n]'
 		return;
 	fi

 	local cmds=()
 	for arg in "$@"; do
 		cmds+=("-c" "${arg}")
 	done
 	_scratch_xfs_db "${cmds[@]}" -c print | __filter_xfs_db_print_fields "${filter}"
 }

 # Fuzz a metadata field
 # The first arg is the field name
 # The second arg is the xfs_db fuzz verb
 # The rest of the arguments are xfs_db commands to find the metadata.
 _scratch_xfs_fuzz_metadata_field() {
 	key="$1"
 	value="$2"
 	shift; shift

 	if [[ "${key}" == *crc ]]; then
 		fuzz_arg="-c"
 	else
 		fuzz_arg="-d"
 	fi
 	oldval="$(_scratch_xfs_get_metadata_field "${key}" "$@")"

 	local cmds=()
 	for arg in "$@"; do
 		cmds+=("-c" "${arg}")
 	done
 	while true; do
 		_scratch_xfs_db -x "${cmds[@]}" -c "fuzz ${fuzz_arg} ${key} ${value}"
 		echo
 		newval="$(_scratch_xfs_get_metadata_field "${key}" "$@" 2> /dev/null)"
 		if [ "${key}" != "random" ] || [ "${oldval}" != "${newval}" ]; then
 			break;
 		fi
 	done
 	if [ "${oldval}" = "${newval}" ]; then
 		echo "Field ${key} already set to ${newval}, skipping test."
 		return 1
 	fi
 	return 0
 }

 # Try to forcibly unmount the scratch fs
 __scratch_xfs_fuzz_unmount()
 {
 	while _scratch_unmount 2>/dev/null; do sleep 0.2; done
 }

 # Restore metadata to scratch device prior to field-fuzzing.
 __scratch_xfs_fuzz_mdrestore()
 {
 	test -e "${POPULATE_METADUMP}" || _fail "Need to set POPULATE_METADUMP"

 	__scratch_xfs_fuzz_unmount
 	$XFS_MDRESTORE_PROG "${POPULATE_METADUMP}" "${SCRATCH_DEV}"
 }

 __fuzz_notify() {
 	echo '========================================'
 	echo "$@"
 	echo '========================================'
 	test -w /dev/ttyprintk && echo "$@" >> /dev/ttyprintk
 }

 # Perform the online repair part of a fuzz test.
 __scratch_xfs_fuzz_field_online() {
 	local fuzz_action="$1"

 	# Mount or else we can't do anything online
 	__fuzz_notify "+ Mount filesystem to try online repair"
 	_try_scratch_mount 2>&1
 	res=$?
 	if [ $res -ne 0 ]; then
 		(>&2 echo "${fuzz_action}: mount failed ($res).")
 		return 1
 	fi

 	# Make sure online scrub will catch whatever we fuzzed
 	__fuzz_notify "++ Detect fuzzed field (online)"
 	_scratch_scrub -n -a 1 -e continue 2>&1
 	res=$?
 	test $res -eq 0 && \
 		(>&2 echo "${fuzz_action}: online scrub didn't fail.")

 	# Does the health status report reflect the corruption?
 	if [ $res -ne 0 ]; then
 		__fuzz_notify "++ Detect fuzzed field ill-health report"
 		_check_xfs_health $SCRATCH_MNT 2>&1
 		res=$?
 		test $res -ne 1 && \
 			(>&2 echo "${fuzz_action}: online health check failed ($res).")
 	fi

 	# Try fixing the filesystem online
 	__fuzz_notify "++ Try to repair filesystem (online)"
 	_scratch_scrub 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: online repair failed ($res).")

 	# Online scrub should pass now
 	__fuzz_notify "++ Make sure error is gone (online)"
 	_scratch_scrub -n -a 1 -e continue 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: online re-scrub failed ($res).")

 	__scratch_xfs_fuzz_unmount

 	# Offline scrub should pass now
 	__fuzz_notify "+ Make sure error is gone (offline)"
 	_scratch_xfs_repair -P -n 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: offline re-scrub failed ($res).")

 	return 0
 }

 # Perform the offline repair part of a fuzz test.
 __scratch_xfs_fuzz_field_offline() {
 	local fuzz_action="$1"

 	# Make sure offline scrub will catch whatever we fuzzed
 	__fuzz_notify "+ Detect fuzzed field (offline)"
 	_scratch_xfs_repair -P -n 2>&1
 	res=$?
 	test $res -eq 0 && \
 		(>&2 echo "${fuzz_action}: offline scrub didn't fail.")

 	# Make sure xfs_repair catches at least as many things as the old
 	# xfs_check did.
 	if [ -n "${SCRATCH_XFS_FUZZ_CHECK}" ]; then
 		__fuzz_notify "+ Detect fuzzed field (xfs_check)"
 		_scratch_xfs_check 2>&1
 		res1=$?
 		if [ $res1 -ne 0 ] && [ $res -eq 0 ]; then
 			(>&2 echo "${fuzz_action}: xfs_repair passed but xfs_check failed ($res1).")
 		fi
 	fi

 	# Repair the filesystem offline
 	__fuzz_notify "+ Try to repair the filesystem (offline)"
 	_repair_scratch_fs 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: offline repair failed ($res).")

 	# See if repair finds a clean fs
 	__fuzz_notify "+ Make sure error is gone (offline)"
 	_scratch_xfs_repair -P -n 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: offline re-scrub failed ($res).")

 	return 0
 }

 # Perform the no-repair part of a fuzz test.
 __scratch_xfs_fuzz_field_norepair() {
 	local fuzz_action="$1"

 	# Make sure offline scrub will catch whatever we fuzzed
 	__fuzz_notify "+ Detect fuzzed field (offline)"
 	_scratch_xfs_repair -P -n 2>&1
 	res=$?
 	test $res -eq 0 && \
 		(>&2 echo "${fuzz_action}: offline scrub didn't fail.")

 	# Mount or else we can't do anything in norepair mode
 	__fuzz_notify "+ Mount filesystem to try online scan"
 	_try_scratch_mount 2>&1
 	res=$?
 	if [ $res -ne 0 ]; then
 		(>&2 echo "${fuzz_action}: mount failed ($res).")
 		return 1
 	fi

 	# Make sure online scrub will catch whatever we fuzzed
 	__fuzz_notify "++ Detect fuzzed field (online)"
 	_scratch_scrub -n -a 1 -e continue 2>&1
 	res=$?
 	test $res -eq 0 && \
 		(>&2 echo "${fuzz_action}: online scrub didn't fail.")

 	# Does the health status report reflect the corruption?
 	if [ $res -ne 0 ]; then
 		__fuzz_notify "++ Detect fuzzed field ill-health report"
 		_check_xfs_health $SCRATCH_MNT 2>&1
 		res=$?
 		test $res -ne 1 && \
 			(>&2 echo "${fuzz_action}: online health check failed ($res).")
 	fi

 	__scratch_xfs_fuzz_unmount

 	return 0
 }

 # Perform the online-then-offline repair part of a fuzz test.
 __scratch_xfs_fuzz_field_both() {
 	local fuzz_action="$1"

 	# Make sure offline scrub will catch whatever we fuzzed
 	__fuzz_notify "+ Detect fuzzed field (offline)"
 	_scratch_xfs_repair -P -n 2>&1
 	res=$?
 	test $res -eq 0 && \
 		(>&2 echo "${fuzz_action}: offline scrub didn't fail.")

 	# Mount or else we can't do anything in both repair mode
 	__fuzz_notify "+ Mount filesystem to try both repairs"
 	_try_scratch_mount 2>&1
 	res=$?
 	if [ $res -ne 0 ]; then
 		(>&2 echo "${fuzz_action}: mount failed ($res).")
 	else
 		# Make sure online scrub will catch whatever we fuzzed
 		__fuzz_notify "++ Detect fuzzed field (online)"
 		_scratch_scrub -n -a 1 -e continue 2>&1
 		res=$?
 		test $res -eq 0 && \
 			(>&2 echo "${fuzz_action}: online scrub didn't fail.")

 		# Does the health status report reflect the corruption?
 		if [ $res -ne 0 ]; then
 			__fuzz_notify "++ Detect fuzzed field ill-health report"
 			_check_xfs_health $SCRATCH_MNT 2>&1
 			res=$?
 			test $res -ne 1 && \
 				(>&2 echo "${fuzz_action}: online health check failed ($res).")
 		fi

 		# Try fixing the filesystem online
 		__fuzz_notify "++ Try to repair filesystem (online)"
 		_scratch_scrub 2>&1
 		res=$?
 		test $res -ne 0 && \
 			(>&2 echo "${fuzz_action}: online repair failed ($res).")

 		__scratch_xfs_fuzz_unmount
 	fi

 	# Repair the filesystem offline if online repair failed?
 	if [ $res -ne 0 ]; then
 		__fuzz_notify "+ Try to repair the filesystem (offline)"
 		_repair_scratch_fs 2>&1
 		res=$?
 		test $res -ne 0 && \
 			(>&2 echo "${fuzz_action}: offline repair failed ($res).")
 	fi

 	# See if repair finds a clean fs
 	__fuzz_notify "+ Make sure error is gone (offline)"
 	_scratch_xfs_repair -P -n 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: offline re-scrub failed ($res).")

 	# Mount so that we can see what scrub says after we've fixed the fs
 	__fuzz_notify "+ Re-mount filesystem to re-try online scan"
 	_try_scratch_mount 2>&1
 	res=$?
 	if [ $res -ne 0 ]; then
 		(>&2 echo "${fuzz_action}: mount failed ($res).")
 		return 1
 	fi

 	# Online scrub should pass now
 	__fuzz_notify "++ Make sure error is gone (online)"
 	_scratch_scrub -n -a 1 -e continue 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: online re-scrub failed ($res).")

 	__scratch_xfs_fuzz_unmount

 	return 0
 }

 # Assess the state of the filesystem after a repair strategy has been run by
 # trying to make changes to it.
 _scratch_xfs_fuzz_field_modifyfs() {
 	local fuzz_action="$1"
 	local repair="$2"

 	# Try to mount the filesystem so that we can make changes
 	__fuzz_notify "+ Mount filesystem to make changes"
 	_try_scratch_mount 2>&1
 	res=$?
 	if [ $res -ne 0 ]; then
 		(>&2 echo "${fuzz_action}: pre-mod mount failed ($res).")
 		return $res
 	fi

 	# Try modifying the filesystem again
 	__fuzz_notify "++ Try to write filesystem again"
 	_scratch_fuzz_modify 2>&1

 	# If we didn't repair anything, there's no point in checking further,
 	# the fs is still corrupt.
 	if [ "${repair}" = "none" ]; then
 		__scratch_xfs_fuzz_unmount
 		return 0
 	fi

 	# Run an online check to make sure the fs is still ok, unless we
 	# are running the norepair strategy.
 	__fuzz_notify "+ Re-check the filesystem (online)"
 	_scratch_scrub -n -e continue 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: online post-mod scrub failed ($res).")

 	__scratch_xfs_fuzz_unmount

 	# Run an offline check to make sure the fs is still ok, unless we
 	# are running the norepair strategy.
 	__fuzz_notify "+ Re-check the filesystem (offline)"
 	_scratch_xfs_repair -P -n 2>&1
 	res=$?
 	test $res -ne 0 && \
 		(>&2 echo "${fuzz_action}: offline post-mod scrub failed ($res).")

 	return 0
 }

 # Fuzz one field of some piece of metadata.
 # First arg is the field name
 # Second arg is the fuzz verb (ones, zeroes, random, add, sub...)
 # Third arg is the repair mode (online, offline, both, none)
 __scratch_xfs_fuzz_field_test() {
 	field="$1"
 	fuzzverb="$2"
 	repair="$3"
 	shift; shift; shift

 	# Set the new field value
 	__fuzz_notify "+ Fuzz ${field} = ${fuzzverb}"
 	_scratch_xfs_fuzz_metadata_field "${field}" ${fuzzverb} "$@"
 	res=$?
 	test $res -ne 0 && return

 	# Try to catch the error with whatever repair strategy we picked.
 	# The fs should not be mounted before or after the strategy call.
 	local fuzz_action="${field} = ${fuzzverb}"
 	case "${repair}" in
 	"online")
 		__scratch_xfs_fuzz_field_online "${fuzz_action}"
 		res=$?
 		;;
 	"offline")
 		__scratch_xfs_fuzz_field_offline "${fuzz_action}"
 		res=$?
 		;;
 	"none")
 		__scratch_xfs_fuzz_field_norepair "${fuzz_action}"
 		res=$?
 		;;
 	"both")
 		__scratch_xfs_fuzz_field_both "${fuzz_action}"
 		res=$?
 		;;
 	*)
 		(>&2 echo "unknown repair strategy ${repair}.")
 		res=2
 		;;
 	esac
 	test $res -eq 0 || return $res

 	# See what happens when we modify the fs
 	_scratch_xfs_fuzz_field_modifyfs "${fuzz_action}" "${repair}"
 	return $?
 }

 # Make sure we have all the pieces we need for field fuzzing
 _require_scratch_xfs_fuzz_fields()
 {
 	_require_scratch_nocheck
 	_require_scrub
 	_require_populate_commands
 	_scratch_mkfs_xfs >/dev/null 2>&1
 	_require_xfs_db_command "fuzz"
 }

 # Grab the list of available fuzzing verbs
 _scratch_xfs_list_fuzz_verbs() {
 	if [ -n "${SCRATCH_XFS_LIST_FUZZ_VERBS}" ]; then
 		echo "${SCRATCH_XFS_LIST_FUZZ_VERBS}" | tr '[ ,]' '[\n\n]'
 		return;
 	fi
 	_scratch_xfs_db -x -c 'sb 0' -c 'fuzz' | grep '^Fuzz commands:' | \
 		sed -e 's/[,.]//g' -e 's/Fuzz commands: //g' -e 's/ /\n/g' | \
 		grep -v '^random$'
 }

 # Fuzz some of the fields of some piece of metadata
 # The first argument is an egrep filter for the field names
 # The second argument is the repair mode (online, offline, both)
 # The rest of the arguments are xfs_db commands to locate the metadata.
 #
 # Users can specify the fuzz verbs via SCRATCH_XFS_LIST_FUZZ_VERBS
 # They can specify the fields via SCRATCH_XFS_LIST_METADATA_FIELDS
 _scratch_xfs_fuzz_metadata() {
 	filter="$1"
 	repair="$2"
 	shift; shift

 	fields="$(_scratch_xfs_list_metadata_fields "${filter}" "$@")"
 	verbs="$(_scratch_xfs_list_fuzz_verbs)"
 	echo "Fields we propose to fuzz under: $@"
 	echo $(echo "${fields}")
 	echo "Verbs we propose to fuzz with:"
 	echo $(echo "${verbs}")

 	# Always capture full core dumps from crashing tools
 	ulimit -c unlimited

 	echo "${fields}" | while read field; do
 		echo "${verbs}" | while read fuzzverb; do
 			__scratch_xfs_fuzz_mdrestore
 			__scratch_xfs_fuzz_field_test "${field}" "${fuzzverb}" "${repair}" "$@"
 		done
 	done
 }
	##/bin/bash
	# SPDX-License-Identifier: GPL-2.0+
	# Copyright (c) 2017 Oracle. All Rights Reserved.
	#
	# Routines for fuzzing and scrubbing a filesystem.

	# Modify various files after a fuzzing operation
	_scratch_fuzz_modify() {
	echo "+++ stressing filesystem"
	mkdir -p $SCRATCH_MNT/data
	_xfs_force_bdev data $SCRATCH_MNT/data
	$FSSTRESS_PROG -n $((TIME_FACTOR * 10000)) -p $((LOAD_FACTOR * 4)) -d $SCRATCH_MNT/data

	is_rt="$($XFS_INFO_PROG "$SCRATCH_MNT" \| grep -c 'rtextents=[1-9]')"
	if [ $is_rt -gt 0 ]; then
	mkdir -p $SCRATCH_MNT/rt
	_xfs_force_bdev realtime $SCRATCH_MNT/data
	$FSSTRESS_PROG -n $((TIME_FACTOR * 10000)) -p $((LOAD_FACTOR * 4)) -d $SCRATCH_MNT/rt
	else
	echo "+++ xfs realtime not configured"
	fi
	}

	# Try to access files after fuzzing
	_scratch_fuzz_test() {
	echo "+++ ls -laR" >> $seqres.full
	ls -laR "${SCRATCH_MNT}/test.1/" >/dev/null 2>&1

	echo "+++ cat files" >> $seqres.full
	(find "${SCRATCH_MNT}/test.1/" -type f -size -1048576k -print0 \| xargs -0 cat) >/dev/null 2>&1
	}

	# Do we have an online scrub program?
	_require_scrub() {
	case "${FSTYP}" in
	"xfs")
	test -x "$XFS_SCRUB_PROG" \|\| _notrun "xfs_scrub not found"
	;;
	*)
	_notrun "No online scrub program for ${FSTYP}."
	;;
	esac
	}

	# Scrub the scratch filesystem metadata (online)
	_scratch_scrub() {
	case "${FSTYP}" in
	"xfs")
	$XFS_SCRUB_PROG -d -T -v "$@" $SCRATCH_MNT
	;;
	*)
	_fail "No online scrub program for ${FSTYP}."
	;;
	esac
	}

	# Expand indexed keys (i.e. arrays) into a long format so that we can filter
	# the array indices individually, and pass regular keys right through.
	#
	# For example, "u3.bmx[0-1] = [foo,bar]" is exploded into:
	# u3.bmx[0] = [foo,bar]
	# u3.bmx[1] = [foo,bar]
	#
	# Note that we restrict array indices to [0-9] to reduce fuzz runtime. The
	# minimum and maximum array indices can be changed by setting the variables
	# SCRATCH_XFS_{MIN,MAX}_ARRAY_IDX.
	#
	# Also filter padding fields.
	__explode_xfs_db_fields() {
	local min_idx="${SCRATCH_XFS_MIN_ARRAY_IDX}"
	local max_idx="${SCRATCH_XFS_MAX_ARRAY_IDX}"

	test -z "${min_idx}" && min_idx=0
	test -z "${max_idx}" && max_idx=9
	test "${max_idx}" = "none" && max_idx=99999

	grep ' = ' \| \
	sed -e 's/^\([.a-zA-Z0-9_]\)\[\([0-9]\)-\([0-9]\)\]\(.\) = \(.*\)$/\1[%d]\4 \2 \3 = \5/g' \
	-e 's/^\([.a-zA-Z0-9_]\)\[\([0-9]\)\]\(.\) = \(.\)$/\1[%d]\3 \2 \2 = \4/g' \| \
	while read name col1 col2 rest; do
	if [[ "${name}" == pad ]]; then
	continue
	fi

	if [ "${col1}" = "=" ]; then
	echo "${name} ${col1} ${col2} ${rest}"
	continue
	fi

	test "${min_idx}" -gt "${col1}" && col1="${min_idx}"
	test "${max_idx}" -lt "${col2}" && col2="${max_idx}"

	seq "${col1}" "${col2}" \| while read idx; do
	printf "${name} %s\n" "${idx}" "${rest}"
	done
	done
	}

	# Filter the xfs_db print command's field debug information
	# into field name and type.
	__filter_xfs_db_print_fields() {
	filter="$1"
	if [ -z "${filter}" ] \|\| [ "${filter}" = "nofilter" ]; then
	filter='^'
	fi
	__explode_xfs_db_fields \| while read key equals value; do
	fuzzkey="$(echo "${key}")"
	if [ -z "${fuzzkey}" ]; then
	continue
	elif [[ "${value}" == "["* ]]; then
	echo "${value}" \| sed -e 's/^.//g' -e 's/.$//g' -e 's/,/\n/g' \| while read subfield; do
	echo "${fuzzkey}.${subfield}"
	done
	else
	echo "${fuzzkey}"
	fi
	done \| egrep "${filter}" \| egrep -v '(^\|\.)(sec\|nsec\|lsn)$'
	}

	# Navigate to some part of the filesystem and print the field info.
	# The first argument is an egrep filter for the fields
	# The rest of the arguments are xfs_db commands to locate the metadata.
	_scratch_xfs_list_metadata_fields() {
	filter="$1"
	shift
	if [ -n "${SCRATCH_XFS_LIST_METADATA_FIELDS}" ]; then
	echo "${SCRATCH_XFS_LIST_METADATA_FIELDS}" \| tr '[ ,]' '[\n\n]'
	return;
	fi

	local cmds=()
	for arg in "$@"; do
	cmds+=("-c" "${arg}")
	done
	_scratch_xfs_db "${cmds[@]}" -c print \| __filter_xfs_db_print_fields "${filter}"
	}

	# Fuzz a metadata field
	# The first arg is the field name
	# The second arg is the xfs_db fuzz verb
	# The rest of the arguments are xfs_db commands to find the metadata.
	_scratch_xfs_fuzz_metadata_field() {
	key="$1"
	value="$2"
	shift; shift

	if [[ "${key}" == *crc ]]; then
	fuzz_arg="-c"
	else
	fuzz_arg="-d"
	fi
	oldval="$(_scratch_xfs_get_metadata_field "${key}" "$@")"

	local cmds=()
	for arg in "$@"; do
	cmds+=("-c" "${arg}")
	done
	while true; do
	_scratch_xfs_db -x "${cmds[@]}" -c "fuzz ${fuzz_arg} ${key} ${value}"
	echo
	newval="$(_scratch_xfs_get_metadata_field "${key}" "$@" 2> /dev/null)"
	if [ "${key}" != "random" ] \|\| [ "${oldval}" != "${newval}" ]; then
	break;
	fi
	done
	if [ "${oldval}" = "${newval}" ]; then
	echo "Field ${key} already set to ${newval}, skipping test."
	return 1
	fi
	return 0
	}

	# Try to forcibly unmount the scratch fs
	__scratch_xfs_fuzz_unmount()
	{
	while _scratch_unmount 2>/dev/null; do sleep 0.2; done
	}

	# Restore metadata to scratch device prior to field-fuzzing.
	__scratch_xfs_fuzz_mdrestore()
	{
	test -e "${POPULATE_METADUMP}" \|\| _fail "Need to set POPULATE_METADUMP"

	__scratch_xfs_fuzz_unmount
	$XFS_MDRESTORE_PROG "${POPULATE_METADUMP}" "${SCRATCH_DEV}"
	}

	__fuzz_notify() {
	echo '========================================'
	echo "$@"
	echo '========================================'
	test -w /dev/ttyprintk && echo "$@" >> /dev/ttyprintk
	}

	# Perform the online repair part of a fuzz test.
	__scratch_xfs_fuzz_field_online() {
	local fuzz_action="$1"

	# Mount or else we can't do anything online
	__fuzz_notify "+ Mount filesystem to try online repair"
	_try_scratch_mount 2>&1
	res=$?
	if [ $res -ne 0 ]; then
	(>&2 echo "${fuzz_action}: mount failed ($res).")
	return 1
	fi

	# Make sure online scrub will catch whatever we fuzzed
	__fuzz_notify "++ Detect fuzzed field (online)"
	_scratch_scrub -n -a 1 -e continue 2>&1
	res=$?
	test $res -eq 0 && \
	(>&2 echo "${fuzz_action}: online scrub didn't fail.")

	# Does the health status report reflect the corruption?
	if [ $res -ne 0 ]; then
	__fuzz_notify "++ Detect fuzzed field ill-health report"
	_check_xfs_health $SCRATCH_MNT 2>&1
	res=$?
	test $res -ne 1 && \
	(>&2 echo "${fuzz_action}: online health check failed ($res).")
	fi

	# Try fixing the filesystem online
	__fuzz_notify "++ Try to repair filesystem (online)"
	_scratch_scrub 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: online repair failed ($res).")

	# Online scrub should pass now
	__fuzz_notify "++ Make sure error is gone (online)"
	_scratch_scrub -n -a 1 -e continue 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: online re-scrub failed ($res).")

	__scratch_xfs_fuzz_unmount

	# Offline scrub should pass now
	__fuzz_notify "+ Make sure error is gone (offline)"
	_scratch_xfs_repair -P -n 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: offline re-scrub failed ($res).")

	return 0
	}

	# Perform the offline repair part of a fuzz test.
	__scratch_xfs_fuzz_field_offline() {
	local fuzz_action="$1"

	# Make sure offline scrub will catch whatever we fuzzed
	__fuzz_notify "+ Detect fuzzed field (offline)"
	_scratch_xfs_repair -P -n 2>&1
	res=$?
	test $res -eq 0 && \
	(>&2 echo "${fuzz_action}: offline scrub didn't fail.")

	# Make sure xfs_repair catches at least as many things as the old
	# xfs_check did.
	if [ -n "${SCRATCH_XFS_FUZZ_CHECK}" ]; then
	__fuzz_notify "+ Detect fuzzed field (xfs_check)"
	_scratch_xfs_check 2>&1
	res1=$?
	if [ $res1 -ne 0 ] && [ $res -eq 0 ]; then
	(>&2 echo "${fuzz_action}: xfs_repair passed but xfs_check failed ($res1).")
	fi
	fi

	# Repair the filesystem offline
	__fuzz_notify "+ Try to repair the filesystem (offline)"
	_repair_scratch_fs 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: offline repair failed ($res).")

	# See if repair finds a clean fs
	__fuzz_notify "+ Make sure error is gone (offline)"
	_scratch_xfs_repair -P -n 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: offline re-scrub failed ($res).")

	return 0
	}

	# Perform the no-repair part of a fuzz test.
	__scratch_xfs_fuzz_field_norepair() {
	local fuzz_action="$1"

	# Make sure offline scrub will catch whatever we fuzzed
	__fuzz_notify "+ Detect fuzzed field (offline)"
	_scratch_xfs_repair -P -n 2>&1
	res=$?
	test $res -eq 0 && \
	(>&2 echo "${fuzz_action}: offline scrub didn't fail.")

	# Mount or else we can't do anything in norepair mode
	__fuzz_notify "+ Mount filesystem to try online scan"
	_try_scratch_mount 2>&1
	res=$?
	if [ $res -ne 0 ]; then
	(>&2 echo "${fuzz_action}: mount failed ($res).")
	return 1
	fi

	# Make sure online scrub will catch whatever we fuzzed
	__fuzz_notify "++ Detect fuzzed field (online)"
	_scratch_scrub -n -a 1 -e continue 2>&1
	res=$?
	test $res -eq 0 && \
	(>&2 echo "${fuzz_action}: online scrub didn't fail.")

	# Does the health status report reflect the corruption?
	if [ $res -ne 0 ]; then
	__fuzz_notify "++ Detect fuzzed field ill-health report"
	_check_xfs_health $SCRATCH_MNT 2>&1
	res=$?
	test $res -ne 1 && \
	(>&2 echo "${fuzz_action}: online health check failed ($res).")
	fi

	__scratch_xfs_fuzz_unmount

	return 0
	}

	# Perform the online-then-offline repair part of a fuzz test.
	__scratch_xfs_fuzz_field_both() {
	local fuzz_action="$1"

	# Make sure offline scrub will catch whatever we fuzzed
	__fuzz_notify "+ Detect fuzzed field (offline)"
	_scratch_xfs_repair -P -n 2>&1
	res=$?
	test $res -eq 0 && \
	(>&2 echo "${fuzz_action}: offline scrub didn't fail.")

	# Mount or else we can't do anything in both repair mode
	__fuzz_notify "+ Mount filesystem to try both repairs"
	_try_scratch_mount 2>&1
	res=$?
	if [ $res -ne 0 ]; then
	(>&2 echo "${fuzz_action}: mount failed ($res).")
	else
	# Make sure online scrub will catch whatever we fuzzed
	__fuzz_notify "++ Detect fuzzed field (online)"
	_scratch_scrub -n -a 1 -e continue 2>&1
	res=$?
	test $res -eq 0 && \
	(>&2 echo "${fuzz_action}: online scrub didn't fail.")

	# Does the health status report reflect the corruption?
	if [ $res -ne 0 ]; then
	__fuzz_notify "++ Detect fuzzed field ill-health report"
	_check_xfs_health $SCRATCH_MNT 2>&1
	res=$?
	test $res -ne 1 && \
	(>&2 echo "${fuzz_action}: online health check failed ($res).")
	fi

	# Try fixing the filesystem online
	__fuzz_notify "++ Try to repair filesystem (online)"
	_scratch_scrub 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: online repair failed ($res).")

	__scratch_xfs_fuzz_unmount
	fi

	# Repair the filesystem offline if online repair failed?
	if [ $res -ne 0 ]; then
	__fuzz_notify "+ Try to repair the filesystem (offline)"
	_repair_scratch_fs 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: offline repair failed ($res).")
	fi

	# See if repair finds a clean fs
	__fuzz_notify "+ Make sure error is gone (offline)"
	_scratch_xfs_repair -P -n 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: offline re-scrub failed ($res).")

	# Mount so that we can see what scrub says after we've fixed the fs
	__fuzz_notify "+ Re-mount filesystem to re-try online scan"
	_try_scratch_mount 2>&1
	res=$?
	if [ $res -ne 0 ]; then
	(>&2 echo "${fuzz_action}: mount failed ($res).")
	return 1
	fi

	# Online scrub should pass now
	__fuzz_notify "++ Make sure error is gone (online)"
	_scratch_scrub -n -a 1 -e continue 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: online re-scrub failed ($res).")

	__scratch_xfs_fuzz_unmount

	return 0
	}

	# Assess the state of the filesystem after a repair strategy has been run by
	# trying to make changes to it.
	_scratch_xfs_fuzz_field_modifyfs() {
	local fuzz_action="$1"
	local repair="$2"

	# Try to mount the filesystem so that we can make changes
	__fuzz_notify "+ Mount filesystem to make changes"
	_try_scratch_mount 2>&1
	res=$?
	if [ $res -ne 0 ]; then
	(>&2 echo "${fuzz_action}: pre-mod mount failed ($res).")
	return $res
	fi

	# Try modifying the filesystem again
	__fuzz_notify "++ Try to write filesystem again"
	_scratch_fuzz_modify 2>&1

	# If we didn't repair anything, there's no point in checking further,
	# the fs is still corrupt.
	if [ "${repair}" = "none" ]; then
	__scratch_xfs_fuzz_unmount
	return 0
	fi

	# Run an online check to make sure the fs is still ok, unless we
	# are running the norepair strategy.
	__fuzz_notify "+ Re-check the filesystem (online)"
	_scratch_scrub -n -e continue 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: online post-mod scrub failed ($res).")

	__scratch_xfs_fuzz_unmount

	# Run an offline check to make sure the fs is still ok, unless we
	# are running the norepair strategy.
	__fuzz_notify "+ Re-check the filesystem (offline)"
	_scratch_xfs_repair -P -n 2>&1
	res=$?
	test $res -ne 0 && \
	(>&2 echo "${fuzz_action}: offline post-mod scrub failed ($res).")

	return 0
	}

	# Fuzz one field of some piece of metadata.
	# First arg is the field name
	# Second arg is the fuzz verb (ones, zeroes, random, add, sub...)
	# Third arg is the repair mode (online, offline, both, none)
	__scratch_xfs_fuzz_field_test() {
	field="$1"
	fuzzverb="$2"
	repair="$3"
	shift; shift; shift

	# Set the new field value
	__fuzz_notify "+ Fuzz ${field} = ${fuzzverb}"
	_scratch_xfs_fuzz_metadata_field "${field}" ${fuzzverb} "$@"
	res=$?
	test $res -ne 0 && return

	# Try to catch the error with whatever repair strategy we picked.
	# The fs should not be mounted before or after the strategy call.
	local fuzz_action="${field} = ${fuzzverb}"
	case "${repair}" in
	"online")
	__scratch_xfs_fuzz_field_online "${fuzz_action}"
	res=$?
	;;
	"offline")
	__scratch_xfs_fuzz_field_offline "${fuzz_action}"
	res=$?
	;;
	"none")
	__scratch_xfs_fuzz_field_norepair "${fuzz_action}"
	res=$?
	;;
	"both")
	__scratch_xfs_fuzz_field_both "${fuzz_action}"
	res=$?
	;;
	*)
	(>&2 echo "unknown repair strategy ${repair}.")
	res=2
	;;
	esac
	test $res -eq 0 \|\| return $res

	# See what happens when we modify the fs
	_scratch_xfs_fuzz_field_modifyfs "${fuzz_action}" "${repair}"
	return $?
	}

	# Make sure we have all the pieces we need for field fuzzing
	_require_scratch_xfs_fuzz_fields()
	{
	_require_scratch_nocheck
	_require_scrub
	_require_populate_commands
	_scratch_mkfs_xfs >/dev/null 2>&1
	_require_xfs_db_command "fuzz"
	}

	# Grab the list of available fuzzing verbs
	_scratch_xfs_list_fuzz_verbs() {
	if [ -n "${SCRATCH_XFS_LIST_FUZZ_VERBS}" ]; then
	echo "${SCRATCH_XFS_LIST_FUZZ_VERBS}" \| tr '[ ,]' '[\n\n]'
	return;
	fi
	_scratch_xfs_db -x -c 'sb 0' -c 'fuzz' \| grep '^Fuzz commands:' \| \
	sed -e 's/[,.]//g' -e 's/Fuzz commands: //g' -e 's/ /\n/g' \| \
	grep -v '^random$'
	}

	# Fuzz some of the fields of some piece of metadata
	# The first argument is an egrep filter for the field names
	# The second argument is the repair mode (online, offline, both)
	# The rest of the arguments are xfs_db commands to locate the metadata.
	#
	# Users can specify the fuzz verbs via SCRATCH_XFS_LIST_FUZZ_VERBS
	# They can specify the fields via SCRATCH_XFS_LIST_METADATA_FIELDS
	_scratch_xfs_fuzz_metadata() {
	filter="$1"
	repair="$2"
	shift; shift

	fields="$(_scratch_xfs_list_metadata_fields "${filter}" "$@")"
	verbs="$(_scratch_xfs_list_fuzz_verbs)"
	echo "Fields we propose to fuzz under: $@"
	echo $(echo "${fields}")
	echo "Verbs we propose to fuzz with:"
	echo $(echo "${verbs}")

	# Always capture full core dumps from crashing tools
	ulimit -c unlimited

	echo "${fields}" \| while read field; do
	echo "${verbs}" \| while read fuzzverb; do
	__scratch_xfs_fuzz_mdrestore
	__scratch_xfs_fuzz_field_test "${field}" "${fuzzverb}" "${repair}" "$@"
	done
	done
	}