scrub/inodes.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2018-2024 Oracle.  All Rights Reserved.
 * Author: Darrick J. Wong <djwong@kernel.org>
 */
#include "xfs.h"
#include <stdint.h>
#include <stdlib.h>
#include <pthread.h>
#include <sys/statvfs.h>
#include "platform_defs.h"
#include "xfs_arch.h"
#include "handle.h"
#include "libfrog/paths.h"
#include "libfrog/workqueue.h"
#include "xfs_scrub.h"
#include "common.h"
#include "inodes.h"
#include "descr.h"
#include "libfrog/fsgeom.h"
#include "libfrog/bulkstat.h"
#include "libfrog/handle_priv.h"
#include "bitops.h"
#include "libfrog/bitmask.h"

/*
 * Iterate a range of inodes.
 *
 * This is a little more involved than repeatedly asking BULKSTAT for a
 * buffer's worth of stat data for some number of inodes.  We want to scan as
 * many of the inodes that the inobt thinks there are, so we use the INUMBERS
 * ioctl to walk all the inobt records in the filesystem and spawn a worker to
 * bulkstat and iterate.  The worker starts with an inumbers record that can
 * look like this:
 *
 * {startino = S, allocmask = 0b11011}
 *
 * Given a starting inumber S and count C=64, bulkstat will return a sorted
 * array of stat information.  The bs_ino of those array elements can look like
 * any of the following:
 *
 * 0. [S, S+1, S+3, S+4]
 * 1. [S+e, S+e+1, S+e+3, S+e+4, S+e+C+1...], where e >= 0
 * 2. [S+e+n], where n >= 0
 * 3. []
 * 4. [], errno == EFSCORRUPTED
 *
 * We know that bulkstat scanned the entire inode range between S and bs_ino of
 * the last array element, even though it only fills out an array element for
 * allocated inodes.  Therefore, we can say in cases 0-2 that S was filled,
 * even if there is no bstat[] record for S.  In turn, we can create a bitmask
 * of inodes that we have seen, and set bits 0 through (bstat[-1].bs_ino - S),
 * being careful not to set any bits past S+C.
 *
 * In case (0) we find that seen mask matches the inumber record
 * exactly, so the caller can walk the stat records and move on.  In case (1)
 * this is also true, but we must be careful to reduce the array length to
 * avoid scanning inodes that are not in the inumber chunk.  In case (3) we
 * conclude that there were no inodes left to scan and terminate.
 *
 * In case (2) and (4) we don't know why bulkstat returned fewer than C
 * elements.  We might have found the end of the filesystem, or the kernel
 * might have found a corrupt inode and stopped.  This we must investigate by
 * trying to fill out the rest of the bstat array starting with the next
 * inumber after the last bstat array element filled, and continuing until S'
 * is beyond S0 + C, or the array is full.  Each time we succeed in loading
 * new records, the kernel increases S' for us; if instead we encounter case
 * (4), we can increment S' ourselves.
 *
 * Inodes that are set in the allocmask but not set in the seen mask are the
 * corrupt inodes.  For each of these cases, we try to populate the bulkstat
 * array one inode at a time.  If the kernel returns a matching record we can
 * use it; if instead we receive an error, we synthesize enough of a record
 * to be able to run online scrub by handle.
 *
 * If the iteration function returns ESTALE, that means that the inode has
 * been deleted and possibly recreated since the BULKSTAT call.  We wil
 * refresh the stat information and try again up to 30 times before reporting
 * the staleness as an error.
 */

/*
 * Return the inumber of the highest inode in the bulkstat data, assuming the
 * records are sorted in inumber order.
 */
static inline uint64_t last_bstat_ino(const struct xfs_bulkstat_req *b)
{
	return b->hdr.ocount ? b->bulkstat[b->hdr.ocount - 1].bs_ino : 0;
}

/*
 * Deduce the bitmask of the inodes in inums that were seen by bulkstat.  If
 * the inode is present in the bstat array this is trivially true; or if it is
 * not in the array but higher inumbers are present, then it was freed.
 */
static __u64
seen_mask_from_bulkstat(
	const struct xfs_inumbers	*inums,
	__u64				breq_startino,
	const struct xfs_bulkstat_req	*breq)
{
	const __u64			limit_ino =
		inums->xi_startino + LIBFROG_BULKSTAT_CHUNKSIZE;
	const __u64			last = last_bstat_ino(breq);
	__u64				ret = 0;
	int				i, maxi;

	/* Ignore the bulkstat results if they don't cover inumbers */
	if (breq_startino > limit_ino || last < inums->xi_startino)
		return 0;

	maxi = min(LIBFROG_BULKSTAT_CHUNKSIZE, last - inums->xi_startino + 1);
	for (i = breq_startino - inums->xi_startino; i < maxi; i++)
		ret |= 1ULL << i;

	return ret;
}

/*
 * Try to fill the rest of orig_breq with bulkstat data by re-running bulkstat
 * with increasing start_ino until we either hit the end of the inumbers info
 * or fill up the bstat array with something.  Returns a bitmask of the inodes
 * within inums that were filled by the bulkstat requests.
 */
static __u64
bulkstat_the_rest(
	struct scrub_ctx		*ctx,
	const struct xfs_inumbers	*inums,
	struct xfs_bulkstat_req		*orig_breq,
	int				orig_error)
{
	struct xfs_bulkstat_req		*new_breq;
	struct xfs_bulkstat		*old_bstat =
		&orig_breq->bulkstat[orig_breq->hdr.ocount];
	const __u64			limit_ino =
		inums->xi_startino + LIBFROG_BULKSTAT_CHUNKSIZE;
	__u64				start_ino = orig_breq->hdr.ino;
	__u64				seen_mask = 0;
	int				error;

	assert(orig_breq->hdr.ocount < orig_breq->hdr.icount);

	/*
	 * If the first bulkstat returned a corruption error, that means
	 * start_ino is corrupt.  Restart instead at the next inumber.
	 */
	if (orig_error == EFSCORRUPTED)
		start_ino++;
	if (start_ino >= limit_ino)
		return 0;

	error = -xfrog_bulkstat_alloc_req(
			orig_breq->hdr.icount - orig_breq->hdr.ocount,
			start_ino, &new_breq);
	if (error)
		return error;
	new_breq->hdr.flags = orig_breq->hdr.flags;

	do {
		/*
		 * Fill the new bulkstat request with stat data starting at
		 * start_ino.
		 */
		error = -xfrog_bulkstat(&ctx->mnt, new_breq);
		if (error == EFSCORRUPTED) {
			/*
			 * start_ino is corrupt, increment and try the next
			 * inode.
			 */
			start_ino++;
			new_breq->hdr.ino = start_ino;
			continue;
		}
		if (error) {
			/*
			 * Any other error means the caller falls back to
			 * single stepping.
			 */
			break;
		}
		if (new_breq->hdr.ocount == 0)
			break;

		/* Copy new results to the original bstat buffer */
		memcpy(old_bstat, new_breq->bulkstat,
		       new_breq->hdr.ocount * sizeof(struct xfs_bulkstat));
		orig_breq->hdr.ocount += new_breq->hdr.ocount;
		old_bstat += new_breq->hdr.ocount;
		seen_mask |= seen_mask_from_bulkstat(inums, start_ino,
					new_breq);

		new_breq->hdr.icount -= new_breq->hdr.ocount;
		start_ino = new_breq->hdr.ino;
	} while (new_breq->hdr.icount > 0 && new_breq->hdr.ino < limit_ino);

	free(new_breq);
	return seen_mask;
}

#define cmp_int(l, r)		((l > r) - (l < r))

/* Compare two bulkstat records by inumber. */
static int
compare_bstat(
	const void		*a,
	const void		*b)
{
	const struct xfs_bulkstat *ba = a;
	const struct xfs_bulkstat *bb = b;

	return cmp_int(ba->bs_ino, bb->bs_ino);
}

/*
 * Walk the xi_allocmask looking for set bits that aren't present in
 * the fill mask.  For each such inode, fill the entries at the end of
 * the array with stat information one at a time, synthesizing them if
 * necessary.  At this point, (xi_allocmask & ~seen_mask) should be the
 * corrupt inodes.
 */
static void
bulkstat_single_step(
	struct scrub_ctx		*ctx,
	const struct xfs_inumbers	*inumbers,
	uint64_t			seen_mask,
	struct xfs_bulkstat_req		*breq)
{
	struct xfs_bulkstat		*bs = NULL;
	int				i;
	int				error;

	for (i = 0; i < LIBFROG_BULKSTAT_CHUNKSIZE; i++) {
		/*
		 * Don't single-step if inumbers said it wasn't allocated or
		 * bulkstat actually filled it.
		 */
		if (!(inumbers->xi_allocmask & (1ULL << i)))
			continue;
		if (seen_mask & (1ULL << i))
			continue;

		assert(breq->hdr.ocount < LIBFROG_BULKSTAT_CHUNKSIZE);

		if (!bs)
			bs = &breq->bulkstat[breq->hdr.ocount];

		/*
		 * Didn't get desired stat data and we've hit the end of the
		 * returned data.  We can't distinguish between the inode being
		 * freed vs. the inode being to corrupt to load, so try a
		 * bulkstat single to see if we can load the inode.
		 */
		error = -xfrog_bulkstat_single(&ctx->mnt,
				inumbers->xi_startino + i, breq->hdr.flags, bs);
		switch (error) {
		case ENOENT:
			/*
			 * This inode wasn't found, and no results were
			 * returned.  We've likely hit the end of the
			 * filesystem, but we'll move on to the next inode in
			 * the mask for the sake of caution.
			 */
			continue;
		case 0:
			/*
			 * If a result was returned but it wasn't the inode
			 * we were looking for, then the missing inode was
			 * freed.  Move on to the next inode in the mask.
			 */
			if (bs->bs_ino != inumbers->xi_startino + i)
				continue;
			break;
		default:
			/*
			 * Some error happened.  Synthesize a bulkstat record
			 * so that phase3 can try to see if there's a corrupt
			 * inode that needs repairing.
			 */
			memset(bs, 0, sizeof(struct xfs_bulkstat));
			bs->bs_ino = inumbers->xi_startino + i;
			bs->bs_blksize = ctx->mnt_sv.f_frsize;
			break;
		}

		breq->hdr.ocount++;
		bs++;
	}

	/* If we added any entries, re-sort the array. */
	if (bs)
		qsort(breq->bulkstat, breq->hdr.ocount,
				sizeof(struct xfs_bulkstat), compare_bstat);
}

/* Return the inumber of the highest allocated inode in the inumbers data. */
static inline uint64_t last_allocmask_ino(const struct xfs_inumbers *i)
{
	return i->xi_startino + xfrog_highbit64(i->xi_allocmask);
}

/*
 * Run bulkstat on an entire inode allocation group, then check that we got
 * exactly the inodes we expected.  If not, load them one at a time (or fake
 * it) into the bulkstat data.
 */
static void
bulkstat_for_inumbers(
	struct scrub_ctx		*ctx,
	const struct xfs_inumbers	*inumbers,
	struct xfs_bulkstat_req		*breq)
{
	const uint64_t			limit_ino =
		inumbers->xi_startino + LIBFROG_BULKSTAT_CHUNKSIZE;
	uint64_t			seen_mask = 0;
	int				i;
	int				error;

	assert(inumbers->xi_allocmask != 0);

	/* First we try regular bulkstat, for speed. */
	breq->hdr.ino = inumbers->xi_startino;
	error = -xfrog_bulkstat(&ctx->mnt, breq);
	if (!error) {
		if (!breq->hdr.ocount)
			return;
		seen_mask |= seen_mask_from_bulkstat(inumbers,
					inumbers->xi_startino, breq);
	}

	/*
	 * If the last allocated inode as reported by inumbers is higher than
	 * the last inode reported by bulkstat, two things could have happened.
	 * Either all the inodes at the high end of the cluster were freed
	 * since the inumbers call; or bulkstat encountered a corrupt inode and
	 * returned early.  Try to bulkstat the rest of the array.
	 */
	if (last_allocmask_ino(inumbers) > last_bstat_ino(breq))
		seen_mask |= bulkstat_the_rest(ctx, inumbers, breq, error);

	/*
	 * Bulkstat might return inodes beyond xi_startino + CHUNKSIZE.  Reduce
	 * ocount to ignore inodes not described by the inumbers record.
	 */
	for (i = breq->hdr.ocount - 1; i >= 0; i--) {
		if (breq->bulkstat[i].bs_ino < limit_ino)
			break;
		breq->hdr.ocount--;
	}

	/*
	 * Fill in any missing inodes that are mentioned in the alloc mask but
	 * weren't previously seen by bulkstat.  These are the corrupt inodes.
	 */
	bulkstat_single_step(ctx, inumbers, seen_mask, breq);
}

/* BULKSTAT wrapper routines. */
struct scan_inodes {
	struct workqueue	wq_bulkstat;
	scrub_inode_iter_fn	fn;
	void			*arg;
	unsigned int		nr_threads;
	bool			aborted;
};

/*
 * A single unit of inode scan work.  This contains a pointer to the parent
 * information, followed by an INUMBERS request structure, followed by a
 * BULKSTAT request structure.  The last two are VLAs, so we can't represent
 * them here.
 */
struct scan_ichunk {
	struct scan_inodes	*si;
};

static inline struct xfs_inumbers_req *
ichunk_to_inumbers(
	struct scan_ichunk	*ichunk)
{
	char			*p = (char *)ichunk;

	return (struct xfs_inumbers_req *)(p + sizeof(struct scan_ichunk));
}

static inline struct xfs_bulkstat_req *
ichunk_to_bulkstat(
	struct scan_ichunk	*ichunk)
{
	char			*p = (char *)ichunk_to_inumbers(ichunk);

	return (struct xfs_bulkstat_req *)(p + XFS_INUMBERS_REQ_SIZE(1));
}

static inline int
alloc_ichunk(
	struct scrub_ctx	*ctx,
	struct scan_inodes	*si,
	uint32_t		agno,
	uint64_t		startino,
	struct scan_ichunk	**ichunkp)
{
	struct scan_ichunk	*ichunk;
	struct xfs_inumbers_req	*ireq;
	struct xfs_bulkstat_req	*breq;

	ichunk = calloc(1, sizeof(struct scan_ichunk) +
			   XFS_INUMBERS_REQ_SIZE(1) +
			   XFS_BULKSTAT_REQ_SIZE(LIBFROG_BULKSTAT_CHUNKSIZE));
	if (!ichunk)
		return -errno;

	ichunk->si = si;

	ireq = ichunk_to_inumbers(ichunk);
	ireq->hdr.icount = 1;
	ireq->hdr.ino = startino;
	ireq->hdr.agno = agno;
	ireq->hdr.flags |= XFS_BULK_IREQ_AGNO;

	breq = ichunk_to_bulkstat(ichunk);
	breq->hdr.icount = LIBFROG_BULKSTAT_CHUNKSIZE;

	/* Scan the metadata directory tree too. */
	if (ctx->mnt.fsgeom.flags & XFS_FSOP_GEOM_FLAGS_METADIR)
		breq->hdr.flags |= XFS_BULK_IREQ_METADIR;

	*ichunkp = ichunk;
	return 0;
}

static int
render_ino_from_bulkstat(
	struct scrub_ctx	*ctx,
	char			*buf,
	size_t			buflen,
	void			*data)
{
	struct xfs_bulkstat	*bstat = data;

	return scrub_render_ino_descr(ctx, buf, buflen, bstat->bs_ino,
			bstat->bs_gen, NULL);
}

static int
render_inumbers_from_agno(
	struct scrub_ctx	*ctx,
	char			*buf,
	size_t			buflen,
	void			*data)
{
	xfs_agnumber_t		*agno = data;

	return snprintf(buf, buflen, _("dev %d:%d AG %u inodes"),
				major(ctx->fsinfo.fs_datadev),
				minor(ctx->fsinfo.fs_datadev),
				*agno);
}

/*
 * Call BULKSTAT for information on a single chunk's worth of inodes and call
 * our iterator function.  We'll try to fill the bulkstat information in
 * batches, but we also can detect iget failures.
 */
static void
scan_ag_bulkstat(
	struct workqueue	*wq,
	xfs_agnumber_t		agno,
	void			*arg)
{
	struct xfs_handle	handle;
	struct scrub_ctx	*ctx = (struct scrub_ctx *)wq->wq_ctx;
	struct scan_ichunk	*ichunk = arg;
	struct xfs_inumbers_req	*ireq = ichunk_to_inumbers(ichunk);
	struct xfs_bulkstat_req	*breq = ichunk_to_bulkstat(ichunk);
	struct scan_inodes	*si = ichunk->si;
	struct xfs_bulkstat	*bs = &breq->bulkstat[0];
	struct xfs_inumbers	*inumbers = &ireq->inumbers[0];
	uint64_t		last_ino = 0;
	int			i;
	int			error;
	int			stale_count = 0;
	DEFINE_DESCR(dsc_bulkstat, ctx, render_ino_from_bulkstat);
	DEFINE_DESCR(dsc_inumbers, ctx, render_inumbers_from_agno);

	descr_set(&dsc_inumbers, &agno);
	handle_from_fshandle(&handle, ctx->fshandle, ctx->fshandle_len);
retry:
	bulkstat_for_inumbers(ctx, inumbers, breq);

	/* Iterate all the inodes. */
	for (i = 0; !si->aborted && i < breq->hdr.ocount; i++, bs++) {
		uint64_t	scan_ino = bs->bs_ino;

		/* ensure forward progress if we retried */
		if (scan_ino < last_ino)
			continue;

		descr_set(&dsc_bulkstat, bs);
		handle_from_bulkstat(&handle, bs);
		error = si->fn(ctx, &handle, bs, si->arg);
		switch (error) {
		case 0:
			break;
		case ESTALE: {
			stale_count++;
			if (stale_count < 30) {
				uint64_t	old_startino;

				ireq->hdr.ino = old_startino =
					inumbers->xi_startino;
				error = -xfrog_inumbers(&ctx->mnt, ireq);
				if (error)
					goto err;
				/*
				 * Retry only if inumbers returns the same
				 * inobt record as the previous record and
				 * there are allocated inodes in it.
				 */
				if (!si->aborted &&
				    ireq->hdr.ocount > 0 &&
				    inumbers->xi_alloccount > 0 &&
				    inumbers->xi_startino == old_startino)
					goto retry;
				goto out;
			}
			str_info(ctx, descr_render(&dsc_bulkstat),
_("Changed too many times during scan; giving up."));
			si->aborted = true;
			goto out;
		}
		case ECANCELED:
			error = 0;
			fallthrough;
		default:
			goto err;
		}
		if (scrub_excessive_errors(ctx)) {
			si->aborted = true;
			goto out;
		}
		last_ino = scan_ino;
	}

err:
	if (error) {
		str_liberror(ctx, error, descr_render(&dsc_bulkstat));
		si->aborted = true;
	}
out:
	free(ichunk);
}

/*
 * Call INUMBERS for information about inode chunks, then queue the inumbers
 * responses in the bulkstat workqueue.  This helps us maximize CPU parallelism
 * if the filesystem AGs are not evenly loaded.
 */
static void
scan_ag_inumbers(
	struct workqueue	*wq,
	xfs_agnumber_t		agno,
	void			*arg)
{
	struct scan_ichunk	*ichunk = NULL;
	struct scan_inodes	*si = arg;
	struct scrub_ctx	*ctx = (struct scrub_ctx *)wq->wq_ctx;
	struct xfs_inumbers_req	*ireq;
	uint64_t		nextino = cvt_agino_to_ino(&ctx->mnt, agno, 0);
	int			error;
	DEFINE_DESCR(dsc, ctx, render_inumbers_from_agno);

	descr_set(&dsc, &agno);

	error = alloc_ichunk(ctx, si, agno, 0, &ichunk);
	if (error)
		goto err;
	ireq = ichunk_to_inumbers(ichunk);

	/* Find the inode chunk & alloc mask */
	error = -xfrog_inumbers(&ctx->mnt, ireq);
	while (!error && !si->aborted && ireq->hdr.ocount > 0) {
		/*
		 * Make sure that we always make forward progress while we
		 * scan the inode btree.
		 */
		if (nextino > ireq->inumbers[0].xi_startino) {
			str_corrupt(ctx, descr_render(&dsc),
	_("AG %u inode btree is corrupt near agino %lu, got %lu"), agno,
				cvt_ino_to_agino(&ctx->mnt, nextino),
				cvt_ino_to_agino(&ctx->mnt,
						ireq->inumbers[0].xi_startino));
			si->aborted = true;
			break;
		}
		nextino = ireq->hdr.ino;

		if (ireq->inumbers[0].xi_alloccount == 0) {
			/*
			 * We can have totally empty inode chunks on
			 * filesystems where there are more than 64 inodes per
			 * block.  Skip these.
			 */
			;
		} else if (si->nr_threads > 0) {
			/* Queue this inode chunk on the bulkstat workqueue. */
			error = -workqueue_add(&si->wq_bulkstat,
					scan_ag_bulkstat, agno, ichunk);
			if (error) {
				si->aborted = true;
				str_liberror(ctx, error,
						_("queueing bulkstat work"));
				goto out;
			}
			ichunk = NULL;
		} else {
			/*
			 * Only one thread, call bulkstat directly.  Remember,
			 * ichunk is freed by the worker before returning.
			 */
			scan_ag_bulkstat(wq, agno, ichunk);
			ichunk = NULL;
			if (si->aborted)
				break;
		}

		if (!ichunk) {
			error = alloc_ichunk(ctx, si, agno, nextino, &ichunk);
			if (error)
				goto err;
		}
		ireq = ichunk_to_inumbers(ichunk);

		error = -xfrog_inumbers(&ctx->mnt, ireq);
	}

err:
	if (error) {
		str_liberror(ctx, error, descr_render(&dsc));
		si->aborted = true;
	}
out:
	if (ichunk)
		free(ichunk);
}

/*
 * Scan all the inodes in a filesystem, including metadata directory files and
 * broken files.  On error, this function will log an error message and return
 * -1.
 */
int
scrub_scan_all_inodes(
	struct scrub_ctx	*ctx,
	scrub_inode_iter_fn	fn,
	void			*arg)
{
	struct scan_inodes	si = {
		.fn		= fn,
		.arg		= arg,
		.nr_threads	= scrub_nproc_workqueue(ctx),
	};
	xfs_agnumber_t		agno;
	struct workqueue	wq_inumbers;
	unsigned int		max_bulkstat;
	int			ret;

	/*
	 * The bulkstat workqueue should queue at most one inobt block's worth
	 * of inode chunk records per worker thread.  If we're running in
	 * single thread mode (nr_threads==0) then we skip the workqueues.
	 */
	max_bulkstat = si.nr_threads * (ctx->mnt.fsgeom.blocksize / 16);

	ret = -workqueue_create_bound(&si.wq_bulkstat, (struct xfs_mount *)ctx,
			si.nr_threads, max_bulkstat);
	if (ret) {
		str_liberror(ctx, ret, _("creating bulkstat workqueue"));
		return -1;
	}

	ret = -workqueue_create(&wq_inumbers, (struct xfs_mount *)ctx,
			si.nr_threads);
	if (ret) {
		str_liberror(ctx, ret, _("creating inumbers workqueue"));
		si.aborted = true;
		goto kill_bulkstat;
	}

	for (agno = 0; agno < ctx->mnt.fsgeom.agcount; agno++) {
		ret = -workqueue_add(&wq_inumbers, scan_ag_inumbers, agno, &si);
		if (ret) {
			si.aborted = true;
			str_liberror(ctx, ret, _("queueing inumbers work"));
			break;
		}
	}

	ret = -workqueue_terminate(&wq_inumbers);
	if (ret) {
		si.aborted = true;
		str_liberror(ctx, ret, _("finishing inumbers work"));
	}
	workqueue_destroy(&wq_inumbers);

kill_bulkstat:
	ret = -workqueue_terminate(&si.wq_bulkstat);
	if (ret) {
		si.aborted = true;
		str_liberror(ctx, ret, _("finishing bulkstat work"));
	}
	workqueue_destroy(&si.wq_bulkstat);

	return si.aborted ? -1 : 0;
}

struct user_bulkstat {
	struct scan_inodes	*si;

	/* vla, must be last */
	struct xfs_bulkstat_req	breq;
};

/* Iterate all the user files returned by a bulkstat. */
static void
scan_user_files(
	struct workqueue	*wq,
	xfs_agnumber_t		agno,
	void			*arg)
{
	struct xfs_handle	handle;
	struct scrub_ctx	*ctx = (struct scrub_ctx *)wq->wq_ctx;
	struct user_bulkstat	*ureq = arg;
	struct xfs_bulkstat	*bs = &ureq->breq.bulkstat[0];
	struct scan_inodes	*si = ureq->si;
	int			i;
	int			error = 0;
	DEFINE_DESCR(dsc_bulkstat, ctx, render_ino_from_bulkstat);

	handle_from_fshandle(&handle, ctx->fshandle, ctx->fshandle_len);

	for (i = 0; !si->aborted && i < ureq->breq.hdr.ocount; i++, bs++) {
		descr_set(&dsc_bulkstat, bs);
		handle_from_bulkstat(&handle, bs);
		error = si->fn(ctx, &handle, bs, si->arg);
		switch (error) {
		case 0:
			break;
		case ESTALE:
		case ECANCELED:
			error = 0;
			fallthrough;
		default:
			goto err;
		}
		if (scrub_excessive_errors(ctx)) {
			si->aborted = true;
			goto out;
		}
	}

err:
	if (error) {
		str_liberror(ctx, error, descr_render(&dsc_bulkstat));
		si->aborted = true;
	}
out:
	free(ureq);
}

/*
 * Run one step of the user files bulkstat scan and schedule background
 * processing of the stat data returned.  Returns 1 to keep going, or 0 to
 * stop.
 */
static int
scan_user_bulkstat(
	struct scrub_ctx	*ctx,
	struct scan_inodes	*si,
	uint64_t		*cursor)
{
	struct user_bulkstat	*ureq;
	const char		*what = NULL;
	int			ret;

	ureq = calloc(1, sizeof(struct user_bulkstat) +
			 XFS_BULKSTAT_REQ_SIZE(LIBFROG_BULKSTAT_CHUNKSIZE));
	if (!ureq) {
		ret = ENOMEM;
		what = _("creating bulkstat work item");
		goto err;
	}
	ureq->si = si;
	ureq->breq.hdr.icount = LIBFROG_BULKSTAT_CHUNKSIZE;
	ureq->breq.hdr.ino = *cursor;

	ret = -xfrog_bulkstat(&ctx->mnt, &ureq->breq);
	if (ret) {
		what = _("user files bulkstat");
		goto err_ureq;
	}
	if (ureq->breq.hdr.ocount == 0) {
		*cursor = NULLFSINO;
		free(ureq);
		return 0;
	}

	*cursor = ureq->breq.hdr.ino;

	/* scan_user_files frees ureq; do not access it */
	ret = -workqueue_add(&si->wq_bulkstat, scan_user_files, 0, ureq);
	if (ret) {
		what = _("queueing bulkstat work");
		goto err_ureq;
	}
	ureq = NULL;

	return 1;

err_ureq:
	free(ureq);
err:
	si->aborted = true;
	str_liberror(ctx, ret, what);
	return 0;
}

/*
 * Scan all the user files in a filesystem in inumber order.  On error, this
 * function will log an error message and return -1.
 */
int
scrub_scan_user_files(
	struct scrub_ctx	*ctx,
	scrub_inode_iter_fn	fn,
	void			*arg)
{
	struct scan_inodes	si = {
		.fn		= fn,
		.arg		= arg,
		.nr_threads	= scrub_nproc_workqueue(ctx),
	};
	uint64_t		ino = 0;
	int			ret;

	/* Queue up to four bulkstat result sets per thread. */
	ret = -workqueue_create_bound(&si.wq_bulkstat, (struct xfs_mount *)ctx,
			si.nr_threads, si.nr_threads * 4);
	if (ret) {
		str_liberror(ctx, ret, _("creating bulkstat workqueue"));
		return -1;
	}

	while ((ret = scan_user_bulkstat(ctx, &si, &ino)) == 1) {
		/* empty */
	}

	ret = -workqueue_terminate(&si.wq_bulkstat);
	if (ret) {
		si.aborted = true;
		str_liberror(ctx, ret, _("finishing bulkstat work"));
	}
	workqueue_destroy(&si.wq_bulkstat);

	return si.aborted ? -1 : 0;
}

/* Open a file by handle, returning either the fd or -1 on error. */
int
scrub_open_handle(
	struct xfs_handle	*handle)
{
	return open_by_fshandle(handle, sizeof(*handle),
			O_RDONLY | O_NOATIME | O_NOFOLLOW | O_NOCTTY);
}