dump/content.c

/*
 * Copyright (c) 2000-2004 Silicon Graphics, Inc.
 * All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it would be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software Foundation,
 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include <unistd.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/prctl.h>
#include <time.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <sys/statvfs.h>
#include <dirent.h>
#include <sys/ioctl.h>
#include <sys/quota.h>
#include <malloc.h>
#include <assert.h>
#include <string.h>
#include <uuid/uuid.h>

#include <xfs/xfs.h>
#include <xfs/jdm.h>
#ifdef linux
#include <linux/limits.h>
#include <xfs/xqm.h>
#endif

#include <attr/attributes.h>

#include "config.h"

#include "hsmapi.h"

#include "exit.h"
#include "types.h"
#include "path.h"
#include "timeutil.h"
#include "util.h"
#include "lock.h"
#include "qlock.h"
#include "mlog.h"
#include "dlog.h"
#include "getopt.h"
#include "stream.h"
#include "cldmgr.h"
#include "global.h"
#include "drive.h"
#include "media.h"
#include "content.h"
#include "content_common.h"
#include "content_inode.h"
#include "fs.h"
#include "inomap.h"
#include "var.h"
#include "inventory.h"
#include "getdents.h"
#include "arch_xlate.h"

/* max "unsigned long long int"
 */
#define ULONGLONG_MAX	18446744073709551615LLU

/* legal range of dump levels
 */
#define LEVEL_DEFAULT	0
#define LEVEL_MAX	9

/* ordinary files as big or bigger than this many pages will be
 * preceeded in the dump by enough padding to align the first byte
 * of that file's data to a page boundary
 */
#define PGALIGNTHRESH	8


/* structure definitions used locally ****************************************/

/* number of bstats bstat_iter fetches at a time
 */
#define BSTATBUFLEN	4096

/* if the source file system type can't be determined, assume it is this
 */
#define FS_DEFAULT	"xfs"

/* marks consist of a opaque drive layer cookie and a startpoint.
 * the drive layer requires that it be passed a pointer to a drive_markrec_t.
 * we tack on content-specific baggage (startpt_t). this works because we
 * allocate and free mark_t's here.
 */
struct mark {
	drive_markrec_t dm;
	startpt_t startpt;
};

typedef struct mark mark_t;

/* Media_mfile_begin() entry state.
 */
enum bes { BES_INIT,	/* in the beginning */
	   BES_ENDOK,   /* last media file successfully flushed to media */
	   BES_ENDEOM,  /* hit EOM while writing last media file */
	   BES_INVAL }; /* to assert protocol being followed */

typedef enum bes bes_t;

/* per-stream context
 */
struct context {
	filehdr_t *cc_filehdrp;
			/* pre-allocated buffer: heads each dumped file
			 */
	extenthdr_t *cc_extenthdrp;
			/* pre-allocated buffer: heads each dumped file extent
			 */
	void *cc_inomap_contextp;
			/* pre-allocated context to speed inomap iteration
			 */
	char *cc_getdentsbufp;
	size_t cc_getdentsbufsz;
			/* pre-allocated buffer for getdents() syscall
			 */
	char *cc_mdirentbufp;
	size_t cc_mdirentbufsz;
			/* pre-allocated buffer for on-media dirent
			 */
	char *cc_extattrlistbufp;
	size_t cc_extattrlistbufsz;
			/* pre-allocated buffer for retrieving a
			 * list of extended file attributes
			 */
	attr_multiop_t *cc_extattrrtrvarrayp;
	size_t cc_extattrrtrvarraylen;
			/* pre-allocated array of ops for retrieving the
			 * values for a list of extended file attributes
			 */
	char *cc_extattrdumpbufp;
	size_t cc_extattrdumpbufsz;
			/* pre-allocated buffer for dumping the names and
			 * values for a list of extended file attributes
			 */
	hsm_f_ctxt_t *cc_hsm_f_ctxtp;
			/* pre-allocated HSM context used for holding HSM
			   state information about a file across subroutine
			   calls.
			*/
	char *cc_readlinkbufp;
	size_t cc_readlinkbufsz;
			/* pre-allocated buffer for readlink()
			 */
	off64_t cc_mfilesz;
			/* total bytes dumped to media file
			 */
	size_t cc_markscommitted;
			/* number of marks committed in mfile. only useful
			 * info is if greater than zero.
			 */
	xfs_ino_t cc_stat_lastino;
			/* monotonic strm nondir ino dumped
			 */
	bool_t cc_completepr;
			/* set if stream completely dumped. useful for
			 * determining if dump was interrupted
			 */
	bool_t cc_Media_useterminatorpr;
			/* true if stream terminators are expected and
			 * will be used
			 */
	char *cc_Media_firstlabel;
			/* optional command line media label. only used
			 * for first media object in stream, and only if
			 * media object does not already have a label
			 */
	bes_t cc_Media_begin_entrystate;
			/* Media_mfile_begin context entry state
			 */
};

typedef struct context context_t;

/* extent group context, used by dump_file()
 */
#define BMAP_LEN	512

struct extent_group_context {
	getbmapx_t eg_bmap[BMAP_LEN];
	getbmapx_t *eg_nextbmapp;	/* ptr to the next extent to dump */
	getbmapx_t *eg_endbmapp;		/* to detect extent exhaustion */
	int eg_fd;			/* file desc. */
	int eg_bmapix;		/* debug info only, not used */
	int eg_gbmcnt;		/* debug, counts getbmapx calls for ino*/
};

typedef struct extent_group_context extent_group_context_t;


/* minimum getdents() buffer size
 */
#define GETDENTSBUF_SZ_MIN	(2 * pgsz)


/* minimum sizes for extended attributes buffers
 */
#define EXTATTR_LISTBUF_SZ	(XATTR_LIST_MAX)
#define EXTATTR_RTRVARRAY_LEN	(1 * pgsz)
#define EXTATTR_DUMPBUF_SZ	(4 * pgsz)

/* for printing ext attr namespace
 */
#define EXTATTR_NAMESPACE(flag)	(((flag) & ATTR_ROOT) ? _("root") : \
				(((flag) & ATTR_SECURE) ? _("secure") : \
				  _("non-root")))

/* for printing file type
 */
#define FILETYPE(statp)		(((statp)->bs_mode & S_IFMT) == S_IFDIR \
				  ? _("dir") : _("nondir"))

/* per-drive status descriptor
 */
struct pds {
	enum { PDS_NULL,		/* per-drive activity not begun */
	       PDS_INOMAP,		/* dumping inomap */
	       PDS_DIRDUMP,		/* dumping dirs */
	       PDS_NONDIR,		/* dumping nondirs */
	       PDS_INVSYNC,		/* waiting for inventory */
	       PDS_INVDUMP,		/* dumping session inventory */
	       PDS_TERMDUMP		/* writing stream terminator */
	} pds_phase;
	size64_t pds_dirdone;		/* number of directories done */
};

typedef struct pds pds_t;

/* declarations of externally defined global symbols *************************/

extern void usage(void);
extern bool_t preemptchk(int);
extern char *homedir;
extern bool_t pipeline;
extern bool_t stdoutpiped;
extern char *sistr;
extern size_t pgsz;


/* forward declarations of locally defined static functions ******************/

/* file dumpers
 */
static rv_t dump_dirs(ix_t strmix,
		       struct xfs_bstat *bstatbufp,
		       size_t bstatbuflen,
		       void *inomap_contextp);
static rv_t dump_dir(ix_t strmix,
		      jdm_fshandle_t *,
		      int,
		      struct xfs_bstat *);
static rv_t dump_file(void *,
		       jdm_fshandle_t *,
		       int,
		       struct xfs_bstat *);
static rv_t dump_file_reg(drive_t *drivep,
			   context_t *contextp,
			   content_inode_hdr_t *scwhdrp,
			   jdm_fshandle_t *,
			   struct xfs_bstat *,
			   bool_t *);
static rv_t dump_file_spec(drive_t *drivep,
			    context_t *contextp,
			    jdm_fshandle_t *,
			    struct xfs_bstat *);
static rv_t dump_filehdr(drive_t *drivep,
			  context_t *contextp,
			  struct xfs_bstat *,
			  off64_t,
			  int);
static rv_t dump_extenthdr(drive_t *drivep,
			    context_t *contextp,
			    int32_t,
			    int32_t,
			    off64_t,
			    off64_t);
static rv_t dump_dirent(drive_t *drivep,
			 context_t *contextp,
			 struct xfs_bstat *,
			 xfs_ino_t,
			 gen_t,
			 char *,
			 size_t);
static rv_t init_extent_group_context(jdm_fshandle_t *,
				       struct xfs_bstat *,
				       extent_group_context_t *);
static void cleanup_extent_group_context(extent_group_context_t *);
static rv_t dump_extent_group(drive_t *drivep,
			       context_t *contextp,
			       struct xfs_bstat *,
			       extent_group_context_t *,
			       off64_t,
			       off64_t,
			       bool_t,
			       off64_t *,
			       off64_t *,
			       bool_t *);
static bool_t dump_session_inv(drive_t *drivep,
			        context_t *contextp,
			        media_hdr_t *mwhdrp,
			        content_inode_hdr_t *scwhdrp);
static rv_t write_pad(drive_t *drivep, size_t);

static void mark_callback(void *, drive_markrec_t *, bool_t);

static void inv_cleanup(void);
static void dump_terminator(drive_t *drivep,
			     context_t *contextp,
			     media_hdr_t *mwhdrp);
static rv_t Media_mfile_begin(drive_t *drivep,
			       context_t *contextp,
			       bool_t intr_allowed);
static rv_t Media_mfile_end(drive_t *drivep,
			     context_t *contextp,
			     media_hdr_t *mwhdrp,
			     off64_t *ncommittedp,
			     bool_t hit_eom);
static bool_t Media_prompt_overwrite(drive_t *drivep);
static rv_t Media_erasechk(drive_t *drivep,
			    int dcaps,
			    bool_t intr_allowed,
			    bool_t prevmediapresentpr);
static bool_t Media_prompt_erase(drive_t *drivep);
static char *Media_prompt_label(drive_t *drivep, char *bufp, size_t bufsz);
static void update_cc_Media_useterminatorpr(drive_t *drivep,
					     context_t *contextp);
static void set_mcflag(ix_t thrdix);
static void clr_mcflag(ix_t thrdix);

static bool_t check_complete_flags(void);

static rv_t dump_extattrs(drive_t *drivep,
			   context_t *contextp,
	       		   jdm_fshandle_t *fshandlep,
			   struct xfs_bstat *statp);
static rv_t dump_extattr_list(drive_t *drivep,
			       context_t *contextp,
	       		       jdm_fshandle_t *fshandlep,
			       struct xfs_bstat *statp,
			       attrlist_t *listp,
			       int flag,
			       bool_t *abortprp);
static char *dump_extattr_buildrecord(struct xfs_bstat *statp,
				       char *dumpbufp,
				       char *dumpbufendp,
				       char *namesrcp,
				       uint32_t valuesz,
				       int flag,
				       char **valuepp);
static rv_t dump_extattrhdr(drive_t *drivep,
			     context_t *contextp,
			     struct xfs_bstat *statp,
			     size_t recsz,
			     size_t valoff,
			     ix_t flags,
			     uint32_t valsz);

static bool_t save_quotas(char *mntpnt,
			   quota_info_t *quotainfo);
static int getxfsqstat(char *fsdev);



/* definition of locally defined global variables ****************************/

bool_t content_media_change_needed;
char *media_change_alert_program = NULL;
hsm_fs_ctxt_t *hsm_fs_ctxtp = NULL;
uint64_t hdr_mfilesz = 0;
uint64_t maxdumpfilesize = 0;
bool_t allowexcludefiles_pr = BOOL_FALSE;

/* definition of locally defined static variables *****************************/

static bool_t sc_preerasepr = BOOL_FALSE;
	/* pre-erase media
	 */
static inv_idbtoken_t sc_inv_idbtoken = INV_TOKEN_NULL;
	/* handle to inventory
	 */
static inv_sestoken_t sc_inv_sestoken = INV_TOKEN_NULL;
	/* handle to inventory session
	 */
static inv_stmtoken_t *sc_inv_stmtokenp = 0;
	/* array of inventory session stream handles
	 */
static bool_t sc_inv_updatepr = BOOL_TRUE;
	/* set if ok to update online inventory with stats of this dump
	 */
static ix_t sc_level = LEVEL_DEFAULT;
	/* dump level requested
	 */
static bool_t sc_incrpr = BOOL_FALSE;
static time32_t sc_incrbasetime;
static ix_t sc_incrbaselevel;
static uuid_t sc_incrbaseid;
	/* if an incremental dump, the base, level and time of the incremental
	 * base dump. TRICKY: if resuming an incremental dump, this is the
	 * base of the original incremental.
	 */
static bool_t sc_resumepr = BOOL_FALSE;
static time32_t sc_resumebasetime = 0;
static uuid_t sc_resumebaseid;
static size_t sc_resumerangecnt = 0;
static drange_t *sc_resumerangep = 0;
	/* if a resumed dump, the id, time and undumped ino/offset ranges
	 * of the interrupted dump being resumed.
	 */
static jdm_fshandle_t *sc_fshandlep = 0;
	/* dmi file system handle
	 */
static int sc_fsfd = -1;
	/* open file descriptor for root directory
	 */
static struct xfs_bstat *sc_rootxfsstatp = 0;
	/* pointer to loaded bulkstat for root directory
	 */
static startpt_t *sc_startptp = 0;
	/* an array of stream ino/offset start points
	 */
static time32_t sc_stat_starttime = 0;
	/* for cacluating elapsed time
	 */
static ix_t sc_stat_inomapphase = 0;
static ix_t sc_stat_inomappass = 0;
static size64_t sc_stat_inomapcnt;
static size64_t sc_stat_inomapdone;
static size64_t sc_stat_dircnt = 0;
	/* total number of directory inodes to be dumped (strm 0)
	 */
static pds_t sc_stat_pds[STREAM_SIMMAX];
	/* per-drive stream status
	 */
static size64_t sc_stat_nondircnt = 0;
	/* total number of non-directory inodes to be dumped (all strms)
	 */
static size64_t sc_stat_nondirdone = 0;
	/* total number of non-directory inodes dumped (all strms)
	 */
static size64_t sc_stat_datasz = 0;
	/* total size in bytes of non-dirs to be dumped (all strms)
	 */
static size64_t sc_stat_datadone = 0;
	/* total size in bytes of non-dirs dumped (all strms)
	 */
static size_t sc_thrdsarrivedcnt = 0;
	/* each thread checks in by bumping this count under lock.
	 * used to decide when its ok to begin waiting for all threads
	 * to arrive at sync pt for session inventory dump.
	 */
static size_t sc_thrdsdonecnt = 0;
	/* number of threads which are ready to dump the session inventory.
	 * when equal to the number of streams remaining (stream_cnt()),
	 * can proceed with inventory dumps
	 */
static context_t *sc_contextp;
	/* an array of per-stream context descriptors
	 */
static bool_t sc_mcflag[STREAM_SIMMAX];
	/* media change flag
	 */
static bool_t sc_dumpextattrpr = BOOL_TRUE;
	/* dump extended attributes
	 */
static bool_t sc_dumpasoffline = BOOL_FALSE;
	/* dump dual-residency HSM files as offline
	 */
static bool_t sc_use_old_direntpr = BOOL_FALSE;
	/* dump dirents as dirent_v1_t instead of dirent_t
	 * (for compat with dump format 2)
	 */

static bool_t sc_savequotas = BOOL_TRUE;
/* save quota information in dump
 */
static quota_info_t quotas[] = {
	{ "user quota",		BOOL_TRUE,	CONTENT_QUOTAFILE,	"", "-uf", XFS_QUOTA_UDQ_ACCT, 0 },
	{ "project quota",	BOOL_TRUE,	CONTENT_PQUOTAFILE,	"", "-pf", XFS_QUOTA_PDQ_ACCT, 0 },
	{ "group quota",	BOOL_TRUE,	CONTENT_GQUOTAFILE,	"", "-gf", XFS_QUOTA_GDQ_ACCT, 0 }
};

/* definition of locally defined global functions ****************************/


/* definition of locally defined static functions ****************************/
static bool_t create_inv_session(
		global_hdr_t *gwhdrtemplatep,
		uuid_t *fsidp,
		const char *mntpnt,
		const char *fsdevice,
		ix_t subtreecnt,
		size_t strmix);

/*
 * Verify that we are asked to dump from the root of the filesystem;
 * test this by checking whether the inode number we've been given matches
 * the inode number for this directory's ".."
 */
static bool_t
check_rootdir(int fd,
	      xfs_ino_t ino)
{
	struct dirent	*gdp;
	size_t		gdsz;
	bool_t		found = BOOL_FALSE;

	gdsz = sizeof(struct dirent) + NAME_MAX + 1;
	if (gdsz < GETDENTSBUF_SZ_MIN)
		gdsz = GETDENTSBUF_SZ_MIN;
	gdp = (struct dirent *)calloc(1, gdsz);
	assert(gdp);

	while (1) {
		struct dirent *p;
		int nread;

		nread = getdents_wrap(fd, (char *)gdp, gdsz);
		/*
		 * negative count indicates something very bad happened;
		 * try to gracefully end this dir.
		 */
		if (nread < 0) {
			mlog(MLOG_NORMAL | MLOG_WARNING,
_("unable to read dirents for directory ino %llu: %s\n"),
			      ino, strerror(errno));
			break;
		}

		/* no more directory entries: break; */
		if (!nread)
			break;

		for (p = gdp; nread > 0;
		     nread -= (int)p->d_reclen,
		     assert(nread >= 0),
		     p = (struct dirent *)((char *)p + p->d_reclen)) {
			if (!strcmp(p->d_name, "..")) {
				if (p->d_ino == ino)
					found = BOOL_TRUE;
				break;
			}
		}
	}
	free(gdp);
	return found;
}

bool_t
content_init(int argc,
	      char *argv[],
	      global_hdr_t *gwhdrtemplatep)
{

	inv_idbtoken_t inv_idbt;
	inv_session_t *sessp = 0;
	drive_hdr_t *dwhdrtemplatep;
	media_hdr_t *mwhdrtemplatep;
	content_hdr_t *cwhdrtemplatep;
	content_inode_hdr_t *scwhdrtemplatep;
	ix_t subtreecnt;
	char **subtreep;
	ix_t subtreeix;
	bool_t resumereqpr = BOOL_FALSE;
	char *srcname;
	char mntpnt[GLOBAL_HDR_STRING_SZ];
	char fsdevice[GLOBAL_HDR_STRING_SZ];
	char fstype[CONTENT_HDR_FSTYPE_SZ];
	bool_t skip_unchanged_dirs = BOOL_FALSE;
	uuid_t fsid;
	bool_t underfoundpr;
	ix_t underlevel = (ix_t)(-1);
	time32_t undertime = 0;
	uuid_t underid;
	bool_t underpartialpr = BOOL_FALSE;
	bool_t underinterruptedpr = BOOL_FALSE;
	bool_t samefoundpr;
	time32_t sametime = 0;
	uuid_t sameid;
	bool_t samepartialpr = BOOL_FALSE;
	bool_t sameinterruptedpr = BOOL_FALSE;
	size_t strmix;
	int c;
	int i;
	int qstat;
	int rval;
	bool_t ok;
	extern char *optarg;
	extern int optind, opterr, optopt;
	char *baseuuidstr = NULL;
	uuid_t baseuuid;
	bool_t baseuuidvalpr;
	uint64_t dircnt;
	uint64_t nondircnt;
	uint64_t datasz;
	uint64_t inocnt;
	uint64_t inomapsz;
	uint64_t direntsz;
	uint64_t filesz;
	uint64_t size_estimate;

	/* basic sanity checks
	 */
	assert(sizeof(mode_t) == MODE_SZ);
	assert(sizeof(timestruct_t) == TIMESTRUCT_SZ);
	assert(sizeof(bstat_t) == BSTAT_SZ);
	assert(sizeof(filehdr_t) == FILEHDR_SZ);
	assert(sizeof(extenthdr_t) == EXTENTHDR_SZ);
	assert(sizeof(direnthdr_t) == DIRENTHDR_SZ);
	assert(sizeof(direnthdr_v1_t) == DIRENTHDR_SZ);
	assert(DIRENTHDR_SZ % DIRENTHDR_ALIGN == 0);
	assert(sizeofmember(content_hdr_t, ch_specific)
		>=
		sizeof(content_inode_hdr_t));
	assert(sizeof(extattrhdr_t) == EXTATTRHDR_SZ);

	/* calculate offsets of portions of the write hdr template
	 */
	dwhdrtemplatep = (drive_hdr_t *)gwhdrtemplatep->gh_upper;
	mwhdrtemplatep = (media_hdr_t *)dwhdrtemplatep->dh_upper;
	cwhdrtemplatep = (content_hdr_t *)mwhdrtemplatep->mh_upper;
	scwhdrtemplatep = (content_inode_hdr_t *) cwhdrtemplatep->ch_specific;

	if (gwhdrtemplatep->gh_version < GLOBAL_HDR_VERSION_3) {
		sc_use_old_direntpr = BOOL_TRUE;
	}

	/* process command line args
	 */
	optind = 1;
	opterr = 0;
	subtreecnt = 0;
	baseuuidvalpr = BOOL_FALSE;
	while ((c = getopt(argc, argv, GETOPT_CMDSTRING)) != EOF) {
		switch (c) {
		case GETOPT_LEVEL:
			if (!optarg || optarg[0] == '-') {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument missing\n"),
				      c);
				usage();
				return BOOL_FALSE;
			}
			sc_level = (ix_t)atoi(optarg);
			if (sc_level > LEVEL_MAX) {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument must be "
				      "between 0 and %d\n"),
				      c,
				      LEVEL_MAX);
				usage();
				return BOOL_FALSE;
			}
			break;
		case GETOPT_SUBTREE:
			if (!optarg || optarg[0] == '-') {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument missing\n"),
				      c);
				usage();
				return BOOL_FALSE;
			}
			if (optarg[0] == '/') {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument (subtree) "
				      "must be a relative pathname\n"),
				      c);
				usage();
				return BOOL_FALSE;
			}
			subtreecnt++;
			break;
		case GETOPT_MAXDUMPFILESIZE:
			if (!optarg || optarg [0] == '-') {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument missing\n"),
				      c);
				usage();
				return BOOL_FALSE;
			}
			maxdumpfilesize = strtoull(optarg, NULL, 0);
			if (maxdumpfilesize == 0 ||
			     maxdumpfilesize > ULONGLONG_MAX / 1024 ||
			     (maxdumpfilesize == ULONGLONG_MAX && errno == ERANGE)) {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument is not a valid file size\n"),
				      c);
				usage();
				return BOOL_FALSE;
			}
			maxdumpfilesize *= 1024;
			break;
		case GETOPT_NOUNCHANGEDDIRS:
			skip_unchanged_dirs = BOOL_TRUE;
			break;
		case GETOPT_EXCLUDEFILES:
			allowexcludefiles_pr = BOOL_TRUE;
			break;
		case GETOPT_RESUME:
			resumereqpr = BOOL_TRUE;
			break;
		case GETOPT_NOINVUPDATE:
			sc_inv_updatepr = BOOL_FALSE;
			break;
		case GETOPT_ERASE:
			sc_preerasepr = BOOL_TRUE;
			break;
		case GETOPT_ALERTPROG:
			if (!optarg || optarg[0] == '-') {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
					"-%c argument missing\n"),
				    c);
				usage();
				return BOOL_FALSE;
			}
			media_change_alert_program = optarg;
			break;
		case GETOPT_NOEXTATTR:
			sc_dumpextattrpr = BOOL_FALSE;
			break;
		case GETOPT_DUMPASOFFLINE:
			sc_dumpasoffline = BOOL_TRUE;
			break;
		case GETOPT_BASED:
			if (!optarg || optarg[0] == '-') {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument missing\n"),
				      c);
				usage();
				return BOOL_FALSE;
			}
			baseuuidstr = optarg;

			if (uuid_parse(baseuuidstr, baseuuid) < 0) {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "-%c argument not a valid "
				      "dump session id\n"),
				      c);
				usage();
				return BOOL_FALSE;
			}
			baseuuidvalpr = BOOL_TRUE;
		}
	}

	if (resumereqpr && baseuuidvalpr) {
		mlog(MLOG_NORMAL | MLOG_ERROR, _(
		      "may not specify both -%c and -%c\n"),
		      GETOPT_BASED,
		      GETOPT_RESUME);
		return BOOL_FALSE;
	}

	/* the user may specify stdout as the destination, by a single
	 * dash ('-') with no option letter. This must appear between
	 * all lettered arguments and the source file system pathname.
	 */
	if (optind < argc && !strcmp(argv[optind ], "-")) {
		optind++;
	}

	/* the last argument must be either the mount point or a
	 * device pathname of the file system to be dumped.
	 */
	if (optind >= argc) {
		mlog(MLOG_NORMAL | MLOG_ERROR, _(
		      "source file system "
		      "not specified\n"));
		usage();
		return BOOL_FALSE;
	}
	srcname = argv[optind];

	if (preemptchk(PREEMPT_FULL)) {
		return BOOL_FALSE;
	}

	/* allocate space for the subtree pointer array and load it
	 */
	if (subtreecnt) {
		subtreep = (char **)calloc(subtreecnt, sizeof(char *));
		assert(subtreep);
		optind = 1;
		opterr = 0;
		subtreeix = 0;
		while ((c = getopt(argc, argv, GETOPT_CMDSTRING)) != EOF) {
			switch (c) {
			case GETOPT_SUBTREE:
				assert(subtreeix < subtreecnt);
				assert(optarg && optarg[0] != '-');
				subtreep[subtreeix++] = optarg;
				break;
			}
		}
		assert(subtreeix == subtreecnt);
	} else {
		subtreep = 0;
	}

	/* call a magic function to figure out if the last argument is
	 * a mount point or a device pathname, and retrieve the file
	 * system type, full pathname of the character special device
	 * containing the file system, the latest mount point, and the file
	 * system ID (uuid). returns BOOL_FALSE if the last
	 * argument doesn't look like a file system.
	 */
	if (!fs_info(fstype,
			sizeof(fstype),
			FS_DEFAULT,
			fsdevice,
			sizeof(fsdevice),
			mntpnt,
			sizeof(mntpnt),
			&fsid,
			srcname)) {

		mlog(MLOG_NORMAL | MLOG_ERROR, _(
		      "%s does not identify a file system\n"),
		      srcname);
		usage();
		return BOOL_FALSE;
	}

	/* verify that the file system is mounted. This must be enhanced
	 * to mount an unmounted file system on a temporary mount point,
	 * if it is not currently mounted.
	 */
	if (!fs_mounted(fstype, fsdevice, mntpnt, &fsid)) {
		mlog(MLOG_NORMAL | MLOG_ERROR, _(
		      "%s must be mounted to be dumped\n"),
		      srcname);
		return BOOL_FALSE;
	}

	/* place the fs info in the write hdr template
	 */
	(void)strncpyterm(cwhdrtemplatep->ch_mntpnt,
			     mntpnt,
			     sizeof(cwhdrtemplatep->ch_mntpnt));
	(void)strncpyterm(cwhdrtemplatep->ch_fsdevice,
			     fsdevice,
			     sizeof(cwhdrtemplatep->ch_fsdevice));
	(void)strncpyterm(cwhdrtemplatep->ch_fstype,
			     fstype,
			     sizeof(cwhdrtemplatep->ch_fstype));
	uuid_copy(cwhdrtemplatep->ch_fsid, fsid);

	/* write quota information */
	if(sc_savequotas) {

		sc_savequotas = BOOL_FALSE;
		for(i = 0; i < (sizeof(quotas) / sizeof(quotas[0])); i++) {
			quotas[i].savequotas = BOOL_FALSE;
			qstat = getxfsqstat(fsdevice);
			if (qstat > 0 && (qstat & quotas[i].statflag)) {
				sprintf(quotas[i].quotapath, "%s/%s", mntpnt, quotas[i].quotafile);
				if(save_quotas(mntpnt, &quotas[i])) {
					if(subtreecnt) {
						subtreecnt++;
						subtreep = (char **) realloc(subtreep,
								subtreecnt * sizeof(char *));
						assert(subtreep);
						subtreep[subtreecnt - 1] = quotas[i].quotafile;
					}
					sc_savequotas = BOOL_TRUE;
					quotas[i].savequotas = BOOL_TRUE;
				} else {
					mlog(MLOG_NORMAL | MLOG_ERROR, _(
					      "failed to save %s information, continuing\n"),
					      quotas[i].desc);
				}
			}
		}
	}


	/* create my /var directory if it doesn't already exist.
	 */
	var_create();

	/* get two session descriptors from the inventory: one for the last
	 * dump at this level, and one for the last dump at a lower level.
	 * the former will be used to check if the last dump at this level
	 * was prematurely terminated; if so, for those inos already dumped
	 * then, we will look only for changes since that dump. the latter
	 * will give us a change date for all other inos.
	 */

	if (preemptchk(PREEMPT_FULL)) {
		return BOOL_FALSE;
	}

	/* briefly open the online dump inventory, so it can be used
	 * to calculate incremental and resumed dumps.
	 */
	inv_idbt = inv_open((inv_predicate_t)INV_BY_UUID,
			     INV_SEARCH_ONLY,
			     (void *)&fsid);

	/* if a based request, look for the indicated session.
	 * if found, and not interrupted, this will be used as an
	 * incremental base. if interrupted, will be used as
	 * resume base.
	 */
	if (baseuuidvalpr) {
		ix_t strix;
		ix_t strcnt;
		inv_stream_t *bsp;
		bool_t interruptedpr;

		underfoundpr = BOOL_FALSE;
		samefoundpr = BOOL_FALSE;
		underinterruptedpr = BOOL_FALSE;
		sameinterruptedpr = BOOL_FALSE;
		interruptedpr = BOOL_FALSE;

		ok = inv_get_session_byuuid(&fsid, &baseuuid, &sessp);
		if (!ok) {
			mlog(MLOG_NORMAL | MLOG_ERROR, _(
			      "could not find specified base dump (%s) "
			      "in inventory\n"),
			      baseuuidstr);
			return BOOL_FALSE;
		}
		strcnt =  (ix_t)sessp->s_nstreams;
		for (strix = 0; strix < strcnt; strix++) {
			bsp = &sessp->s_streams[strix];
			if (bsp->st_interrupted) {
				interruptedpr = BOOL_TRUE;
				break;
			}
		}

		if (interruptedpr) {
			sc_level = (ix_t)sessp->s_level;
			resumereqpr = BOOL_TRUE;
			samefoundpr = BOOL_TRUE;
			sametime = sessp->s_time;
			uuid_copy (sameid, sessp->s_sesid);
			samepartialpr = sessp->s_ispartial;
			sameinterruptedpr = BOOL_TRUE;
			sc_resumerangecnt =  (size_t)sessp->s_nstreams;
			sc_resumerangep = (drange_t *)calloc(sc_resumerangecnt,
								sizeof(drange_t));
			assert(sc_resumerangep);
			for (strmix = 0; strmix < sc_resumerangecnt; strmix++) {
				inv_stream_t *bsp;
				inv_stream_t *esp;
				drange_t *p = &sc_resumerangep[strmix];
				bsp = &sessp->s_streams[strmix];
				esp = (strmix < sc_resumerangecnt - 1)
				      ?
				      bsp + 1
				      :
				      0;
				if (bsp->st_interrupted) {
					sameinterruptedpr = BOOL_TRUE;
					p->dr_begin.sp_ino = bsp->st_endino;
					p->dr_begin.sp_offset = bsp->st_endino_off;
					if (esp) {
						p->dr_end.sp_ino = esp->st_startino;
						p->dr_end.sp_offset =
								esp->st_startino_off;
						mlog(MLOG_DEBUG,
						      "resume range stream %u "
						      "ino %llu:%lld to "
						      "%llu:%lld\n",
						      strmix,
						      p->dr_begin.sp_ino,
						      p->dr_begin.sp_offset,
						      p->dr_end.sp_ino,
						      p->dr_end.sp_offset);
					} else {
						p->dr_end.sp_flags = STARTPT_FLAGS_END;
						mlog(MLOG_DEBUG,
						      "resume range stream %u "
						      "ino %llu:%lld to "
						      "end\n",
						      strmix,
						      p->dr_begin.sp_ino,
						      p->dr_begin.sp_offset);
					}
				} else {
					/* set the range start pt's END flag to
					 * indicate the range was not interrupted.
					 */
					p->dr_begin.sp_flags = STARTPT_FLAGS_END;
				}
			}
		} else {
			if (sessp->s_level >= LEVEL_MAX) {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "cannot select dump session %d as base "
				      "for incremental dump: "
				      "level must be less than %d\n"),
				      sessp->s_level,
				      LEVEL_MAX);
				return BOOL_FALSE;
			}
			sc_level = (ix_t)sessp->s_level + 1;
			undertime = sessp->s_time;
			underlevel = (ix_t)sessp->s_level;
			uuid_copy (underid, sessp->s_sesid);
			underpartialpr = sessp->s_ispartial;
			underinterruptedpr = BOOL_FALSE;
			underfoundpr = BOOL_TRUE;
		}
		inv_free_session(&sessp);
		sessp = 0;
		ok = inv_close(inv_idbt);
		assert(ok);
		inv_idbt = INV_TOKEN_NULL;
		goto baseuuidbypass;
	}

	/* look for the most recent dump at a level less than the level
	 * of this dump. extract the time, level, id, and predicates partial
	 * and interrupted.
	 */
	underfoundpr = BOOL_FALSE;
	if (sc_level > 0) {
		if (inv_idbt == INV_TOKEN_NULL) {
			mlog(MLOG_NORMAL | MLOG_ERROR, _(
			      "cannot calculate incremental dump: "
			      "online inventory not available\n"));
			return BOOL_FALSE;
		}
		ok = inv_lastsession_level_lessthan(&fsid,
						    inv_idbt,
						    (u_char_t)sc_level,
						    &sessp);
		if (!ok) {
			sessp = 0;
		}

		if (sessp) {
			ix_t strix;
			ix_t strcnt;
			inv_stream_t *bsp;

			undertime = sessp->s_time;
			underlevel = (ix_t)sessp->s_level;
			uuid_copy (underid, sessp->s_sesid);
			underpartialpr = sessp->s_ispartial;
			underinterruptedpr = BOOL_FALSE;
			strcnt =  (ix_t)sessp->s_nstreams;
			for (strix = 0; strix < strcnt; strix++) {
				bsp = &sessp->s_streams[strix];
				if (bsp->st_interrupted) {
					underinterruptedpr = BOOL_TRUE;
					break;
				}
			}
			underfoundpr = BOOL_TRUE;
			inv_free_session(& sessp);
			sessp = 0;
		}
	}

	/* look for the most recent dump at a level equal to the level
	 * of this dump. extract the time, level, id, and predicates partial
	 * and interrupted, and for each stream the range of ino/offset
	 * values not dumped.
	 */
	if (inv_idbt != INV_TOKEN_NULL) {
		/* REFERENCED */
		bool_t ok1;
		ok = inv_lastsession_level_equalto(&fsid,
						   inv_idbt,
						   (u_char_t)sc_level,
						   &sessp);
		ok1 = inv_close(inv_idbt);
		assert(ok1);
		if (!ok) {
			sessp = 0;
		}
		inv_idbt = INV_TOKEN_NULL;
	} else {
		sessp = 0;
	}

	samefoundpr = BOOL_FALSE;
	if (sessp) {
		sametime = sessp->s_time;
		uuid_copy(sameid, sessp->s_sesid);
		samepartialpr = sessp->s_ispartial;
		sameinterruptedpr = BOOL_FALSE;
		sc_resumerangecnt =  (size_t)sessp->s_nstreams;
		sc_resumerangep = (drange_t *)calloc(sc_resumerangecnt,
						        sizeof(drange_t));
		assert(sc_resumerangep);
		for (strmix = 0; strmix < sc_resumerangecnt; strmix++) {
			inv_stream_t *bsp;
			inv_stream_t *esp;
			drange_t *p = &sc_resumerangep[strmix];
			bsp = &sessp->s_streams[strmix];
			esp = (strmix < sc_resumerangecnt - 1)
			      ?
			      bsp + 1
			      :
			      0;
			if (bsp->st_interrupted) {
				sameinterruptedpr = BOOL_TRUE;
				p->dr_begin.sp_ino = bsp->st_endino;
				p->dr_begin.sp_offset = bsp->st_endino_off;
				if (esp) {
					p->dr_end.sp_ino = esp->st_startino;
					p->dr_end.sp_offset =
							esp->st_startino_off;
					mlog(MLOG_DEBUG,
					      "resume range stream %u "
					      "ino %llu:%lld to "
					      "%llu:%lld\n",
					      strmix,
					      p->dr_begin.sp_ino,
					      p->dr_begin.sp_offset,
					      p->dr_end.sp_ino,
					      p->dr_end.sp_offset);
				} else {
					p->dr_end.sp_flags = STARTPT_FLAGS_END;
					mlog(MLOG_DEBUG,
					      "resume range stream %u "
					      "ino %llu:%lld to "
					      "end\n",
					      strmix,
					      p->dr_begin.sp_ino,
					      p->dr_begin.sp_offset);
				}
			} else {
				/* set the range start pt's END flag to
				 * indicate the range was not interrupted.
				 */
				p->dr_begin.sp_flags = STARTPT_FLAGS_END;
			}
		}
		inv_free_session(& sessp);
		sessp = 0;
		samefoundpr = BOOL_TRUE;
	}

baseuuidbypass:

	/* now determine the incremental and resume bases, if any.
	 */
	if (samefoundpr && !sameinterruptedpr) {
		free((void *)sc_resumerangep);
		sc_resumerangep = 0;
		samefoundpr = BOOL_FALSE;
	}
	if (samefoundpr && !resumereqpr) {
		if (!underfoundpr || undertime <= sametime) {
			mlog(MLOG_VERBOSE | MLOG_WARNING, _(
			      "most recent level %d dump "
			      "was interrupted, "
			      "but not resuming that dump since "
			      "resume (-R) option not specified\n"),
			      sc_level);
		}
		free((void *)sc_resumerangep);
		sc_resumerangep = 0;
		samefoundpr = BOOL_FALSE;
	}
	if (underfoundpr) {
		assert(underlevel <= LEVEL_MAX);
		assert(undertime);
		if (samefoundpr) {
			if (undertime >= sametime) {
				if (underinterruptedpr) {
					mlog(MLOG_NORMAL | MLOG_WARNING, _(
					      "most recent base for "
					      "incremental dump was "
					      "interrupted (level %u): "
					      "must resume or redump "
					      "at or below level %d\n"),
					      underlevel,
					      sc_level);
					return BOOL_FALSE;
				}
				if (subtreecnt && !underpartialpr) {
					mlog(MLOG_NORMAL | MLOG_WARNING, _(
					      "level %u incremental "
					      "subtree dump "
					      "will be based on non-subtree "
					      "level %u dump\n"),
					      sc_level,
					      underlevel);
				}
				if (!subtreecnt && underpartialpr) {
					mlog(MLOG_NORMAL | MLOG_WARNING, _(
					      "level %u incremental "
					      "non-subtree dump "
					      "will be based on subtree "
					      "level %u dump\n"),
					      sc_level,
					      underlevel);
				}
				sc_incrpr = BOOL_TRUE;
				sc_incrbasetime = undertime;
				sc_incrbaselevel = underlevel;
				uuid_copy(sc_incrbaseid, underid);
				sc_resumepr = BOOL_FALSE;
				assert(sc_resumerangep);
				free((void *)sc_resumerangep);
				sc_resumerangep = 0;
			} else {
				if (subtreecnt && !samepartialpr) {
					mlog(MLOG_NORMAL | MLOG_WARNING, _(
					      "level %u incremental "
					      "subtree dump "
					      "will be based on non-subtree "
					      "level %u resumed dump\n"),
					      sc_level,
					      sc_level);
				}
				if (!subtreecnt && samepartialpr) {
					mlog(MLOG_NORMAL | MLOG_WARNING, _(
					      "level %u incremental "
					      "non-subtree dump "
					      "will be based on subtree "
					      "level %u resumed dump\n"),
					      sc_level,
					      sc_level);
				}
				assert(sametime);
				sc_incrpr = BOOL_TRUE;
				sc_incrbasetime = undertime;
				sc_incrbaselevel = underlevel;
				sc_resumepr = BOOL_TRUE;
				sc_resumebasetime = sametime;
				uuid_copy(sc_resumebaseid, sameid);
				assert(sc_resumerangep);
			}
		} else {
			if (underinterruptedpr) {
				mlog(MLOG_NORMAL | MLOG_WARNING, _(
				      "most recent base for "
				      "incremental dump was "
				      "interrupted (level %u): "
				      "must resume or redump "
				      "at or below level %d\n"),
				      underlevel,
				      sc_level);
				return BOOL_FALSE;
			}
			if (subtreecnt && !underpartialpr) {
				mlog(MLOG_NORMAL | MLOG_WARNING, _(
				      "level %u incremental "
				      "subtree dump "
				      "will be based on non-subtree "
				      "level %u dump\n"),
				      sc_level,
				      underlevel);
			}
			if (!subtreecnt && underpartialpr) {
				mlog(MLOG_NORMAL | MLOG_WARNING, _(
				      "level %u incremental "
				      "non-subtree dump "
				      "will be based on subtree "
				      "level %u dump\n"),
				      sc_level,
				      underlevel);
			}
			sc_incrpr = BOOL_TRUE;
			sc_incrbasetime = undertime;
			sc_incrbaselevel = underlevel;
			uuid_copy(sc_incrbaseid, underid);
			sc_resumepr = BOOL_FALSE;
			assert(!sc_resumerangep);
		}
	} else {
		if (samefoundpr) {
			assert(sametime);
			if (subtreecnt && !samepartialpr) {
				mlog(MLOG_NORMAL | MLOG_WARNING, _(
				      "level %u "
				      "subtree dump "
				      "will be based on non-subtree "
				      "level %u resumed dump\n"),
				      sc_level,
				      sc_level);
			}
			if (!subtreecnt && samepartialpr) {
				mlog(MLOG_NORMAL | MLOG_WARNING, _(
				      "level %u "
				      "non-subtree dump "
				      "will be based on subtree "
				      "level %u resumed dump\n"),
				      sc_level,
				      sc_level);
			}
			sc_incrpr = BOOL_FALSE;
			sc_resumepr = BOOL_TRUE;
			sc_resumebasetime = sametime;
			uuid_copy(sc_resumebaseid, sameid);
			assert(sc_resumerangep);
		} else {
			sc_incrpr = BOOL_FALSE;
			sc_resumepr = BOOL_FALSE;
			assert(!sc_resumerangep);
			if (sc_level > 0) {
				mlog(MLOG_NORMAL | MLOG_ERROR, _(
				      "cannot find earlier dump "
				      "to base level %d increment upon\n"),
				      sc_level);
				return BOOL_FALSE;
			}
		}
	}

	/* don't allow interrupted dumps of a lesser level to be bases
	 */
	if (sc_incrpr && underinterruptedpr) {
		mlog(MLOG_NORMAL | MLOG_ERROR, _(
		      "most recent base dump (level %d begun %s) "
		      "was interrupted: aborting\n"),
		      sc_incrbaselevel,
		      ctimennl(&sc_incrbasetime));
		return BOOL_FALSE;
	}

	/* reject if resume (-R) specified, but base was not interrupted
	 */
	if (!sc_resumepr && resumereqpr) {
		mlog(MLOG_NORMAL | MLOG_ERROR, _(
		      "resume (-R) option inappropriate: "
		      "no interrupted level %d dump to resume\n"),
		      sc_level);
		return BOOL_FALSE;
	}

	/* announce the dump characteristics
	 */
	if (sc_incrpr) {
		if (sc_resumepr) {
			char restimestr[30];
			char incrtimestr[30];

			strcpy(restimestr, ctimennl(&sc_resumebasetime));
			assert(strlen(restimestr) < sizeof(restimestr));
			strcpy(incrtimestr, ctimennl(&sc_incrbasetime));
			assert(strlen(incrtimestr) < sizeof(incrtimestr));

			mlog(MLOG_VERBOSE, _(
			      "resuming level %d incremental dump of %s:%s "
			      "begun %s "
			      "(incremental base level %d begun %s)\n"),
			      sc_level,
			      gwhdrtemplatep->gh_hostname,
			      mntpnt,
			      restimestr,
			      sc_incrbaselevel,
			      incrtimestr);
		} else {
			mlog(MLOG_VERBOSE, _(
			      "level %d incremental dump of %s:%s "
			      "based on level %d dump begun %s\n"),
			      sc_level,
			      gwhdrtemplatep->gh_hostname,
			      mntpnt,
			      sc_incrbaselevel,
			      ctimennl(&sc_incrbasetime));
		}
	} else {
		if (sc_resumepr) {
			mlog(MLOG_VERBOSE, _(
			      "resuming level %d dump of %s:%s begun %s\n"),
			      sc_level,
			      gwhdrtemplatep->gh_hostname,
			      mntpnt,
			      ctimennl(&sc_resumebasetime));
		} else {
			mlog(MLOG_VERBOSE, _(
			      "level %d dump of %s:%s\n"),
			      sc_level,
			      gwhdrtemplatep->gh_hostname,
			      mntpnt);
		}
	}

	if (preemptchk(PREEMPT_FULL)) {
		return BOOL_FALSE;
	}

	/* announce the dump time
	 */
	mlog(MLOG_VERBOSE, _(
	      "dump date: %s\n"),
	      ctimennl(&gwhdrtemplatep->gh_timestamp));

	/* display the session UUID
	 */
	{
		char string_uuid[UUID_STR_LEN + 1];
		uuid_unparse(gwhdrtemplatep->gh_dumpid, string_uuid);
		mlog(MLOG_VERBOSE, _(
		      "session id: %s\n"),
		      string_uuid);
	}

	/* display the session label
	 */
	mlog(MLOG_VERBOSE, _(
	      "session label: \"%s\"\n"),
	      gwhdrtemplatep->gh_dumplabel);

	/* get a file descriptor for the file system. any file
	 * contained in the file system will do; use the mntpnt.
	 * needed by bigstat.
	 */
	sc_fsfd = open(mntpnt, O_RDONLY);
	if (sc_fsfd < 0) {
		mlog(MLOG_NORMAL, _(
		      "unable to open %s: %s\n"),
		      mntpnt,
		      strerror(errno));
		return BOOL_FALSE;
	}

	/* figure out the ino for the root directory of the fs
	 * and get its xfs_bstat_t for inomap_build()
	 */
	{
		stat64_t rootstat;
		rval = fstat64(sc_fsfd, &rootstat);
		if (rval) {
			mlog(MLOG_NORMAL, _(
			      "could not stat %s\n"),
			      mntpnt);
			return BOOL_FALSE;
		}

		if (!check_rootdir(sc_fsfd, rootstat.st_ino)) {
			mlog(MLOG_ERROR,
_("%s is not the root of the filesystem (bind mount?) - use primary mountpoint\n"),
			     mntpnt);
			return BOOL_FALSE;
		}

		sc_rootxfsstatp =
			(struct xfs_bstat *)calloc(1, sizeof(struct xfs_bstat));
		assert(sc_rootxfsstatp);

		if (bigstat_one(sc_fsfd, rootstat.st_ino, sc_rootxfsstatp) < 0) {
			mlog(MLOG_ERROR,
			      _("failed to get bulkstat information for root inode\n"));
			return BOOL_FALSE;
		}
	}

	/* alloc a file system handle, to be used with the jdm_open()
	 * functions.
	 */
	sc_fshandlep = jdm_getfshandle(mntpnt);
	if (!sc_fshandlep) {
		mlog(MLOG_NORMAL, _(
		      "unable to construct a file system handle for %s: %s\n"),
		      mntpnt,
		      strerror(errno));
		return BOOL_FALSE;
	}

	if (preemptchk(PREEMPT_FULL)) {
		return BOOL_FALSE;
	}

	/* If GETOPT_DUMPASOFFLINE was specified, allocate a filesystem context
	 * for use by the HSM routines.
	 */

	if (sc_dumpasoffline) {
		hsm_fs_ctxtp = HsmInitFsysContext(mntpnt, HSM_API_VERSION_1);
	}

	/* set now so statline can be displayed
	 */
	sc_stat_starttime = gwhdrtemplatep->gh_timestamp;

	/* allocate storage for the stream startpoints, and build inomap.
	 * inomap_build() also fills in the start points. storage only needed
	 * until the startpoints are copied into each streams header. will
	 * be freed at the end of this function.
	 */
	sc_stat_inomapcnt = (size64_t)fs_getinocnt(mntpnt);

	sc_startptp = (startpt_t *)calloc(drivecnt, sizeof(startpt_t));
	assert(sc_startptp);
	ok = inomap_build(sc_fshandlep,
			   sc_fsfd,
			   sc_rootxfsstatp,
			   sc_incrpr,
			   sc_incrbasetime,
			   sc_resumepr,
			   sc_resumebasetime,
			   sc_resumerangecnt,
			   sc_resumerangep,
			   subtreep,
			   subtreecnt,
			   skip_unchanged_dirs,
			   sc_startptp,
			   drivecnt,
			   &sc_stat_inomapphase,
			   &sc_stat_inomappass,
			   sc_stat_inomapcnt,
			   &sc_stat_inomapdone);
	free((void *)subtreep);
	subtreep = 0;
	if (!ok) {
		return BOOL_FALSE;
	}

	/* ask var to ask inomap to skip files under var if var is in
	 * the fs being dumped
	 */
	var_skip(&fsid, inomap_skip);

	/* fill in write header template content info. always produce
	 * an inomap for each media file. the dirdump flag will be set
	 * in content_stream_dump() for streams which dump the directories.
	 */
	assert(sizeof(cwhdrtemplatep->ch_specific) >= sizeof(*scwhdrtemplatep));
	scwhdrtemplatep->cih_mediafiletype = CIH_MEDIAFILETYPE_DATA;
	scwhdrtemplatep->cih_level = (int32_t)sc_level;
	scwhdrtemplatep->cih_dumpattr = CIH_DUMPATTR_INOMAP;
	if (subtreecnt) {
		scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_SUBTREE;
	}
	if (sc_inv_updatepr) {
		scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_INVENTORY;
	}
	scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_FILEHDR_CHECKSUM;
	scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_EXTENTHDR_CHECKSUM;
	scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_DIRENTHDR_CHECKSUM;
	scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_DIRENTHDR_GEN;
	if (sc_incrpr) {
		scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_INCREMENTAL;
		scwhdrtemplatep->cih_last_time = sc_incrbasetime;
		uuid_copy(scwhdrtemplatep->cih_last_id, sc_incrbaseid);
		if (skip_unchanged_dirs) {
			scwhdrtemplatep->cih_dumpattr |=
				CIH_DUMPATTR_NOTSELFCONTAINED;
		}
	}
	if (sc_resumepr) {
		scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_RESUME;
		scwhdrtemplatep->cih_resume_time = sc_resumebasetime;
		uuid_copy(scwhdrtemplatep->cih_resume_id, sc_resumebaseid);
	}
	if (sc_dumpextattrpr) {
		scwhdrtemplatep->cih_dumpattr |= CIH_DUMPATTR_EXTATTR;
		scwhdrtemplatep->cih_dumpattr |=
					CIH_DUMPATTR_EXTATTRHDR_CHECKSUM;
	}

	scwhdrtemplatep->cih_rootino = sc_rootxfsstatp->bs_ino;
	inomap_writehdr(scwhdrtemplatep);

	/* log the dump size. just a rough approx.
	 */
	dircnt = scwhdrtemplatep->cih_inomap_dircnt;
	nondircnt = scwhdrtemplatep->cih_inomap_nondircnt;
	datasz = scwhdrtemplatep->cih_inomap_datasz;
	inocnt = dircnt + nondircnt;
	inomapsz = inomap_getsz();
	direntsz = inocnt * (uint64_t)(DIRENTHDR_SZ + 8);
	filesz = inocnt * (uint64_t)(FILEHDR_SZ + EXTENTHDR_SZ);

	hdr_mfilesz =	GLOBAL_HDR_SZ
			+
			inomapsz
			+
			direntsz;
	size_estimate = hdr_mfilesz
			+
			filesz
			+
			datasz;
	mlog(MLOG_VERBOSE, _(
	      "estimated dump size: %llu bytes\n"),
	      size_estimate);

	if (drivecnt > 1) {
	    mlog(MLOG_VERBOSE, _(
		  "estimated dump size per stream: %llu bytes\n"),
		    hdr_mfilesz + (filesz + datasz) / drivecnt);
	}
	mlog(MLOG_DEBUG,
	      "estimated dump header size: %llu bytes\n",
	      hdr_mfilesz);
	mlog(MLOG_DEBUG,
	      "estimated component sizes: global hdr: %llu bytes, "
	      "inomap: %llu bytes,  dir entries: %llu bytes, "
	      "file hdrs: %llu bytes, datasz: %llu bytes\n",
	      GLOBAL_HDR_SZ, inomapsz, direntsz,
	      filesz, datasz);

	/* extract the progress stat denominators from the write hdr
	 * template. placed there by inomap_writehdr()
	 */
	sc_stat_dircnt = scwhdrtemplatep->cih_inomap_dircnt;
	sc_stat_nondircnt = scwhdrtemplatep->cih_inomap_nondircnt;
	sc_stat_datasz = scwhdrtemplatep->cih_inomap_datasz;

	/* allocate and populate per-stream context descriptors
	 */
	sc_contextp = (context_t *)calloc(drivecnt, sizeof(context_t));
	assert(sc_contextp);
	for (strmix = 0; strmix < drivecnt; strmix++) {
		context_t *contextp = &sc_contextp[strmix];

		contextp->cc_filehdrp =
				(filehdr_t *)calloc(1, sizeof(filehdr_t));
		assert(contextp->cc_filehdrp);

		contextp->cc_extenthdrp =
			    (extenthdr_t *)calloc(1, sizeof(extenthdr_t));
		assert(contextp->cc_extenthdrp);

		contextp->cc_getdentsbufsz = sizeof(struct dirent)
					       +
					       NAME_MAX + 1;
		if (contextp->cc_getdentsbufsz < GETDENTSBUF_SZ_MIN) {
			contextp->cc_getdentsbufsz = GETDENTSBUF_SZ_MIN;
		}
		contextp->cc_getdentsbufp =
			   (char *) calloc(1, contextp->cc_getdentsbufsz);
		assert(contextp->cc_getdentsbufp);

		contextp->cc_mdirentbufsz = sizeof(direnthdr_t)
					    +
					    NAME_MAX + 1
					    +
					    DIRENTHDR_ALIGN;
		contextp->cc_mdirentbufp =
			   (char *) calloc(1, contextp->cc_mdirentbufsz);
		assert(contextp->cc_mdirentbufp);

		contextp->cc_extattrlistbufsz = EXTATTR_LISTBUF_SZ;
		contextp->cc_extattrrtrvarraylen = EXTATTR_RTRVARRAY_LEN;
		contextp->cc_extattrdumpbufsz = 2 * ATTR_MAX_VALUELEN;
		if (contextp->cc_extattrdumpbufsz < EXTATTR_DUMPBUF_SZ) {
			contextp->cc_extattrdumpbufsz = EXTATTR_DUMPBUF_SZ;
		}
		contextp->cc_extattrlistbufp =
			   (char *)calloc(1, contextp->cc_extattrlistbufsz);
		assert(contextp->cc_extattrlistbufp);
		contextp->cc_extattrrtrvarrayp =
		  (attr_multiop_t *)calloc(contextp->cc_extattrrtrvarraylen,
				    sizeof(attr_multiop_t));
		assert(contextp->cc_extattrrtrvarrayp);
		contextp->cc_extattrdumpbufp =
			   (char *)memalign(sizeof(extattrhdr_t),
					       contextp->cc_extattrdumpbufsz);
		assert(contextp->cc_extattrdumpbufp);
		if (hsm_fs_ctxtp) {
			contextp->cc_hsm_f_ctxtp = HsmAllocateFileContext(
				hsm_fs_ctxtp);
		} else {
			contextp->cc_hsm_f_ctxtp = NULL;
		}

		contextp->cc_readlinkbufsz = MAXPATHLEN + SYMLINK_ALIGN;
		contextp->cc_readlinkbufp =
			   (char *) calloc(1, contextp->cc_readlinkbufsz);
		assert(contextp->cc_readlinkbufp);

		contextp->cc_inomap_contextp = inomap_alloc_context();
	}

	/* look for command line media labels. these will be assigned
	 * to each stream as found. this label is only for the media
	 * object currently in the drive. subsequently inserted media
	 * objects must get a label via prompting.
	 */

	{
		context_t *cp = sc_contextp;
		context_t *ep = sc_contextp + drivecnt;

		optind = 1;
		opterr = 0;
		while ((c = getopt(argc, argv, GETOPT_CMDSTRING)) != EOF) {
			switch (c) {
			case GETOPT_MEDIALABEL:
				if (cp >= ep) {
					mlog(MLOG_NORMAL, _(
					      "more -%c arguments "
					      "than number of drives\n"),
					      c);
					usage();
					return BOOL_FALSE;
				}
				if (!optarg || optarg[0] == '-') {
					mlog(MLOG_NORMAL, _(
					      "-%c argument missing\n"),
					      c);
					usage();
					return BOOL_FALSE;
				}
				cp->cc_Media_firstlabel = optarg;
				cp++;
				break;
			}
		}

		if (cp > sc_contextp && cp < ep) {
			mlog(MLOG_NORMAL | MLOG_WARNING, _(
			      "media labels given for only %d out of %d "
			      "drives\n"),
			      cp - sc_contextp,
			      drivecnt);
		}
	}

	if (preemptchk(PREEMPT_FULL)) {
		return BOOL_FALSE;
	}

	/* open the dump inventory and a dump inventory write session
	 * if an inventory update is to be done.
	 */
	if (sc_inv_updatepr) {
		bool_t result;
		sigset_t tty_set, orig_set;

		/* hold tty signals while creating a new inventory session */
		sigemptyset(&tty_set);
		sigaddset(&tty_set, SIGINT);
		sigaddset(&tty_set, SIGQUIT);
		sigaddset(&tty_set, SIGHUP);
		pthread_sigmask(SIG_BLOCK, &tty_set, &orig_set);

		result = create_inv_session(gwhdrtemplatep, &fsid, mntpnt,
					     fsdevice, subtreecnt, strmix);

		pthread_sigmask(SIG_SETMASK, &orig_set, NULL);

		if (!result) {
			return BOOL_FALSE;
		}
	}

	/* set media change flags to FALSE;
	 */
	{
		ix_t ix;
		ix_t endix = sizeof(sc_mcflag)
			     /
			     sizeof(sc_mcflag[0]);
		for (ix = 0; ix < endix; ix++) {
			sc_mcflag[ix] = BOOL_FALSE;
		}
	}
	content_media_change_needed = BOOL_FALSE;

	/* initialize the per-drive status
	 */
	{
		ix_t driveix;
		for (driveix = 0; driveix < STREAM_SIMMAX; driveix++) {
			sc_stat_pds[driveix].pds_phase = PDS_NULL;
		}
	}

	return BOOL_TRUE;
}

#define STATLINESZ	160

size_t
content_statline(char **linespp[])
{
	static char statlinebuf[STREAM_SIMMAX + 1][STATLINESZ];
	static char *statline[STREAM_SIMMAX + 1];
	size_t statlinecnt;
	size64_t nondirdone;
	size64_t datadone;
	double percent;
	time_t elapsed;
	time_t now;
	struct tm *tmp;
	ix_t i;

	/* build and supply the line array
	 */
	for (i = 0; i < STREAM_SIMMAX + 1; i++) {
		statline[i] = &statlinebuf[i][0];
	}
	*linespp = statline;
	statlinecnt = 0;

	/* if start time not initialized, return no strings
	 */
	if (!sc_stat_starttime) {
		return 0;
	}

	/* calculate the elapsed time
	 */
	now = time(0);
	elapsed = now - sc_stat_starttime;

	/* get local time
	 */
	tmp = localtime(&now);

	/* if inomap phase indicated, report on that
	 */
	if (sc_stat_inomapphase && sc_stat_inomapcnt) {
		if (sc_stat_inomappass) {
			sprintf(statline[0],
				 "status at %02d:%02d:%02d: "
				 "inomap phase %u pass %u "
				 "%llu/%llu inos scanned, "
				 "%ld seconds elapsed\n",
				 tmp->tm_hour,
				 tmp->tm_min,
				 tmp->tm_sec,
				 (unsigned int)sc_stat_inomapphase,
				 (unsigned int)sc_stat_inomappass,
				 (unsigned long long)sc_stat_inomapdone,
				 (unsigned long long)sc_stat_inomapcnt,
				 elapsed);
			assert(strlen(statline[0]) < STATLINESZ);
		} else {
			sprintf(statline[0],
				 "status at %02d:%02d:%02d: "
				 "inomap phase %u "
				 "%llu/%llu inos scanned, "
				 "%ld seconds elapsed\n",
				 tmp->tm_hour,
				 tmp->tm_min,
				 tmp->tm_sec,
				 (unsigned int)sc_stat_inomapphase,
				 (unsigned long long)sc_stat_inomapdone,
				 (unsigned long long)sc_stat_inomapcnt,
				 elapsed);
			assert(strlen(statline[0]) < STATLINESZ);
		}
		return 1;
	}

	/* get the accumulated totals for non-dir inos and data bytes dumped
	 */
	lock();
	nondirdone = sc_stat_nondirdone;
	datadone = sc_stat_datadone;
	unlock();

	/* non-dir dump phase */
	if (nondirdone || datadone) {
		/* calculate percentage of data dumped
		*/
		if (sc_stat_datasz) {
			percent = (double)datadone
				/
				(double)sc_stat_datasz;
			percent *= 100.0;
		} else {
			percent = 100.0;
		}
		if (percent > 100.0) {
			percent = 100.0;
		}

		/* format the status line in a local static buffer (non-re-entrant!)
		*/
		sprintf(statline[0],
				"status at %02d:%02d:%02d: %llu/%llu files dumped, "
				"%.1lf%%%% data dumped, "
				"%ld seconds elapsed\n",
				tmp->tm_hour,
				tmp->tm_min,
				tmp->tm_sec,
				(unsigned long long) nondirdone,
				(unsigned long long) sc_stat_nondircnt,
				percent,
				elapsed);
	} else {
		sprintf(statline[0],
				"status at %02d:%02d:%02d: "
				"%ld seconds elapsed\n",
				tmp->tm_hour,
				tmp->tm_min,
				tmp->tm_sec,
				elapsed);
	}

	assert(strlen(statline[0]) < STATLINESZ);

	/* optionally create stat lines for each drive
	 */
	statlinecnt = 1;
	for (i = 0; i < drivecnt; i++) {
		pds_t *pdsp = &sc_stat_pds[i];
		if (pdsp->pds_phase == PDS_NULL
		     ||
		     pdsp->pds_phase == PDS_NONDIR) {
			continue;
		}
		statline[statlinecnt][0] = 0;
		if (drivecnt > 1) {
			sprintf(statline[statlinecnt],
				 "drive %u: ",
				 (unsigned int)i);
		}
		switch(pdsp->pds_phase) {
		case PDS_INOMAP:
			strcat(statline[statlinecnt],
				"dumping inomap");
			break;
		case PDS_DIRDUMP:
			sprintf(&statline[statlinecnt]
					  [strlen(statline[statlinecnt])],
				 "%llu/%llu directories dumped",
				 (unsigned long long)pdsp->pds_dirdone,
				 (unsigned long long)sc_stat_dircnt);
			break;
		case PDS_INVSYNC:
			strcat(statline[statlinecnt],
				"waiting to dump inventory");
			break;
		case PDS_INVDUMP:
			strcat(statline[statlinecnt],
				"dumping inventory");
			break;
		case PDS_TERMDUMP:
			strcat(statline[statlinecnt],
				"dumping stream terminator");
			break;
		default:
			break;
		}
		sprintf(&statline[statlinecnt]
				  [strlen(statline[statlinecnt])],
			 "\n");
		assert(strlen(statline[statlinecnt]) < STATLINESZ);
		statlinecnt++;
	}

	return statlinecnt;
}

static bool_t
create_inv_session(
		global_hdr_t *gwhdrtemplatep,
		uuid_t *fsidp,
		const char *mntpnt,
		const char *fsdevice,
		ix_t subtreecnt,
		size_t strmix)
{
	int rval;
	char *qmntpnt;
	char *qfsdevice;

	/* create a cleanup handler to close the inventory on exit. */
	rval = atexit(inv_cleanup);
	assert(!rval);

	sc_inv_idbtoken = inv_open((inv_predicate_t)INV_BY_UUID,
					INV_SEARCH_N_MOD,
					(void *)fsidp);
	if (sc_inv_idbtoken == INV_TOKEN_NULL) {
		return BOOL_FALSE;
	}
	qmntpnt = (char *)calloc(1, strlen(gwhdrtemplatep->gh_hostname)
					+ 1 + strlen(mntpnt) + 1);
	assert(qmntpnt);
	assert(strlen(gwhdrtemplatep->gh_hostname));
	sprintf(qmntpnt, "%s:%s", gwhdrtemplatep->gh_hostname, mntpnt);
	qfsdevice = (char *)calloc(1, strlen(gwhdrtemplatep->gh_hostname)
					 + 1 + strlen(fsdevice) + 1);
	assert(qfsdevice);
	sprintf(qfsdevice, "%s:%s", gwhdrtemplatep->gh_hostname, fsdevice);

	sc_inv_sestoken = inv_writesession_open(sc_inv_idbtoken,
						fsidp,
						&gwhdrtemplatep->gh_dumpid,
						gwhdrtemplatep->gh_dumplabel,
						subtreecnt ? BOOL_TRUE
							   : BOOL_FALSE,
						sc_resumepr,
						(u_char_t)sc_level,
						drivecnt,
						gwhdrtemplatep->gh_timestamp,
						qmntpnt,
						qfsdevice);
	if (sc_inv_sestoken == INV_TOKEN_NULL) {
		return BOOL_FALSE;
	}

	/* open an inventory stream for each stream
	*/
	sc_inv_stmtokenp = (inv_stmtoken_t *)
				calloc(drivecnt, sizeof(inv_stmtoken_t));
	assert(sc_inv_stmtokenp);
	for (strmix = 0; strmix < drivecnt; strmix++) {
		drive_t *drivep = drivepp[strmix];
		char *drvpath;

		if (strcmp(drivep->d_pathname, "stdio")) {
			drvpath = path_reltoabs(drivep->d_pathname, homedir);
		} else {
			drvpath = drivep->d_pathname;
		}
		sc_inv_stmtokenp[strmix] = inv_stream_open(sc_inv_sestoken,
								drvpath);
		if (strcmp(drivep->d_pathname, "stdio")) {
			free((void *)drvpath);
		}
		if (sc_inv_stmtokenp[strmix] == INV_TOKEN_NULL) {
			return BOOL_FALSE;
		}
	}

	return BOOL_TRUE;
}

static void
mark_set(drive_t *drivep, xfs_ino_t ino, off64_t offset, int32_t flags)
{
	drive_ops_t *dop = drivep->d_opsp;
	mark_t *markp = (mark_t *)calloc(1, sizeof(mark_t));
	assert(markp);

	if (flags & STARTPT_FLAGS_NULL) {
		mlog(MLOG_DEBUG,
		      "setting media NULL mark\n");
	} else if (flags & STARTPT_FLAGS_END) {
		mlog(MLOG_DEBUG,
		      "setting media END mark\n");
	} else {
		mlog(MLOG_DEBUG,
		      "setting media mark"
		      " for ino %llu offset %lld\n",
		      ino,
		      offset);
	}

	markp->startpt.sp_ino = ino;
	markp->startpt.sp_offset = offset;
	markp->startpt.sp_flags = flags;
	(*dop->do_set_mark)(drivep,
				mark_callback,
				(void *)drivep->d_index,
				(drive_markrec_t *)markp);
}

static void
mark_callback(void *p, drive_markrec_t *dmp, bool_t committed)
{
	/* get context
	 */
	ix_t strmix = (ix_t)p;
	context_t *contextp = &sc_contextp[strmix];
	drive_t *drivep = drivepp[strmix];
	drive_hdr_t *dwhdrp = drivep->d_writehdrp;
	media_hdr_t *mwhdrp = (media_hdr_t *)dwhdrp->dh_upper;
	content_hdr_t *cwhdrp = (content_hdr_t *)mwhdrp->mh_upper;
	content_inode_hdr_t *scwhdrp = (content_inode_hdr_t *)
				       (void *)
				       cwhdrp->ch_specific;

	/* this is really a mark_t, allocated by mark_set()
	 */
	mark_t *markp = (mark_t *)dmp;

	if (committed) {
		/* bump the per-mfile mark committed count
		 */
		contextp->cc_markscommitted++;

		/* copy the mark into the write header: this establishes the
		 * starting point should we need to retry the non-dir portion
		 * of the dump
		 */

		/* log the mark commit
		 */
		if (markp->startpt.sp_flags & STARTPT_FLAGS_NULL) {
			mlog(MLOG_DEBUG,
			      "media NULL mark committed"
			      " in media file %d\n",
			      mwhdrp->mh_dumpfileix);
			scwhdrp->cih_startpt.sp_flags |= STARTPT_FLAGS_NULL;
		} else if (markp->startpt.sp_flags & STARTPT_FLAGS_END) {
			mlog(MLOG_DEBUG,
			      "media END mark committed"
			      " in media file %d\n",
			      mwhdrp->mh_dumpfileix);
			if (scwhdrp->cih_endpt.sp_flags & STARTPT_FLAGS_END) {
				scwhdrp->cih_startpt.sp_ino++;
				scwhdrp->cih_startpt.sp_offset = 0;
			} else {
				scwhdrp->cih_startpt = scwhdrp->cih_endpt;
			}
			scwhdrp->cih_startpt.sp_flags |= STARTPT_FLAGS_END;
		} else {
			mlog(MLOG_DEBUG,
			      "media mark committed"
			      " for ino %llu offset %lld"
			      " in media file %d\n",
			      markp->startpt.sp_ino,
			      markp->startpt.sp_offset,
			      mwhdrp->mh_dumpfileix);
			scwhdrp->cih_startpt = markp->startpt;
		}
	} else {
		/* note the mark was not committed
		 */
		if (markp->startpt.sp_flags & STARTPT_FLAGS_NULL) {
			mlog(MLOG_DEBUG,
			      "media NULL mark -NOT- committed\n");
		} else if (markp->startpt.sp_flags & STARTPT_FLAGS_END) {
			mlog(MLOG_DEBUG,
			      "media END mark -NOT- committed\n");
		} else {
			mlog(MLOG_DEBUG,
			      "media mark -NOT- committed"
			      " for ino %llu offset %lld\n",
			      markp->startpt.sp_ino,
			      markp->startpt.sp_offset);
		}
	}

	/* get rid of this mark (it was allocated by mark_set())
	 */
	free((void *)markp);
}

/* begin - called by stream process to invoke the dump stream
 */
int
content_stream_dump(ix_t strmix)
{
	context_t *contextp = &sc_contextp[strmix];
	drive_t *drivep = drivepp[strmix];
	drive_hdr_t *dwhdrp = drivep->d_writehdrp;
	media_hdr_t *mwhdrp = (media_hdr_t *)dwhdrp->dh_upper;
	content_hdr_t *cwhdrp = (content_hdr_t *)mwhdrp->mh_upper;
	content_inode_hdr_t *scwhdrp = (content_inode_hdr_t *)
				       cwhdrp->ch_specific;
	void *inomap_contextp;
	bool_t all_nondirs_committed;
	bool_t empty_mediafile;
	time_t elapsed;
	inv_stmtoken_t inv_stmt;
	struct xfs_bstat *bstatbufp;
	const size_t bstatbuflen = BSTATBUFLEN;
	int rval;
	rv_t rv;

	/* sanity checks
	 */
	assert(RV_OK == 0); /* bigstat_iter depends on this */

	/* allocate a buffer for use by bstat_iter
	 */
	bstatbufp = (struct xfs_bstat *)calloc(bstatbuflen,
					     sizeof(struct xfs_bstat));
	assert(bstatbufp);

	/* allocate an inomap context */
	inomap_contextp = inomap_alloc_context();
	assert(inomap_contextp);

	/* determine if stream terminators will be used and are expected.
	 * this will be revised each time a new media file is begun.
	 */
	update_cc_Media_useterminatorpr(drivep, contextp);

	/* check in
	 */
	lock();
	sc_thrdsarrivedcnt++;
	unlock();

	/* fill in write hdr stream start and end points
	 */
	scwhdrp->cih_startpt = sc_startptp[strmix];
	if (strmix < drivecnt - 1) {
		scwhdrp->cih_endpt = sc_startptp[strmix + 1];
	} else {
		scwhdrp->cih_endpt.sp_flags = STARTPT_FLAGS_END;
	}

	// the first stream dumps the directories
	if (strmix == 0) {
		scwhdrp->cih_dumpattr |= CIH_DUMPATTR_DIRDUMP;
	}

	/* fill in inomap fields of write hdr
	 */
	inomap_writehdr(scwhdrp);

	/* used to decide if any non-dirs not yet on media
	 */
	all_nondirs_committed = BOOL_FALSE;

	/* used to guarantee we don't count the same ino more than once
	 * in the progress stats
	 */
	contextp->cc_stat_lastino = 0;

	/* used to detect generation of an empty media file;
	 * contains at most an inomap and dirdump and null file hdr.
	 */
	empty_mediafile = BOOL_FALSE;

	/* get the inventory stream token
	 */
	if (sc_inv_stmtokenp) {
		inv_stmt = sc_inv_stmtokenp[strmix];
	} else {
		inv_stmt = INV_TOKEN_NULL;
	}

	/* loop, dumping media files, until the entire stream is dumped.
	 * each time we hit EOM/EOF, repeat the inomap and directory dump.
	 * dump the non-dirs beginning with the current startpoint.
	 * The current startpoint will be updated each time a media mark
	 * is committed.
	 */
	for (;;) {
		xfs_ino_t startino;
		bool_t stop_requested;
		bool_t hit_eom;
		bool_t all_dirs_committed;
		bool_t all_nondirs_sent;
		off64_t startoffset;
		off64_t ncommitted;
		bool_t done;

		/* used to decide whether or not to go back for more.
		 */
		stop_requested = BOOL_FALSE;

		/* TRUE if hit EOM while dumping
		 */
		hit_eom = BOOL_FALSE;

		/* used to decide if the media file contains all
		 * of the inomap and dirdump.
		 */
		all_dirs_committed = BOOL_FALSE;

		/* used to decide if all non-dirs were sent (not necessarily
		 * committed)
		 */
		all_nondirs_sent = BOOL_FALSE;

		/* always clear the NULL flag from the stream startpoint
		 * before beginning the media file. allows detection
		 * of null file hdr commit.
		 */
		scwhdrp->cih_startpt.sp_flags &= ~STARTPT_FLAGS_NULL;

		/* save the original start points, to be given to
		 * the inventory at the end of each media file.
		 */
		startino = scwhdrp->cih_startpt.sp_ino;
		startoffset = scwhdrp->cih_startpt.sp_offset;

		/* set the accumulated file size to zero.
		 * this will be monitored by dump_file() to decide
		 * if the current dump file is too long. if so,
		 * it will set a startpoint and spoof an EOF.
		 * this will cause a new dump file to be started,
		 * beginning at the startpoint.
		 */
		contextp->cc_mfilesz = 0;

		/* tell the Media abstraction to position a media object
		 * and begin a new media file. This will dump the media
		 * file header if successful.
		 */
		rv = Media_mfile_begin(drivep, contextp, BOOL_TRUE);
		if (rv == RV_INTR) {
			return mlog_exit(EXIT_INTERRUPT, rv);
		}
		if (rv == RV_TIMEOUT) {
			mlog(MLOG_VERBOSE | MLOG_WARNING, _(
			      "media change timeout will be treated as "
			      "a request to stop using drive: "
			      "can resume later\n"));
			mlog_exit_hint(RV_QUIT);
			return mlog_exit(EXIT_NORMAL, rv);
		}
		if (rv == RV_QUIT) {
			mlog(MLOG_VERBOSE | MLOG_WARNING, _(
			      "media change decline will be treated as "
			      "a request to stop using drive: "
			      "can resume later\n"));
			mlog_exit_hint(RV_QUIT);
			return mlog_exit(EXIT_NORMAL, rv);
		}
		if (rv == RV_DRIVE) {
			return mlog_exit(EXIT_NORMAL, rv);
		}
		if (rv == RV_ERROR) {
			return mlog_exit(EXIT_ERROR, rv);
		}
		if (rv == RV_CORE) {
			return mlog_exit(EXIT_FAULT, rv);
		}
		assert(rv == RV_OK);
		if (rv != RV_OK) {
			return mlog_exit(EXIT_FAULT, rv);
		}

		/* sync up here with other streams if reasonable
		 */
		mlog(MLOG_VERBOSE, _(
		      "creating dump session media file %u "
		      "(media %u, file %u)\n"),
		      mwhdrp->mh_dumpfileix,
		      mwhdrp->mh_mediaix,
		      mwhdrp->mh_mediafileix);

		/* initialize the count of marks committed in the media file.
		 * will be bumped by mark_callback().
		 */
		contextp->cc_markscommitted = 0;

		/* first dump the inomap
		 */
		mlog(MLOG_VERBOSE, _(
		      "dumping ino map\n"));
		sc_stat_pds[strmix].pds_phase = PDS_INOMAP;
		rv = inomap_dump(drivep);
		if (rv == RV_INTR) {
			stop_requested = BOOL_TRUE;
			goto decision_more;
		}
		if (rv == RV_EOM) {
			hit_eom = BOOL_TRUE;
			goto decision_more;
		}
		if (rv == RV_DRIVE) {
			free((void *)bstatbufp);
			return mlog_exit(EXIT_NORMAL, rv);
		}
		if (rv == RV_ERROR) {
			free((void *)bstatbufp);
			return mlog_exit(EXIT_ERROR, rv);
		}
		if (rv == RV_CORE) {
			free((void *)bstatbufp);
			return mlog_exit(EXIT_FAULT, rv);
		}
		assert(rv == RV_OK);
		if (rv != RV_OK) {
			free((void *)bstatbufp);
			return mlog_exit(EXIT_FAULT, rv);
		}

		/* now dump the directories, if this is a stream that dumps
		 * directories. use the bigstat iterator capability to call
		 * my dump_dir function for each directory in the bitmap.
		 */
		if (scwhdrp->cih_dumpattr & CIH_DUMPATTR_DIRDUMP) {
			sc_stat_pds[strmix].pds_dirdone = 0;
			rv = dump_dirs(strmix,
					bstatbufp,
					bstatbuflen,
					inomap_contextp);
			if (rv == RV_INTR) {
				stop_requested = BOOL_TRUE;
				goto decision_more;
			}
			if (rv == RV_EOM) {
				hit_eom = BOOL_TRUE;
				goto decision_more;
			}
			if (rv == RV_DRIVE) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_NORMAL, rv);
			}
			if (rv == RV_ERROR) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_ERROR, rv);
			}
			if (rv == RV_CORE) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_FAULT, rv);
			}
			assert(rv == RV_OK);
			if (rv != RV_OK) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_FAULT, rv);
			}
		}

		/* finally, dump the non-directory files beginning with this
		 * stream's startpoint. Note that dump_file will set one or
		 * more media marks; the callback will update the hdr's
		 * startpoint; thus each time a demarcated portion of a
		 * non-directory file is fully committed to media,
		 * the starting point for the next media file will be advanced.
		 */
		if (!all_nondirs_committed) {
			mlog(MLOG_VERBOSE, _(
			      "dumping non-directory files\n"));
			sc_stat_pds[strmix].pds_phase = PDS_NONDIR;
			rv = RV_OK;
			inomap_reset_context(inomap_contextp);
			rval = bigstat_iter(sc_fshandlep,
					     sc_fsfd,
					     BIGSTAT_ITER_NONDIR,
					     scwhdrp->cih_startpt.sp_ino,
					     (bstat_cbfp_t)dump_file,
					     (void *)strmix,
					     inomap_next_nondir,
					     inomap_contextp,
					     (int *)&rv,
					     pipeline ?
					       (bool_t (*)(int))preemptchk : 0,
					     bstatbufp,
					     bstatbuflen);
			if (rval) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_FAULT, RV_CORE);
			}
			if (rv == RV_INTR) {
				stop_requested = BOOL_TRUE;
				goto decision_more;
			}
			if (rv == RV_EOM) {
				hit_eom = BOOL_TRUE;
				goto decision_more;
			}
			if (rv == RV_EOF) {
				goto decision_more;
			}
			if (rv == RV_DRIVE) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_NORMAL, rv);
			}
			if (rv == RV_ERROR) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_ERROR, rv);
			}
			if (rv == RV_CORE) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_FAULT, rv);
			}
			assert(rv == RV_OK || rv == RV_NOMORE);
			if (rv != RV_OK && rv != RV_NOMORE) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_FAULT, rv);
			}
		}

		/* if we got here, all files were sent without hitting
		 * the end of the current media object, or hitting the
		 * media file size limit. send the special END mark.
		 * this is only send at the end of the last media file in the
		 * dump session. if it actually gets committed to media,
		 * we know the last data written to media all made it.
		 * we won't know if this mark is committed to media until
		 * we attempt to end the write stream.
		 */
		all_nondirs_sent = BOOL_TRUE;
		mark_set(drivep,
			  INO64MAX,
			  OFF64MAX,
			  STARTPT_FLAGS_END);

decision_more:
		/* write a null file hdr, to let restore recognize
		 * the end of the media file. the flags indicate
		 * whether or not this is intended to be the last
		 * media file in the stream. don't bother if we hit
		 * EOM.
		 */
		if (!hit_eom) {
			rv = dump_filehdr(drivep,
					   contextp,
					   0,
					   0,
					   all_nondirs_sent
					   ?
					   (FILEHDR_FLAGS_NULL
					     |
					     FILEHDR_FLAGS_END)
					   :
					   FILEHDR_FLAGS_NULL);
			if (rv == RV_DRIVE) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_NORMAL, rv);
			}
			if (rv == RV_CORE) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_FAULT, rv);
			}
			if (rv == RV_ERROR) {
				free((void *)bstatbufp);
				return mlog_exit(EXIT_ERROR, rv);
			}

			/* send a mark to detect if the null file header made
			 * it. mark callback will adjust start pt before this
			 * call returns if the null file header made it.
			 */
			mark_set(drivep,
				  INO64MAX,
				  OFF64MAX,
				  all_nondirs_sent
				  ?
				  STARTPT_FLAGS_NULL | STARTPT_FLAGS_END
				  :
				  STARTPT_FLAGS_NULL);
		}

		/* tell the Media abstraction to end the media file.
		 * this is done before the inventory update, to
		 * see how much was actually committed to media.
		 * will invoke drive end_write, which will flush
		 * all pending marks.
		 */
		mlog(MLOG_VERBOSE, _(
		      "ending media file\n"));
		ncommitted = 0;
		rv = Media_mfile_end(drivep,
				      contextp,
				      mwhdrp,
				      &ncommitted,
				      hit_eom);
		if (rv == RV_DRIVE) {
			free((void *)bstatbufp);
			return mlog_exit(EXIT_NORMAL, rv);
		}
		if (rv == RV_CORE) {
			free((void *)bstatbufp);
			return mlog_exit(EXIT_FAULT, rv);
		}
		mlog(MLOG_VERBOSE, _(
		      "media file size %lld bytes\n"),
		      ncommitted);

		/* if at least one mark committed, we know all of
		 * the inomap and dirdump was committed.
		 */
		all_dirs_committed = (contextp->cc_markscommitted > 0);

		/* at this point we can check the new start point
		 * to determine if all nondirs have been committed.
		 * if this flag was already set, then this is a
		 * inomap and dirdump-only media file.
		 */
		if (scwhdrp->cih_startpt.sp_flags & STARTPT_FLAGS_END) {
			if (all_nondirs_committed) {
				empty_mediafile = BOOL_TRUE;
			}
			all_nondirs_committed = BOOL_TRUE;
		}

		/* we are done if all nondirs have been committed.
		 * it is not necessary for the null file header to have
		 * been committed.
		 */
		done = all_nondirs_committed;

		/* tell the inventory about the media file
		 */
		if (inv_stmt != INV_TOKEN_NULL) {
			bool_t ok;

			if (!all_dirs_committed) {
				mlog(MLOG_DEBUG,
				      "giving inventory "
				      "partial dirdump media file\n");
			} else if (done && empty_mediafile) {
				mlog(MLOG_DEBUG,
				      "giving inventory "
				      "empty last media file: "
				      "%llu:%lld\n",
				       startino,
				       startoffset);
			} else if (empty_mediafile) {
				mlog(MLOG_DEBUG,
				      "giving inventory "
				      "empty media file: "
				      "%llu:%lld\n",
				       startino,
				       startoffset);
			} else if (done) {
				mlog(MLOG_DEBUG,
				      "giving inventory "
				      "last media file: "
				      "%llu:%lld\n",
				       startino,
				       startoffset);
			} else {
				mlog(MLOG_DEBUG,
				      "giving inventory "
				      "media file: "
				      "%llu:%lld - %llu:%lld\n",
				       startino,
				       startoffset,
				       scwhdrp->cih_startpt.sp_ino,
				       scwhdrp->cih_startpt.sp_offset);
			}

			/* already thread-safe, don't need to lock
			 */
			ok = inv_put_mediafile(inv_stmt,
						&mwhdrp->mh_mediaid,
						mwhdrp->mh_medialabel,
					(uint)mwhdrp->mh_mediafileix,
						startino,
						startoffset,
						scwhdrp->cih_startpt.sp_ino,
						scwhdrp->cih_startpt.sp_offset,
						ncommitted,
					        all_dirs_committed
						&&
						!empty_mediafile,
						BOOL_FALSE);
			if (!ok) {
				mlog(MLOG_NORMAL, _(
				      "inventory media file put failed\n"));
			}
		}
		if (done) {
			contextp->cc_completepr = BOOL_TRUE;
			    /* so inv_end_stream and main will know
			     */
		}

		/* don't go back for more if done or stop was requested
		 */
		if (done || stop_requested) {
			break;
		}
	} /* end main dump loop */

	/* check in
	 */
	lock();
	sc_thrdsdonecnt++;
	unlock();

	/* dump the session inventory and terminator here, if the drive
	 * supports multiple media files. must wait until all
	 * streams have completed or given up, so all media files
	 * from all streams have been registered.
	 */
	if (drivep->d_capabilities & DRIVE_CAP_FILES) {
		if (stream_cnt() > 1) {
			mlog(MLOG_VERBOSE, _(
			      "waiting for synchronized "
			      "session inventory dump\n"));
			sc_stat_pds[strmix].pds_phase = PDS_INVSYNC;
		}

		/* first be sure all threads have begun
		*/
		while (sc_thrdsarrivedcnt < drivecnt) {
			sleep(1);
		}
		/* now wait for survivors to checkin
		*/
		while (sc_thrdsdonecnt < stream_cnt()) {
			sleep(1);
		}
		/* proceeed
		 */
		sc_stat_pds[strmix].pds_phase = PDS_INVDUMP;
		if (dump_session_inv(drivep, contextp, mwhdrp, scwhdrp)) {
			sc_stat_pds[strmix].pds_phase = PDS_TERMDUMP;
			dump_terminator(drivep, contextp, mwhdrp);
		}
	}

	sc_stat_pds[strmix].pds_phase = PDS_NULL;

	free((void *)bstatbufp);

	elapsed = time(0) - sc_stat_starttime;

	mlog(MLOG_TRACE, _(
	      "ending stream: %ld seconds elapsed\n"),
	      elapsed);

	return mlog_exit(EXIT_NORMAL, rv);
}

/* indicates if the dump was complete.
 * easy to tell: initially contextp->cc_completepr is false for each stream.
 * only set true if stream complete. if any stream NOT complete,
 * dump is not complete.
 */
bool_t
content_complete(void)
{
	time_t elapsed;
	bool_t completepr;
	int i;

	completepr = check_complete_flags();

	elapsed = time(0) - sc_stat_starttime;

	mlog(MLOG_VERBOSE, _(
	      "dump size (non-dir files) : %llu bytes\n"),
	      sc_stat_datadone);

	if (completepr) {
		if(sc_savequotas) {
			for(i = 0; i < (sizeof(quotas) / sizeof(quotas[0])); i++) {
				if(quotas[i].savequotas && unlink(quotas[i].quotapath) < 0) {
					mlog(MLOG_ERROR, _(
					"unable to remove %s: %s\n"),
					quotas[i].quotapath,
					strerror (errno));
				}
			}
		}

		mlog(MLOG_VERBOSE, _(
		      "dump complete"
		      ": %ld seconds elapsed"
		      "\n"),
		      elapsed);
	} else {
		if (sc_inv_updatepr) {
			mlog(MLOG_VERBOSE | MLOG_NOTE, _(
			      "dump interrupted"
			      ": %ld seconds elapsed"
			      ": may resume later using -%c option"
			      "\n"),
			      elapsed,
			      GETOPT_RESUME);
			mlog_exit_hint(RV_INTR);
		} else {
			mlog(MLOG_VERBOSE | MLOG_NOTE, _(
			      "dump interrupted"
			      ": %ld seconds elapsed"
			      "\n"),
			      elapsed);
			mlog_exit_hint(RV_INTR);
		}
	}
	return completepr;
}

#define PREAMBLEMAX	3
#define QUERYMAX	1
#define CHOICEMAX	30
#define ACKMAX		3
#define POSTAMBLEMAX	3
#define DLOG_TIMEOUT	300
#define DLOG_TIMEOUT_MEDIA	3600

#define CHOICESTRSZ	10
typedef struct { ix_t thrdix; char choicestr[CHOICESTRSZ]; } cttm_t;

char *
content_mediachange_query(void)
{
	cttm_t choicetothrdmap[STREAM_SIMMAX];
	char *querystr[QUERYMAX];
	size_t querycnt;
	char *choicestr[CHOICEMAX];
	size_t choicecnt;
	size_t maxdrvchoiceix;
	size_t nochangeix;
	size_t responseix;
	ix_t thrdix;

	querycnt = 0;
	querystr[querycnt++ ] = "select a drive to acknowledge media change\n";
	choicecnt = 0;
	maxdrvchoiceix = 0;
	for (thrdix = 0; thrdix < STREAM_SIMMAX; thrdix++) {
		if (sc_mcflag[thrdix]) {
			choicetothrdmap[choicecnt].thrdix = thrdix;
			sprintf(choicetothrdmap[choicecnt].choicestr,
				 "drive %u",
				 (unsigned int)thrdix);
			choicestr[choicecnt] =
					choicetothrdmap[choicecnt].choicestr;
			maxdrvchoiceix = choicecnt;
			choicecnt++;
		}
	}
	nochangeix = choicecnt;
	choicestr[choicecnt++ ] = "continue";
	assert(choicecnt <= CHOICEMAX);
	responseix = dlog_multi_query(querystr,
				       querycnt,
				       choicestr,
				       choicecnt,
				       0,           /* hilitestr */
				       IXMAX,       /* hiliteix */
				       0,           /* defaultstr */
				       nochangeix,  /* defaultix */
				       DLOG_TIMEOUT_MEDIA,
				       nochangeix, /* timeout ix */
				       nochangeix, /* sigint ix */
				       nochangeix, /* sighup ix */
				       nochangeix);/* sigquit ix */
	if (responseix <= maxdrvchoiceix) {
		clr_mcflag(choicetothrdmap[responseix].thrdix);
		return "media change acknowledged\n";
	}
	assert(responseix == nochangeix);
	return "continuing\n";
}


static void
update_cc_Media_useterminatorpr(drive_t *drivep, context_t *contextp)
{
	int dcaps = drivep->d_capabilities;

	contextp->cc_Media_useterminatorpr = BOOL_TRUE;
	if (!(dcaps & DRIVE_CAP_FILES)) {
		contextp->cc_Media_useterminatorpr = BOOL_FALSE;
	}
	if (!(dcaps & DRIVE_CAP_OVERWRITE)) {
		contextp->cc_Media_useterminatorpr = BOOL_FALSE;
	}
	if (!(dcaps & DRIVE_CAP_BSF)) {
		contextp->cc_Media_useterminatorpr = BOOL_FALSE;
	}
	if (!(dcaps & DRIVE_CAP_APPEND)) {
		contextp->cc_Media_useterminatorpr = BOOL_FALSE;
	}
}

static rv_t
dump_dirs(ix_t strmix,
	   struct xfs_bstat *bstatbufp,
	   size_t bstatbuflen,
	   void *inomap_contextp)
{
	xfs_ino_t lastino;
	size_t bulkstatcallcnt;
        struct xfs_fsop_bulkreq bulkreq;

	inomap_reset_context(inomap_contextp);

	/* begin iteration at ino zero
	 */
	lastino = 0;
	for (bulkstatcallcnt = 0 ; ; bulkstatcallcnt++) {
		struct xfs_bstat *p;
		struct xfs_bstat *endp;
		__s32 buflenout;
		int rval;

		if (bulkstatcallcnt == 0) {
			mlog(MLOG_VERBOSE, _(
			      "dumping directories\n"));
		}
		sc_stat_pds[strmix].pds_phase = PDS_DIRDUMP;

		/* check for interruption
		 */
		if (cldmgr_stop_requested()) {
			return RV_INTR;
		}

		/* get a bunch of bulkstats
		 */
		mlog(MLOG_NITTY,
		      "dump_dirs SGI_FS_BULKSTAT %u buf len %u\n",
		      bulkstatcallcnt,
		      bstatbuflen);

		bulkreq.lastip = (__u64 *)&lastino;
		bulkreq.icount = bstatbuflen;
		bulkreq.ubuffer = bstatbufp;
		bulkreq.ocount = &buflenout;

		rval = ioctl(sc_fsfd, XFS_IOC_FSBULKSTAT, &bulkreq);

		if (rval) {
			mlog(MLOG_NORMAL, _(
			      "SGI_FS_BULKSTAT failed: "
			      "%s (%d)\n"),
			      strerror(errno),
			      errno);
			return RV_ERROR;
		}
		mlog(MLOG_NITTY,
		      "dump_dirs SGI_FS_BULKSTAT returns %d entries\n",
		      buflenout);

		/* check if done
		 */
		if (buflenout == 0) {
			return RV_OK;
		}

		/* step through each node, dumping if
		 * appropriate
		 */
		for (p = bstatbufp, endp = bstatbufp + buflenout
		      ;
		      p < endp
		      ;
		      p++) {
			rv_t rv;

			if (p->bs_ino == 0)
				continue;

			if (!p->bs_nlink || !p->bs_mode) {
				/* inode being modified, get synced data */
				mlog(MLOG_NITTY,
				      "ino %llu needs second bulkstat\n",
				      p->bs_ino);

				if (bigstat_one(sc_fsfd, p->bs_ino, p) < 0) {
					mlog(MLOG_WARNING,  _(
					      "failed to get bulkstat information for inode %llu\n"),
					      p->bs_ino);
					continue;
				}
				if (!p->bs_nlink || !p->bs_mode || !p->bs_ino) {
					mlog(MLOG_TRACE,
					      "failed to get valid bulkstat information for inode %llu\n",
					      p->bs_ino);
					continue;
				}
			}
			if ((p->bs_mode & S_IFMT) != S_IFDIR) {
				continue;
			}

			rv = dump_dir(strmix, sc_fshandlep, sc_fsfd, p);
			if (rv != RV_OK) {
				return rv;
			}
		}

		lastino = inomap_next_dir(inomap_contextp, lastino);
		if (lastino == INO64MAX) {
			mlog(MLOG_DEBUG, "bulkstat seeked to EOS\n");
			return 0;
		}

		mlog(MLOG_DEBUG, "bulkstat seeked to %llu\n", lastino);

		lastino = (lastino > 0) ? lastino - 1 : 0;
	}
	/* NOTREACHED */
}

static rv_t
dump_dir(ix_t strmix,
	  jdm_fshandle_t *fshandlep,
	  int fsfd,
	  struct xfs_bstat *statp)
{
	context_t *contextp = &sc_contextp[strmix];
	drive_t *drivep = drivepp[strmix];
	void *inomap_contextp = contextp->cc_inomap_contextp;
	int state;
	int fd;
	struct dirent *gdp = (struct dirent *)contextp->cc_getdentsbufp;
	size_t gdsz = contextp->cc_getdentsbufsz;
	int gdcnt;
	gen_t gen;
	rv_t rv;

	/* no way this can be non-dir, but check anyway
	 */
	assert((statp->bs_mode & S_IFMT) == S_IFDIR);
	if ((statp->bs_mode & S_IFMT) != S_IFDIR) {
		return RV_OK;
	}

	/* skip if no links
	 */
	if (statp->bs_nlink < 1) {
		return RV_OK;
	}

	/* see what the inomap says about this ino
	 */
	state = inomap_get_state(inomap_contextp, statp->bs_ino);

	/* skip if not in inomap
	 */
	if (state == MAP_INO_UNUSED
	     ||
	     state == MAP_DIR_NOCHNG
	     ||
	     state == MAP_NDR_NOCHNG) {
		if (state == MAP_NDR_NOCHNG) {
			mlog(MLOG_DEBUG,
			      "inomap inconsistency ino %llu: "
			      "map says is non-dir but is dir: skipping\n",
			      statp->bs_ino);
		}
		return RV_OK;
	}

	/* note if map says a non-dir
	 */
	if (state == MAP_NDR_CHANGE) {
		mlog(MLOG_DEBUG,
		      "inomap inconsistency ino %llu: "
		      "map says non-dir but is dir: skipping\n",
		      statp->bs_ino);
		return RV_OK;
	}

	/* bump the stats now. a bit early, but fewer lines of code
	 */
	sc_stat_pds[strmix].pds_dirdone++;

        /* if bulkstat ino# occupied more than 32 bits and
         * linux ino# for getdents is 32 bits then
         * warn and skip.
         */
	if (statp->bs_ino > (xfs_ino_t)INOMAX) {
		mlog(MLOG_NORMAL | MLOG_WARNING, _(
		      "unable to dump directory: ino %llu too large\n"),
		      statp->bs_ino);
		return RV_OK; /* continue anyway */
	}

	mlog(MLOG_TRACE,
	      "dumping directory ino %llu\n",
	      statp->bs_ino);

	/* open the directory named by statp
	 */
	fd = jdm_open(fshandlep, statp, O_RDONLY);
	if (fd < 0) {
		mlog(MLOG_NORMAL | MLOG_WARNING, _(
		      "unable to open directory: ino %llu: %s\n"),
		      statp->bs_ino, strerror(errno));
		return RV_OK; /* continue anyway */
	}

	/* dump the file header.
	 */
	rv = dump_filehdr(drivep, contextp, statp, 0, 0);
	if (rv != RV_OK) {
		close(fd);
		return rv;
	}

	/* dump dirents - lots of buffering done here, to achieve OS-
	 * independence. if proves to be to much overhead, can streamline.
	 */
	for (gdcnt = 1, rv = RV_OK; rv == RV_OK; gdcnt++) {
		struct dirent *p;
		int nread;
		register size_t reclen;

		nread = getdents_wrap(fd, (char *)gdp, gdsz);

		/* negative count indicates something very bad happened;
		 * try to gracefully end this dir.
		 */
		if (nread < 0) {
			mlog(MLOG_NORMAL | MLOG_WARNING, _(
			      "unable to read dirents (%d) for "
			      "directory ino %llu: %s\n"),
			      gdcnt,
			      statp->bs_ino,
			      strerror(errno));
			/* !!! curtis looked at this, and pointed out that
			 * we could take some recovery action here. if the
			 * errno is appropriate, lseek64 to the value of
			 * doff field of the last dirent successfully
			 * obtained, and contiue the loop.
			 */
			nread = 0; /* pretend we are done */
		}

		/* no more directory entries: break;
		 */
		if (nread == 0) {
			break;
		}

		/* translate and dump each entry: skip "." and ".."
		 * and null entries.
		 */
		for (p = gdp,
		      reclen = (size_t)p->d_reclen
		      ;
		      nread > 0
		      ;
		      nread -= (int)reclen,
		      assert(nread >= 0),
		      p = (struct dirent *)((char *)p + reclen),
		      reclen = (size_t)p->d_reclen) {
			xfs_ino_t ino;
			register size_t namelen = strlen(p->d_name);
#ifdef DEBUG
			register size_t nameszmax = (size_t)reclen
						    -
						    offsetofmember(struct dirent,
								    d_name);

			/* getdents(2) guarantees that the string will
			 * be null-terminated, but the record may have
			 * padding after the null-termination.
			 */
			assert(namelen < nameszmax);
#endif

			/* skip "." and ".."
			 */
			if (*(p->d_name + 0) == '.'
			     &&
			     (*(p->d_name + 1) == 0
			       ||
			       (*(p->d_name + 1) == '.'
				 &&
				 *(p->d_name + 2) == 0))) {
				continue;
			}

			ino = (xfs_ino_t)p->d_ino;

			if (ino == 0) {
				mlog(MLOG_NORMAL | MLOG_WARNING, _(
				      "encountered 0 ino (%s) in "
				      "directory ino %llu: NOT dumping\n"),
				      p->d_name,
				      statp->bs_ino);
				continue;
			}

			/* lookup the gen number in the ino-to-gen map.
			 * if it's not there, we have to get it the slow way.
			 */
			if (inomap_get_gen(NULL, p->d_ino, &gen)) {
				struct xfs_bstat statbuf;
				int scrval;

				scrval = bigstat_one(fsfd,
						      p->d_ino,
						      &statbuf);
				if (scrval) {
					mlog(MLOG_NORMAL | MLOG_WARNING, _(
					      "could not stat "
					      "dirent %s ino %llu: %s: "
					      "using null generation count "
					      "in directory entry\n"),
					      p->d_name,
					      (xfs_ino_t)p->d_ino,
					      strerror(errno));
					gen = 0;
				} else {
					gen = statbuf.bs_gen;
				}
			}

			rv = dump_dirent(drivep,
					  contextp,
					  statp,
					  ino,
					  gen,
					  p->d_name,
					  namelen);
			if (rv != RV_OK) {
				break;
			}
		}
	}

	/* write a null dirent hdr, unless trouble encountered in the loop
	 */
	if (rv == RV_OK) {
		rv = dump_dirent(drivep, contextp, statp, 0, 0, 0, 0);
	}

	if (rv == RV_OK
	     &&
	     sc_dumpextattrpr
	     &&
	     (statp->bs_xflags & XFS_XFLAG_HASATTR)) {
		rv = dump_extattrs(drivep, contextp, fshandlep, statp);
	}

	close(fd);

	/* if an error occurred, just return the error
	 */
	return rv;
}

static rv_t
dump_extattrs(drive_t *drivep,
	       context_t *contextp,
	       jdm_fshandle_t *fshandlep,
	       struct xfs_bstat *statp)
{
	ix_t pass;
	int flag;
	attrlist_cursor_t cursor;
	rv_t rv;
	bool_t abort;

	/* dump a file header specially marked as heading extended attributes
	 */
	mlog(MLOG_NITTY,
	      "dumping %s ino %llu extended attributes filehdr\n",
	      FILETYPE(statp),
	      statp->bs_ino);

	rv = dump_filehdr(drivep, contextp, statp, 0, FILEHDR_FLAGS_EXTATTR);
	if (rv != RV_OK) {
		return rv;
	}

	/* loop three times: once for the non-root, once for root, and
	 * again for the secure attributes.
	 */
	for (pass = 0; pass < 3; pass++) {
		bool_t more;

		if (pass == 0)
			flag = 0;
		else if (pass == 1)