email/aiaiai-email-lda - pub/scm/linux/kernel/git/will/aiaiai - Git at Google

 #!/bin/sh -efu

 # Copyright 2011-2012 Intel Corporation
 # Author: Artem Bityutskiy
 # License: GPLv2

 srcdir="$(readlink -ev -- ${0%/*})"
 PATH="$srcdir:$srcdir/..:$srcdir/../helpers:$srcdir/../helpers/libshell:$PATH"

 . shell-error
 . shell-args
 . shell-signal
 . aiaiai-sh-functions
 . aiaiai-email-sh-functions

 PROG="${0##*/}"
 message_time="yes"

 # This is a small trick to make sure the script is portable - check if 'dash'
 # is present, and if yes - use it.
 if can_switch_to_dash; then
 	exec dash -euf -- "$srcdir/$PROG" "$@"
 	exit $?
 fi

 show_usage()
 {
 	cat <<-EOF
 Usage: $PROG [options] <workdir> <config.ini>

 This is a very simple local LDA which filters out all e-mails except patches
 and stores the result in the "<workdir>/queue and <workdir>/queue_saved"
 directories (both stand-alone patches and patch-sets are stored as one mbox
 file). The "<workdir>/lda_tmp" directory is used for temporary storing not
 yet complete patch series.

 Additionally, this program stores all the incoming mail in the
 "<workdir>/mail" mail archive.

 The "<config.ini>" file is the same file which "aiaiai-email-test-patchest"
 uses. When this program receives an invalid e-mail (not a patch, invalid
 subject, etc), it sends an e-mail notifications back. The "<config.ini>" file
 is required for extracting basic information for sending the notifications (own
 e-mail address, etc).

 <workdir>    - the work directory
 <config.ini> - the configuration file

 Options:
   -v, --verbose              be verbose;
   -h, --help                 show this text and exit.
 EOF
 }

 fail_usage()
 {
 	[ -z "$1" ] || printf "%s\n" "$1"
 	show_usage
 	exit 1
 }

 reject()
 {
 	local file="$1"; shift
 	local reason="$1" shift
 	local subj from id

 	fetch_header_or_die subj "Subject" < "$file"
 	fetch_header_or_die from "From" < "$file"
 	fetch_header_or_die id "Message-Id" < "$file"

 	verbose "Rejecting \"$from: $subj (Message-Id: $id)\""
 	verbose "Reason: $1"
 	rm $verbose -- "$file" >&2
 }

 reject_and_reply()
 {
 	local file="$1"; shift
 	local reply_to reply_cc reply_id reply_subj to cc

 	fetch_header_or_die reply_to "From" < "$file"
 	fetch_header_or_die reply_subj "Subject" < "$file"
 	fetch_header_or_die reply_id "Message-Id" < "$file"

 	to="$(fetch_header "To" < "$file")"
 	cc="$(fetch_header "Cc" < "$file")"

 	reply_cc="$(merge_addresses "$to" "$cc")"

 	prj="$(fetch_project_name "$reply_cc" "$cfg_ownmail")"
 	verbose "Project \"$prj\""
 	parse_prj_config "$cfgfile" "$prj"

 	if [ -n "$pcfg_name" ] && [ "$pcfg_reply_to_all" = "1" ]; then
 		# Strip own address
 		reply_cc="$(merge_addresses "$reply_cc" "$pcfg_always_cc")"
 		reply_cc="$(strip_address "$reply_cc" "$cfg_ownmail")"
 	else
 		reply_cc=
 	fi

 	compose_email "$reply_to" "$reply_cc" "$reply_subj" "$reply_id" \
 		> "$lda_tmp/mail"

 	[ -z "$verbose" ] || cat -- "$lda_tmp/mail" >&2

 	if [ "$cfg_disable_notifications" != "1" ]; then
 		mutt -x -H "$lda_tmp/mail" </dev/null
 	else
 		verbose "Email notifications have been disabled in the configuration file"
 	fi
 }

 # Generate unique name for a file or directory in the "$1" subdirectory.
 # Currently we use the same format: <date>_<suffix>-<counter>, where <date> is
 # the current date (taken from the global $date variable), <suffix> is the
 # a suffix supplied by the caller via "$2", and <counter> is usually 0, but if
 # there is already a file with such name, it gets increased to 1, and so on.
 #
 # NOTE! The 'process_all_series()' function relies on this directory name
 # format, so do not forget to amend the function if you change the format.
 generate_file_name()
 {
 	local where="$1"; shift
 	local suffix="$1"; shift
 	local path="$where/${date}_${suffix}-"
 	local i=0

 	while [ -e "${path}${i}" ]; do
 		i="$(($i+1))"
 	done

 	printf "%s" "${path}${i}"
 }

 # Queue the mbox file for validation. Basically queuing is about moving the
 # file to the queue directory, and the Aiaiai dispatcher will then pick it from
 # there (it uses inotify to get notifications about new files in this
 # directories). This function also saves a copy of the mbox in the
 # 'queue_saved' directory. The mbox may contain a single patch, or entire
 # patch-set.
 queue_mboxfile()
 {
 	local mbox="$1"; shift
 	local n="$1"; shift
 	local fname="$(generate_file_name "$queue" "$n")"

 	cp $verbose -- "$mbox" "$queue_saved/${fname##*/}" >&2
 	mv $verbose -- "$mbox" "$fname" >&2
 }

 find_relatives()
 {
 	local where="$1"; shift
 	local id="$1"; shift
 	local parent_id="$1"; shift

 	# Find the parent, children, and brothers
 	LC_ALL=C grep -i -l -r -- "\(In-Reply-To\|References\): $id" "$where" ||:
 	if [ -n "$parent_id" ]; then
 		LC_ALL=C grep -i -l -r -- "Message-Id: $parent_id" "$where" ||:
 		LC_ALL=C grep -i -l -r -- "\(In-Reply-To\|References\): $parent_id" "$where" ||:
 	fi
 }

 series_is_complete()
 {
 	local dir="$1"; shift
 	local n="$1"; shift

 	# First check if we have all the non-cover patches yet
 	local cnt="$(ls -1 --ignore=0 -- "$dir" | wc -l)"
 	if [ "$cnt" -eq "$n" ]; then
 		local first_parent="$(fetch_header "In-Reply-To" < "$dir/1")"
 		[ -z "$first_parent" ] && first_parent="$(fetch_header "References" < "$dir/1")"
 		if [ -n "$first_parent" ]; then
 			# The first patch has a parent, must be the cover letter
 			if [ -f "$dir/0" ]; then
 				verbose "Series in $dir is complete and has cover letter"
 				return 0
 			else
 				verbose "Series in $dir is not complete, we are missing the cover letter"
 				return 1
 			fi
 		else
 			verbose "Series is in $dir complete no cover letter was sent"
 			return 0
 		fi
 	else
 		verbose "Series in $dir is not complete, we have only $cnt out of $n patches"
 		return 1
 	fi
 }

 # Queue a complete series of patches.
 queue_series()
 {
 	local mbox="$1"; shift
 	local dir="$1"; shift
 	local n="$1"; shift

 	verbose "Patch-set at \"$dir\" is complete, queue it"
 	# Don't add the 0th patch to the final mbox, as it is just the
 	# cover letter and does not contain any patch
 	for fname in $(ls --ignore=0 -A -- "$dir" | sort -n); do
 		cat -- "$dir/$fname" >> "$mbox"
 		printf "\n" >> "$mbox"
 	done

 	if [ -f "$dir/0" ]; then
 		# Save the subject and message ID of the cover letter in the
 		# final mbox in order to be able to reply to the cover letter
 		# later.
 		local subj="$(fetch_header "Subject" < "$dir/0")"
 		subj="X-Aiaiai-Cover-Letter-Subject: $subj"
 		insert_header "$mbox" "$subj"

 		local id="$(fetch_header "Message-Id" < "$dir/0")"
 		id="X-Aiaiai-Cover-Letter-Message-Id: $id"
 		insert_header "$mbox" "$id"
 	fi
 	queue_mboxfile "$mbox" "$n"
 	rm $verbose -rf -- "$dir" >&2
 }

 move_to_series()
 {
 	local file="$1"; shift
 	local dir="$1"; shift
 	local m="$1"; shift
 	local n="$1"; shift
 	local subj

 	fetch_header_or_die subj "Subject" < "$file"
 	m="$(subject_m "$subj")"

 	verbose "Moving patch (\"$subj\") the series directory"

 	# Sanity check - n has to be the same
 	if ! [ "$(subject_n "$subj")" -eq "$n" ]; then
 		reject "$file" "We were processing patch m/$n, but found a relative $m/$n"
 		return
 	fi

 	# File $m must not exist
 	if [ -f "$dir/$m" ]; then
 		reject "$file" "File \"$dir/$m\" already exists!"
 	else
 		mv $verbose -- "$file" "$dir/$m" >&2
 	fi
 }

 # Process a patch which is parte of a series. The patch is passed via "$1". Its
 # message ID is passed via "$2". Its number in the series is passed via "$3",
 # and amount of patches in the series is passed via "$4". This function will
 # try to collect all patches belonging to the series, and when it receives the
 # last patch, it queues the entire series.
 #
 # The series is collected using message ID headers - each patch, except for the
 # first one, must refer the previous patches ID via the "In-Reply-To:" header
 # or "References:" header.
 process_series_mbox()
 {
 	local mbox="$1"; shift
 	local id="$1"; shift
 	local m="$1"; shift
 	local n="$1"; shift
 	local fname dir

 	# Only patch 0/n or 1/n is allowed to have no parent
 	local parent_id="$(fetch_header "In-Reply-To" < "$mbox")"
 	[ -z "$parent_id" ] && parent_id="$(fetch_header "References" < "$mbox")"

 	if [ -z "$parent_id" ] && [ "$m" != 1 ] && [ "$m" != 0 ]; then
 		reject_and_reply "$mbox" <<EOF
 You sent a patch that does not conform to the requirements for Aiaiai's Local
 Delivery Agent. This patch is part of a series as indicated by its subject
 prefix.  However, it does not contain the correct "In-Reply-To:" header or
 "References:" header which is required for Aiaiai to determine which patch
 series it belongs with.

 The most common reason for this issue is from not using the git-send-email
 utility, and using your own script which does not generate proper In-Reply-To:
 header chains.

 If you do use your own email commands, please add "--thread" to your
 git-format-patch commands, in order to ensure that the patch series
 contains proper threading.

 If you do not understand this email, or believe you have received this email in
 error, please contact "$cfg_adminname" <$cfg_adminmail>.
 EOF
 		return
 	fi

 	[ -z "$parent_id" ] || verbose "In-Reply-To: $parent_id"

 	local staging_relatives
 	local series_relatives
 	staging_relatives="$(find_relatives "$staging" "$id" "$parent_id" | sort -u)"
 	series_relatives="$(find_relatives "$series" "$id" "$parent_id" | sort -u)"

 	verbose "Staging relatives: $staging_relatives"
 	verbose "Series relatives:  $series_relatives"

 	if [ -z "$staging_relatives" ] && [ -z "$series_relatives" ]; then
 		# Save the file in the staging area
 		fname="$(generate_file_name "$staging" "$m-of-$n")"
 		verbose "No relatives found, temporarily save in staging"
 		mv $verbose -- "$mbox" "$fname" >&2
 		return
 	fi

 	verbose "Found relatives"

 	if [ -z "$series_relatives" ]; then
 		# The series directory does not exist yet - create it
 		dir="$(generate_file_name "$series" "$n")"
 		verbose "Creating the series directory \"$dir\""
 		mkdir $verbose -- "$dir" >&2
 	else
 		dir="$(printf "%s" "$series_relatives" | sed -e "s/\(.*\)\/.*/\1/" | sort -u)"
 		verbose "Found series directory \"$dir\""
 		[ "$(printf "%s" "$dir" | wc -l)" -eq 0 ] ||
 			die "Relatives live in different series directories!"
 	fi

 	# Move the relatives from the staging to the series directory
 	move_to_series "$mbox" "$dir" "$m" "$n"

 	local relative
 	for relative in $staging_relatives; do
 		move_to_series "$relative" "$dir" "$m" "$n"
 	done

 	# If the series is complete - queue it
 	if series_is_complete "$dir" "$n"; then
 		queue_series "$mbox" "$dir" "$n"
 	fi
 }

 separator()
 {
 	if [ -n "$verbose" ]; then
 		printf "\n%s"   "----------------------------------------------------" >&2
 		printf "%s\n\n" "----------------------------------------------------" >&2
 	fi
 }

 # Proccess one e-mail. This e-mail may be part of a patch-set, may contain an
 # individual patch, or may not contain a patch at all. The patches and
 # non-patches are distinguished using is_email_patch() which is based on "git
 # apply --stat". Patch-set emails are distinquished from patches via use of the
 # subject line. Individual patches are queued right away. Patch-sets are stored
 # in a temporary location and queued when the entire patch set is collected.
 process_mbox()
 {
 	local mbox="$1";

 	separator

 	# Make sure important headers are there
 	grep -i -q -- "^Message-Id:[[:blank:]]" "$mbox" ||
 		{ verbose "The \"Message-Id:\" header not found, ignoring"; return; }
 	grep -i -q -- "^From:[[:blank:]]" "$mbox" ||
 		{ verbose "The \"From:\" header not found, ignoring"; return; }
 	grep -i -q -- "^To:[[:blank:]]" "$mbox" ||
 		{ verbose "The \"To:\" header not found, ignoring"; return; }
 	grep -i -q -- "^Subject:[[:blank:]]" "$mbox" ||
 		{ verbose "The \"Subject:\" header not found, ignoring"; return; }

 	local subj="$(fetch_header "Subject" < "$mbox")"
 	local from="$(fetch_header "From" < "$mbox")"
 	local id="$(fetch_header "Message-Id" < "$mbox")"

 	verbose "Looking at: From: $from"
 	verbose "            Subject: $subj"
 	verbose "            Message-Id: $id"


 	# Filter out e-mails which do not contain a patch
 	if ! email_contains_patch "$mbox" ; then
 		# Keep cover letter patches, but discard anything else
 		[ "$(subject_m "$subj")" = "0" ] ||
 			{ reject "$mbox" "mbox contains no patch and is not a cover letter"; return ; }
 	fi

 	# Queue patches which don't match subject format immediately
 	if ! subject_check "$subj" ; then
 		verbose "Queuing stand-alone patch with unrecognized subject format \"$subj\""
 		queue_mboxfile "$mbox" ""
 		return
 	fi

 	# If the patch prefix contains m/n, fetch m and n.
 	local m="$(subject_m "$subj")"
 	local n="$(subject_n "$subj")"

 	if [ -z "$m" ]; then
 		verbose "Queuing stand-alone patch \"$subj\""
 		queue_mboxfile "$mbox" "$n"
 	else
 		verbose "Processing member $m/$n of a series (\"$subj\")"
 		[ "$n" -ne 0 ] || \
 			{ reject "$mbox" "Prefix \"$prefix_format\" cannot have n = 0";
 			  return; }
 		process_series_mbox "$mbox" "$id" "$m" "$n"
 	fi
 }

 # This function goes through all the incomplete path series which we have
 # collected so far. And if the series happens to be complete, it queues them.
 # Normally, there should be no complete series in the "$series" directory.
 # However, sometimes this script misbehaves due to some bugs, and fails to add
 # a patch to the series, so it never gets queued. Then the Aiaiai admin may do
 # this manually. However, the series does not get queued anyway, because this
 # scrpt only checks the series when it adds a patch there.
 #
 # And this is where this function comes handy. It will go through the series
 # and queue all the complete ones.
 #
 # The only parameter of this function is the file to use as a temporary storage
 # for mboxes.
 process_all_series()
 {
 	local mbox="$1"; shift
 	local dir n

 	separator
 	message "Going through all partial series and checking if some of them became complete"

 	# Get all the current series
 	printf "%s\n" "$(find "$series" -mindepth 1 -maxdepth 1 -type d)" | \
 	    while IFS= read -r dir; do
 		[ -n "$dir" ] || continue
 		# Extract the series number
 		n="${dir%-*}"
 		n="${n##*_}"

 		# Clear everything from the current mbox
 		truncate -s0 -- "$mbox"

 		if series_is_complete "$dir" "$n"; then
 			queue_series "$mbox" "$dir" "$n"
 		fi
 	done
 }

 reap_old()
 {
 	local dir="$1"; shift
 	local min="$1"; shift

 	verbose "Reaping files older than \"$min\" minutes in \"$dir\""
 	find "$dir" -mindepth 1 -maxdepth 1 -mmin +"$min" -exec rm $verbose -r -f -- "{}" ";" >&2
 }

 mbox=
 lda_tmp_lock=
 cleanup_handler()
 {
 	rm $verbose -rf -- "$mbox" >&2
 	rm $verbose -rf -- "$lda_tmp_lock" >&2
 }
 set_cleanup_handler cleanup_handler

 TEMP=`getopt -n $PROG -o v,h --long reap-archive:,reap-incomplete:,verbose,help -- "$@"` ||
 	fail_usage ""
 eval set -- "$TEMP"

 verbose=

 while true; do
 	case "$1" in
 	-v|--verbose) verbose=-v
 		;;
 	-h|--help)
 		show_usage
 		exit 0
 		;;
 	--) shift; break
                 ;;
 	*) fail_usage "Unrecognized option: $1"
 		;;
 	esac
 	shift
 done

 [ "$#" -eq 2 ] || fail_usage "Insufficient or too many arguments"

 program_required "mutt" ""
 program_required "find" ""
 program_required "grep" ""
 program_required "sed" ""
 program_required "formail" ""

 workdir="$(readlink -fv -- "$1")"; shift
 cfgfile="$(readlink -fv -- "$1")"; shift

 parse_config "$cfgfile"

 lda_tmp="$workdir/lda_tmp"
 mkdir -p $verbose -- "$lda_tmp" >&2

 # In the staging directory we temporarily store patches which belong to a
 # patch-set but, but we do not know which one yet.
 staging="$lda_tmp/staging"
 # The series directory stores partial patch-sets.
 series="$lda_tmp/series"
 # The mail directory sores all the incoming mail
 mail="$workdir/mail"
 # The queue directory stores patches/patch-sets queued for further processing
 queue="$workdir/queue"
 # It is also convenient for debugging to store a copy of complete patch-series
 # separately
 queue_saved="$workdir/queue_saved"

 mkdir -p $verbose -- "$staging" "$series" "$mail" "$queue" "$queue_saved" >&2

 # All file we create have the date in the name
 date="$(date "+%Y-%m-%d_%H:%M:%S")"

 # We get the input from stdin, and it may contain several mails. We then
 # separate them, and process one-by-one. And the mail which is currently being
 # processed is stored in this temporary file.
 mbox="$(mktemp -t "$PROG.mbox.XXXX")"

 # We lock the lda and queue directories when using them
 lda_tmp_lock="$workdir/lda_tmp.lock"

 verbose "Taking $lda_tmp_lock lock file (timeout - 10 min)"
 lockfile -r 75 "$lda_tmp_lock"

 prev=
 first=y

 # Read stdin and separate out individual mails
 verbose "Saving incoming mbox in ${mail}/${date}.mbox"
 while IFS= read -r line; do
 	printf "%s\n" "$line" >> "${mail}/${date}.mbox"
 	if printf "%s" "$line" | grep -i -q -- "^From [^@ ]\+@[^@ ]\+ .\+$" || \
 	   printf "%s" "$line" | grep -i -q -- "^From [[:xdigit:]]\{40\} .\+$"; then
 		if [ -z "$first" ]; then
 			if [ -n "$prev" ]; then
 				verbose "The last line of previous mbox is not blank"
 				verbose "line: $line"
 				verbose "prev: $prev"
 				die "exiting"
 			fi
 			process_mbox "$mbox"
 			truncate -s0 -- "$mbox"
 		fi
 	else
 		printf "%s\n" "$prev" >> "$mbox"
 	fi
 	prev="$line"
 	first=
 done

 printf "%s\n" "$prev" >> "$mbox"
 process_mbox "$mbox"

 process_all_series "$mbox"

 [ -z "$cfg_lda_reap_incomplete" ] || reap_old "$lda_tmp" "$cfg_lda_reap_incomplete"
 [ -z "$cfg_lda_reap_archive" ] || reap_old "$mail" "$cfg_lda_reap_archive"
 [ -z "$cfg_lda_reap_archive" ] || reap_old "$queue_saved" "$cfg_lda_reap_archive"

 rm $verbose -f -- "$lda_tmp_lock" >&2
	#!/bin/sh -efu

	# Copyright 2011-2012 Intel Corporation
	# Author: Artem Bityutskiy
	# License: GPLv2

	srcdir="$(readlink -ev -- ${0%/*})"
	PATH="$srcdir:$srcdir/..:$srcdir/../helpers:$srcdir/../helpers/libshell:$PATH"

	. shell-error
	. shell-args
	. shell-signal
	. aiaiai-sh-functions
	. aiaiai-email-sh-functions

	PROG="${0##*/}"
	message_time="yes"

	# This is a small trick to make sure the script is portable - check if 'dash'
	# is present, and if yes - use it.
	if can_switch_to_dash; then
	exec dash -euf -- "$srcdir/$PROG" "$@"
	exit $?
	fi

	show_usage()
	{
	cat <<-EOF
	Usage: $PROG [options] <workdir> <config.ini>

	This is a very simple local LDA which filters out all e-mails except patches
	and stores the result in the "<workdir>/queue and <workdir>/queue_saved"
	directories (both stand-alone patches and patch-sets are stored as one mbox
	file). The "<workdir>/lda_tmp" directory is used for temporary storing not
	yet complete patch series.

	Additionally, this program stores all the incoming mail in the
	"<workdir>/mail" mail archive.

	The "<config.ini>" file is the same file which "aiaiai-email-test-patchest"
	uses. When this program receives an invalid e-mail (not a patch, invalid
	subject, etc), it sends an e-mail notifications back. The "<config.ini>" file
	is required for extracting basic information for sending the notifications (own
	e-mail address, etc).

	<workdir> - the work directory
	<config.ini> - the configuration file

	Options:
	-v, --verbose be verbose;
	-h, --help show this text and exit.
	EOF
	}

	fail_usage()
	{
	[ -z "$1" ] \|\| printf "%s\n" "$1"
	show_usage
	exit 1
	}

	reject()
	{
	local file="$1"; shift
	local reason="$1" shift
	local subj from id

	fetch_header_or_die subj "Subject" < "$file"
	fetch_header_or_die from "From" < "$file"
	fetch_header_or_die id "Message-Id" < "$file"

	verbose "Rejecting \"$from: $subj (Message-Id: $id)\""
	verbose "Reason: $1"
	rm $verbose -- "$file" >&2
	}

	reject_and_reply()
	{
	local file="$1"; shift
	local reply_to reply_cc reply_id reply_subj to cc

	fetch_header_or_die reply_to "From" < "$file"
	fetch_header_or_die reply_subj "Subject" < "$file"
	fetch_header_or_die reply_id "Message-Id" < "$file"

	to="$(fetch_header "To" < "$file")"
	cc="$(fetch_header "Cc" < "$file")"

	reply_cc="$(merge_addresses "$to" "$cc")"

	prj="$(fetch_project_name "$reply_cc" "$cfg_ownmail")"
	verbose "Project \"$prj\""
	parse_prj_config "$cfgfile" "$prj"

	if [ -n "$pcfg_name" ] && [ "$pcfg_reply_to_all" = "1" ]; then
	# Strip own address
	reply_cc="$(merge_addresses "$reply_cc" "$pcfg_always_cc")"
	reply_cc="$(strip_address "$reply_cc" "$cfg_ownmail")"
	else
	reply_cc=
	fi

	compose_email "$reply_to" "$reply_cc" "$reply_subj" "$reply_id" \
	> "$lda_tmp/mail"

	[ -z "$verbose" ] \|\| cat -- "$lda_tmp/mail" >&2

	if [ "$cfg_disable_notifications" != "1" ]; then
	mutt -x -H "$lda_tmp/mail" </dev/null
	else
	verbose "Email notifications have been disabled in the configuration file"
	fi
	}

	# Generate unique name for a file or directory in the "$1" subdirectory.
	# Currently we use the same format: <date>_<suffix>-<counter>, where <date> is
	# the current date (taken from the global $date variable), <suffix> is the
	# a suffix supplied by the caller via "$2", and <counter> is usually 0, but if
	# there is already a file with such name, it gets increased to 1, and so on.
	#
	# NOTE! The 'process_all_series()' function relies on this directory name
	# format, so do not forget to amend the function if you change the format.
	generate_file_name()
	{
	local where="$1"; shift
	local suffix="$1"; shift
	local path="$where/${date}_${suffix}-"
	local i=0

	while [ -e "${path}${i}" ]; do
	i="$(($i+1))"
	done

	printf "%s" "${path}${i}"
	}

	# Queue the mbox file for validation. Basically queuing is about moving the
	# file to the queue directory, and the Aiaiai dispatcher will then pick it from
	# there (it uses inotify to get notifications about new files in this
	# directories). This function also saves a copy of the mbox in the
	# 'queue_saved' directory. The mbox may contain a single patch, or entire
	# patch-set.
	queue_mboxfile()
	{
	local mbox="$1"; shift
	local n="$1"; shift
	local fname="$(generate_file_name "$queue" "$n")"

	cp $verbose -- "$mbox" "$queue_saved/${fname##*/}" >&2
	mv $verbose -- "$mbox" "$fname" >&2
	}

	find_relatives()
	{
	local where="$1"; shift
	local id="$1"; shift
	local parent_id="$1"; shift

	# Find the parent, children, and brothers
	LC_ALL=C grep -i -l -r -- "\(In-Reply-To\\|References\): $id" "$where" \|\|:
	if [ -n "$parent_id" ]; then
	LC_ALL=C grep -i -l -r -- "Message-Id: $parent_id" "$where" \|\|:
	LC_ALL=C grep -i -l -r -- "\(In-Reply-To\\|References\): $parent_id" "$where" \|\|:
	fi
	}

	series_is_complete()
	{
	local dir="$1"; shift
	local n="$1"; shift

	# First check if we have all the non-cover patches yet
	local cnt="$(ls -1 --ignore=0 -- "$dir" \| wc -l)"
	if [ "$cnt" -eq "$n" ]; then
	local first_parent="$(fetch_header "In-Reply-To" < "$dir/1")"
	[ -z "$first_parent" ] && first_parent="$(fetch_header "References" < "$dir/1")"
	if [ -n "$first_parent" ]; then
	# The first patch has a parent, must be the cover letter
	if [ -f "$dir/0" ]; then
	verbose "Series in $dir is complete and has cover letter"
	return 0
	else
	verbose "Series in $dir is not complete, we are missing the cover letter"
	return 1
	fi
	else
	verbose "Series is in $dir complete no cover letter was sent"
	return 0
	fi
	else
	verbose "Series in $dir is not complete, we have only $cnt out of $n patches"
	return 1
	fi
	}

	# Queue a complete series of patches.
	queue_series()
	{
	local mbox="$1"; shift
	local dir="$1"; shift
	local n="$1"; shift

	verbose "Patch-set at \"$dir\" is complete, queue it"
	# Don't add the 0th patch to the final mbox, as it is just the
	# cover letter and does not contain any patch
	for fname in $(ls --ignore=0 -A -- "$dir" \| sort -n); do
	cat -- "$dir/$fname" >> "$mbox"
	printf "\n" >> "$mbox"
	done

	if [ -f "$dir/0" ]; then
	# Save the subject and message ID of the cover letter in the
	# final mbox in order to be able to reply to the cover letter
	# later.
	local subj="$(fetch_header "Subject" < "$dir/0")"
	subj="X-Aiaiai-Cover-Letter-Subject: $subj"
	insert_header "$mbox" "$subj"

	local id="$(fetch_header "Message-Id" < "$dir/0")"
	id="X-Aiaiai-Cover-Letter-Message-Id: $id"
	insert_header "$mbox" "$id"
	fi
	queue_mboxfile "$mbox" "$n"
	rm $verbose -rf -- "$dir" >&2
	}

	move_to_series()
	{
	local file="$1"; shift
	local dir="$1"; shift
	local m="$1"; shift
	local n="$1"; shift
	local subj

	fetch_header_or_die subj "Subject" < "$file"
	m="$(subject_m "$subj")"

	verbose "Moving patch (\"$subj\") the series directory"

	# Sanity check - n has to be the same
	if ! [ "$(subject_n "$subj")" -eq "$n" ]; then
	reject "$file" "We were processing patch m/$n, but found a relative $m/$n"
	return
	fi

	# File $m must not exist
	if [ -f "$dir/$m" ]; then
	reject "$file" "File \"$dir/$m\" already exists!"
	else
	mv $verbose -- "$file" "$dir/$m" >&2
	fi
	}

	# Process a patch which is parte of a series. The patch is passed via "$1". Its
	# message ID is passed via "$2". Its number in the series is passed via "$3",
	# and amount of patches in the series is passed via "$4". This function will
	# try to collect all patches belonging to the series, and when it receives the
	# last patch, it queues the entire series.
	#
	# The series is collected using message ID headers - each patch, except for the
	# first one, must refer the previous patches ID via the "In-Reply-To:" header
	# or "References:" header.
	process_series_mbox()
	{
	local mbox="$1"; shift
	local id="$1"; shift
	local m="$1"; shift
	local n="$1"; shift
	local fname dir

	# Only patch 0/n or 1/n is allowed to have no parent
	local parent_id="$(fetch_header "In-Reply-To" < "$mbox")"
	[ -z "$parent_id" ] && parent_id="$(fetch_header "References" < "$mbox")"

	if [ -z "$parent_id" ] && [ "$m" != 1 ] && [ "$m" != 0 ]; then
	reject_and_reply "$mbox" <<EOF
	You sent a patch that does not conform to the requirements for Aiaiai's Local
	Delivery Agent. This patch is part of a series as indicated by its subject
	prefix. However, it does not contain the correct "In-Reply-To:" header or
	"References:" header which is required for Aiaiai to determine which patch
	series it belongs with.

	The most common reason for this issue is from not using the git-send-email
	utility, and using your own script which does not generate proper In-Reply-To:
	header chains.

	If you do use your own email commands, please add "--thread" to your
	git-format-patch commands, in order to ensure that the patch series
	contains proper threading.

	If you do not understand this email, or believe you have received this email in
	error, please contact "$cfg_adminname" <$cfg_adminmail>.
	EOF
	return
	fi

	[ -z "$parent_id" ] \|\| verbose "In-Reply-To: $parent_id"

	local staging_relatives
	local series_relatives
	staging_relatives="$(find_relatives "$staging" "$id" "$parent_id" \| sort -u)"
	series_relatives="$(find_relatives "$series" "$id" "$parent_id" \| sort -u)"

	verbose "Staging relatives: $staging_relatives"
	verbose "Series relatives: $series_relatives"

	if [ -z "$staging_relatives" ] && [ -z "$series_relatives" ]; then
	# Save the file in the staging area
	fname="$(generate_file_name "$staging" "$m-of-$n")"
	verbose "No relatives found, temporarily save in staging"
	mv $verbose -- "$mbox" "$fname" >&2
	return
	fi

	verbose "Found relatives"

	if [ -z "$series_relatives" ]; then
	# The series directory does not exist yet - create it
	dir="$(generate_file_name "$series" "$n")"
	verbose "Creating the series directory \"$dir\""
	mkdir $verbose -- "$dir" >&2
	else
	dir="$(printf "%s" "$series_relatives" \| sed -e "s/\(.\)\/./\1/" \| sort -u)"
	verbose "Found series directory \"$dir\""
	[ "$(printf "%s" "$dir" \| wc -l)" -eq 0 ] \|\|
	die "Relatives live in different series directories!"
	fi

	# Move the relatives from the staging to the series directory
	move_to_series "$mbox" "$dir" "$m" "$n"

	local relative
	for relative in $staging_relatives; do
	move_to_series "$relative" "$dir" "$m" "$n"
	done

	# If the series is complete - queue it
	if series_is_complete "$dir" "$n"; then
	queue_series "$mbox" "$dir" "$n"
	fi
	}

	separator()
	{
	if [ -n "$verbose" ]; then
	printf "\n%s" "----------------------------------------------------" >&2
	printf "%s\n\n" "----------------------------------------------------" >&2
	fi
	}

	# Proccess one e-mail. This e-mail may be part of a patch-set, may contain an
	# individual patch, or may not contain a patch at all. The patches and
	# non-patches are distinguished using is_email_patch() which is based on "git
	# apply --stat". Patch-set emails are distinquished from patches via use of the
	# subject line. Individual patches are queued right away. Patch-sets are stored
	# in a temporary location and queued when the entire patch set is collected.
	process_mbox()
	{
	local mbox="$1";

	separator

	# Make sure important headers are there
	grep -i -q -- "^Message-Id:[[:blank:]]" "$mbox" \|\|
	{ verbose "The \"Message-Id:\" header not found, ignoring"; return; }
	grep -i -q -- "^From:[[:blank:]]" "$mbox" \|\|
	{ verbose "The \"From:\" header not found, ignoring"; return; }
	grep -i -q -- "^To:[[:blank:]]" "$mbox" \|\|
	{ verbose "The \"To:\" header not found, ignoring"; return; }
	grep -i -q -- "^Subject:[[:blank:]]" "$mbox" \|\|
	{ verbose "The \"Subject:\" header not found, ignoring"; return; }

	local subj="$(fetch_header "Subject" < "$mbox")"
	local from="$(fetch_header "From" < "$mbox")"
	local id="$(fetch_header "Message-Id" < "$mbox")"

	verbose "Looking at: From: $from"
	verbose " Subject: $subj"
	verbose " Message-Id: $id"



	# Filter out e-mails which do not contain a patch
	if ! email_contains_patch "$mbox" ; then
	# Keep cover letter patches, but discard anything else
	[ "$(subject_m "$subj")" = "0" ] \|\|
	{ reject "$mbox" "mbox contains no patch and is not a cover letter"; return ; }
	fi

	# Queue patches which don't match subject format immediately
	if ! subject_check "$subj" ; then
	verbose "Queuing stand-alone patch with unrecognized subject format \"$subj\""
	queue_mboxfile "$mbox" ""
	return
	fi

	# If the patch prefix contains m/n, fetch m and n.
	local m="$(subject_m "$subj")"
	local n="$(subject_n "$subj")"

	if [ -z "$m" ]; then
	verbose "Queuing stand-alone patch \"$subj\""
	queue_mboxfile "$mbox" "$n"
	else
	verbose "Processing member $m/$n of a series (\"$subj\")"
	[ "$n" -ne 0 ] \|\| \
	{ reject "$mbox" "Prefix \"$prefix_format\" cannot have n = 0";
	return; }
	process_series_mbox "$mbox" "$id" "$m" "$n"
	fi
	}

	# This function goes through all the incomplete path series which we have
	# collected so far. And if the series happens to be complete, it queues them.
	# Normally, there should be no complete series in the "$series" directory.
	# However, sometimes this script misbehaves due to some bugs, and fails to add
	# a patch to the series, so it never gets queued. Then the Aiaiai admin may do
	# this manually. However, the series does not get queued anyway, because this
	# scrpt only checks the series when it adds a patch there.
	#
	# And this is where this function comes handy. It will go through the series
	# and queue all the complete ones.
	#
	# The only parameter of this function is the file to use as a temporary storage
	# for mboxes.
	process_all_series()
	{
	local mbox="$1"; shift
	local dir n

	separator
	message "Going through all partial series and checking if some of them became complete"

	# Get all the current series
	printf "%s\n" "$(find "$series" -mindepth 1 -maxdepth 1 -type d)" \| \
	while IFS= read -r dir; do
	[ -n "$dir" ] \|\| continue
	# Extract the series number
	n="${dir%-*}"
	n="${n##*_}"

	# Clear everything from the current mbox
	truncate -s0 -- "$mbox"

	if series_is_complete "$dir" "$n"; then
	queue_series "$mbox" "$dir" "$n"
	fi
	done
	}

	reap_old()
	{
	local dir="$1"; shift
	local min="$1"; shift

	verbose "Reaping files older than \"$min\" minutes in \"$dir\""
	find "$dir" -mindepth 1 -maxdepth 1 -mmin +"$min" -exec rm $verbose -r -f -- "{}" ";" >&2
	}

	mbox=
	lda_tmp_lock=
	cleanup_handler()
	{
	rm $verbose -rf -- "$mbox" >&2
	rm $verbose -rf -- "$lda_tmp_lock" >&2
	}
	set_cleanup_handler cleanup_handler

	TEMP=`getopt -n $PROG -o v,h --long reap-archive:,reap-incomplete:,verbose,help -- "$@"` \|\|
	fail_usage ""
	eval set -- "$TEMP"

	verbose=

	while true; do
	case "$1" in
	-v\|--verbose) verbose=-v
	;;
	-h\|--help)
	show_usage
	exit 0
	;;
	--) shift; break
	;;
	*) fail_usage "Unrecognized option: $1"
	;;
	esac
	shift
	done

	[ "$#" -eq 2 ] \|\| fail_usage "Insufficient or too many arguments"

	program_required "mutt" ""
	program_required "find" ""
	program_required "grep" ""
	program_required "sed" ""
	program_required "formail" ""

	workdir="$(readlink -fv -- "$1")"; shift
	cfgfile="$(readlink -fv -- "$1")"; shift

	parse_config "$cfgfile"

	lda_tmp="$workdir/lda_tmp"
	mkdir -p $verbose -- "$lda_tmp" >&2

	# In the staging directory we temporarily store patches which belong to a
	# patch-set but, but we do not know which one yet.
	staging="$lda_tmp/staging"
	# The series directory stores partial patch-sets.
	series="$lda_tmp/series"
	# The mail directory sores all the incoming mail
	mail="$workdir/mail"
	# The queue directory stores patches/patch-sets queued for further processing
	queue="$workdir/queue"
	# It is also convenient for debugging to store a copy of complete patch-series
	# separately
	queue_saved="$workdir/queue_saved"

	mkdir -p $verbose -- "$staging" "$series" "$mail" "$queue" "$queue_saved" >&2

	# All file we create have the date in the name
	date="$(date "+%Y-%m-%d_%H:%M:%S")"

	# We get the input from stdin, and it may contain several mails. We then
	# separate them, and process one-by-one. And the mail which is currently being
	# processed is stored in this temporary file.
	mbox="$(mktemp -t "$PROG.mbox.XXXX")"

	# We lock the lda and queue directories when using them
	lda_tmp_lock="$workdir/lda_tmp.lock"

	verbose "Taking $lda_tmp_lock lock file (timeout - 10 min)"
	lockfile -r 75 "$lda_tmp_lock"

	prev=
	first=y

	# Read stdin and separate out individual mails
	verbose "Saving incoming mbox in ${mail}/${date}.mbox"
	while IFS= read -r line; do
	printf "%s\n" "$line" >> "${mail}/${date}.mbox"
	if printf "%s" "$line" \| grep -i -q -- "^From [^@ ]\+@[^@ ]\+ .\+$" \|\| \
	printf "%s" "$line" \| grep -i -q -- "^From [[:xdigit:]]\{40\} .\+$"; then
	if [ -z "$first" ]; then
	if [ -n "$prev" ]; then
	verbose "The last line of previous mbox is not blank"
	verbose "line: $line"
	verbose "prev: $prev"
	die "exiting"
	fi
	process_mbox "$mbox"
	truncate -s0 -- "$mbox"
	fi
	else
	printf "%s\n" "$prev" >> "$mbox"
	fi
	prev="$line"
	first=
	done

	printf "%s\n" "$prev" >> "$mbox"
	process_mbox "$mbox"

	process_all_series "$mbox"

	[ -z "$cfg_lda_reap_incomplete" ] \|\| reap_old "$lda_tmp" "$cfg_lda_reap_incomplete"
	[ -z "$cfg_lda_reap_archive" ] \|\| reap_old "$mail" "$cfg_lda_reap_archive"
	[ -z "$cfg_lda_reap_archive" ] \|\| reap_old "$queue_saved" "$cfg_lda_reap_archive"

	rm $verbose -f -- "$lda_tmp_lock" >&2