t/lei-convert.t

#!perl -w
# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use v5.12; use PublicInbox::TestCommon;
use PublicInbox::MboxReader;
use PublicInbox::MdirReader;
use PublicInbox::NetReader;
use PublicInbox::Eml;
use IO::Uncompress::Gunzip;
use File::Path qw(remove_tree);
use PublicInbox::Spawn qw(which run_qx);
use File::Compare;
use autodie qw(open);
require_mods(qw(lei -imapd -nntpd Mail::IMAPClient Net::NNTP));
my ($tmpdir, $for_destroy) = tmpdir;
my $sock = tcp_server;
my $cmd = [ '-imapd', '-W0', "--stdout=$tmpdir/i1", "--stderr=$tmpdir/i2" ];
my ($ro_home, $cfg_path) = setup_public_inboxes;
my $env = { PI_CONFIG => $cfg_path };
my $tdi = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-imapd: $?");
my $imap_host_port = tcp_host_port($sock);
$sock = tcp_server;
$cmd = [ '-nntpd', '-W0', "--stdout=$tmpdir/n1", "--stderr=$tmpdir/n2" ];
my $tdn = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-nntpd: $?");
my $nntp_host_port = tcp_host_port($sock);
undef $sock;

test_lei({ tmpdir => $tmpdir }, sub {
	my $d = $ENV{HOME};
	lei_ok('convert', '-o', "mboxrd:$d/foo.mboxrd",
		"imap://$imap_host_port/t.v2.0");
	my ($nc0) = ($lei_err =~ /converted (\d+) messages/);
	ok(-f "$d/foo.mboxrd", 'mboxrd created from imap://');

	lei_ok qw(convert -o), "v2:$d/v2-test", "mboxrd:$d/foo.mboxrd";
	my ($nc) = ($lei_err =~ /converted (\d+) messages/);
	is $nc, $nc0, 'converted all messages messages';
	lei_ok qw(q z:0.. -f jsonl --only), "$d/v2-test";
	is(scalar(split(/^/sm, $lei_out)), $nc, 'got all messages in v2-test');

	lei_ok qw(convert -o), "mboxrd:$d/from-v2.mboxrd", "$d/v2-test";
	like $lei_err, qr/converted $nc messages/;
	is(compare("$d/foo.mboxrd", "$d/from-v2.mboxrd"), 0,
		'convert mboxrd -> v2 ->mboxrd roundtrip') or
			diag run_qx([qw(git diff --no-index),
					"$d/foo.mboxrd", "$d/from-v2.mboxrd"]);

	lei_ok [qw(convert -F eml -o), "$d/v2-test"], undef,
		{ 0 => \<<'EOM', %$lei_opt };
From: f@example.com
To: t@example.com
Subject: append-to-v2-on-convert
Message-ID: <append-to-v2-on-convert@example>
Date: Fri, 02 Oct 1993 00:00:00 +0000
EOM
	like $lei_err, qr/converted 1 messages/, 'only one message added';
	lei_ok qw(q z:0.. -f jsonl --only), "$d/v2-test";
	is(scalar(split(/^/sm, $lei_out)), $nc + 1,
		'got expected number of messages after append convert');
	like $lei_out, qr/append-to-v2-on-convert/;

	lei_ok('convert', '-o', "mboxrd:$d/nntp.mboxrd",
		"nntp://$nntp_host_port/t.v2");
	ok(-f "$d/nntp.mboxrd", 'mboxrd created from nntp://');

	my (@mboxrd, @mboxcl2);
	open my $fh, '<', "$d/foo.mboxrd" or BAIL_OUT $!;
	PublicInbox::MboxReader->mboxrd($fh, sub { push @mboxrd, shift });
	ok(scalar(@mboxrd) > 1, 'got multiple messages');

	open $fh, '<', "$d/nntp.mboxrd" or BAIL_OUT $!;
	my $i = 0;
	PublicInbox::MboxReader->mboxrd($fh, sub {
		my ($eml) = @_;
		is($eml->body, $mboxrd[$i]->body, "body matches #$i");
		$i++;
	});

	lei_ok('convert', '-o', "mboxcl2:$d/cl2", "mboxrd:$d/foo.mboxrd");
	ok(-s "$d/cl2", 'mboxcl2 non-empty') or diag $lei_err;
	open $fh, '<', "$d/cl2" or BAIL_OUT $!;
	PublicInbox::MboxReader->mboxcl2($fh, sub {
		my $eml = shift;
		$eml->header_set($_) for (qw(Content-Length Lines));
		push @mboxcl2, $eml;
	});
	is_deeply(\@mboxcl2, \@mboxrd, 'mboxrd and mboxcl2 have same mail');

	lei_ok('convert', '-o', "$d/md", "mboxrd:$d/foo.mboxrd");
	ok(-d "$d/md", 'Maildir created');
	my @md;
	PublicInbox::MdirReader->new->maildir_each_eml("$d/md", sub {
		push @md, $_[2];
	});
	is(scalar(@md), scalar(@mboxrd), 'got expected emails in Maildir') or
		diag $lei_err;
	@md = sort { ${$a->{bdy}} cmp ${$b->{bdy}} } @md;
	@mboxrd = sort { ${$a->{bdy}} cmp ${$b->{bdy}} } @mboxrd;
	my @rd_nostatus = map {
		my $eml = PublicInbox::Eml->new(\($_->as_string));
		$eml->header_set('Status');
		$eml;
	} @mboxrd;
	is_deeply(\@md, \@rd_nostatus, 'Maildir output matches mboxrd');

	my @bar;
	lei_ok('convert', '-o', "mboxrd:$d/bar.mboxrd", "$d/md");
	open $fh, '<', "$d/bar.mboxrd" or BAIL_OUT $!;
	PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
	@bar = sort { ${$a->{bdy}} cmp ${$b->{bdy}} } @bar;
	is_deeply(\@mboxrd, \@bar,
			'mboxrd round-tripped through Maildir w/ flags');

	open my $in, '<', "$d/foo.mboxrd" or BAIL_OUT;
	my $rdr = { 0 => $in, 1 => \(my $out), 2 => \$lei_err };
	lei_ok([qw(convert --stdin -F mboxrd -o mboxrd:/dev/stdout)],
		undef, $rdr);
	open $fh, '<', "$d/foo.mboxrd" or BAIL_OUT;
	my $exp = do { local $/; <$fh> };
	is($out, $exp, 'stdin => stdout');

	lei_ok qw(convert -F eml -o mboxcl2:/dev/fd/1 t/plack-qp.eml);
	open $fh, '<', \$lei_out or BAIL_OUT;
	@bar = ();
	PublicInbox::MboxReader->mboxcl2($fh, sub {
		my $eml = shift;
		for my $h (qw(Content-Length Lines)) {
			ok(defined($eml->header_raw($h)),
				"$h defined for mboxcl2");
			$eml->header_set($h);
		}
		push @bar, $eml;
	});
	my $qp_eml = eml_load('t/plack-qp.eml');
	$qp_eml->header_set('Status', 'O');
	is_deeply(\@bar, [ $qp_eml ], 'eml => mboxcl2');

	lei_ok qw(convert t/plack-qp.eml -o), "mboxrd:$d/qp.gz";
	open $fh, '<', "$d/qp.gz" or xbail $!;
	ok(-s $fh, 'not empty');
	$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
	@bar = ();
	PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
	is_deeply(\@bar, [ $qp_eml ], 'wrote gzipped mboxrd');
	lei_ok qw(convert -o mboxrd:/dev/stdout), "mboxrd:$d/qp.gz";
	open $fh, '<', \$lei_out or xbail;
	@bar = ();
	PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
	is_deeply(\@bar, [ $qp_eml ], 'readed gzipped mboxrd');

	# Status => Maildir flag => Status round trip
	$lei_out =~ s/^Status: O/Status: RO/sm or xbail "`seen' Status";
	$rdr = { 0 => \($in = $lei_out), %$lei_opt };
	lei_ok([qw(convert -F mboxrd -o), "$d/md2"], undef, $rdr);
	@md = glob("$d/md2/*/*");
	is(scalar(@md), 1, 'one message');
	like($md[0], qr/:2,S\z/, "`seen' flag set in Maildir");
	lei_ok(qw(convert -o mboxrd:/dev/stdout), "$d/md2");
	like($lei_out, qr/^Status: RO/sm, "`seen' flag preserved");

	SKIP: {
		my $ok;
		for my $x (($ENV{GZIP}//''), qw(pigz gzip)) {
			$x && (`$x -h 2>&1`//'') =~ /--rsyncable\b/s or next;
			$ok = $x;
			last;
		}
		skip 'pigz || gzip do not support --rsyncable', 1 if !$ok;
		lei_ok qw(convert --rsyncable), "mboxrd:$d/qp.gz",
			'-o', "mboxcl2:$d/qp2.gz";
		undef $fh; # necessary to make IO::Uncompress::Gunzip happy
		open $fh, '<', "$d/qp2.gz";
		$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
		my @tmp;
		PublicInbox::MboxReader->mboxcl2($fh, sub {
			my ($eml) = @_;
			$eml->header_set($_) for qw(Content-Length Lines);
			push @tmp, $eml;
		});
		is_deeply(\@tmp, \@bar, 'read rsyncable-gzipped mboxcl2');
	}
	my $cp = which('cp') or xbail 'cp(1) not available (WTF?)';
	for my $v (1, 2) {
		my $ibx_dir = "$ro_home/t$v";
		lei_ok qw(convert -f mboxrd), $ibx_dir,
				\"dump v$v inbox to mboxrd";
		my $out = $lei_out;
		lei_ok qw(convert -f mboxrd), "v$v:$ibx_dir",
				\"dump v$v inbox to mboxrd w/ v$v:// prefix";
		is $out, $lei_out, "v$v:// prefix accepted";
		open my $fh, '<', \$out;
		my (@mb, @md, @md2);
		PublicInbox::MboxReader->mboxrd($fh, sub {
			$_[0]->header_set('Status');
			push @mb, $_[0]->as_string;
		});
		undef $out;
		ok(scalar(@mb), 'got messages output');
		my $mdir = "$d/v$v-mdir";
		lei_ok qw(convert -o), $mdir, $ibx_dir,
			\"dump v$v inbox to Maildir";
		PublicInbox::MdirReader->new->maildir_each_eml($mdir, sub {
			push @md, $_[2]->as_string;
		});
		@md = sort { $a cmp $b } @md;
		@mb = sort { $a cmp $b } @mb;
		is_deeply(\@mb, \@md, 'got matching inboxes');
		xsys_e([$cp, '-Rp', $ibx_dir, "$d/tv$v" ]);
		remove_tree($mdir, "$d/tv$v/public-inbox",
				glob("$d/tv$v/xap*"));

		lei_ok qw(convert -o), $mdir, "$d/tv$v",
			\"dump u indexed v$v inbox to Maildir";
		PublicInbox::MdirReader->new->maildir_each_eml($mdir, sub {
			push @md2, $_[2]->as_string;
		});
		@md2 = sort { $a cmp $b } @md2;
		is_deeply(\@md, \@md2, 'got matching inboxes even unindexed');
	}
});
done_testing;