# -*- coding: utf-8 -*-
# Copyright (C) 2013-2020 by The Linux Foundation and contributors
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import grokmirror
import logging

import time
import json
import random
import datetime
import shutil
import gc
import fnmatch
import io
import smtplib

from pathlib import Path

from email.message import EmailMessage
from fcntl import lockf, LOCK_EX, LOCK_UN, LOCK_NB

# default basic logger. We override it later.
logger = logging.getLogger(__name__)


def log_errors(fullpath, cmdargs, lines):
    logger.critical('%s reports errors:', fullpath)
    with open(os.path.join(fullpath, 'grokmirror.fsck.err'), 'w') as fh:
        fh.write('# Date: %s\n' % datetime.datetime.today().strftime('%F'))
        fh.write('# Cmd : git %s\n' % ' '.join(cmdargs))
        count = 0
        for line in lines:
            fh.write('%s\n' % line)
            logger.critical('\t%s', line)
            count += 1
            if count > 10:
                logger.critical('\t [ %s more lines skipped ]', len(lines) - 10)
                logger.critical('\t [ see %s/grokmirror.fsck.err ]', os.path.basename(fullpath))
                break


def gen_preload_bundle(fullpath, config):
    outdir = config['fsck'].get('preload_bundle_outdir')
    Path(outdir).mkdir(parents=True, exist_ok=True)
    bname = '%s.bundle' % os.path.basename(fullpath)[:-4]
    args = ['bundle', 'create', os.path.join(outdir, bname), '--all']
    logger.info(' bundling: %s', bname)
    grokmirror.run_git_command(fullpath, args)


def get_blob_set(fullpath):
    bset = set()
    size = 0
    blobcache = os.path.join(fullpath, 'grokmirror.blobs')
    if os.path.exists(blobcache):
        # Did it age out? Hardcode to 30 days.
        expage = time.time() - 86400*30
        st = os.stat(blobcache)
        if st.st_mtime < expage:
            os.unlink(blobcache)
    try:
        with open(blobcache) as fh:
            while True:
                line = fh.readline()
                if not len(line):
                    break
                if line[0] == '#':
                    continue
                chunks = line.strip().split()
                bhash = chunks[0]
                bsize = int(chunks[1])
                size += bsize
                bset.add((bhash, bsize))
        return bset, size
    except FileNotFoundError:
        pass

    # This only makes sense for repos not using alternates, so make sure you check first
    logger.info(' bloblist: %s', fullpath)
    gitargs = ['cat-file', '--batch-all-objects', '--batch-check', '--unordered']
    retcode, output, error = grokmirror.run_git_command(fullpath, gitargs)
    if retcode == 0:
        with open(blobcache, 'w') as fh:
            fh.write('# Blobs and sizes used for sibling calculation\n')
            for line in output.split('\n'):
                if line.find(' blob ') < 0:
                    continue
                chunks = line.strip().split()
                fh.write(f'{chunks[0]} {chunks[2]}\n')
                bhash = chunks[0]
                bsize = int(chunks[2])
                size += bsize
                bset.add((bhash, bsize))

    return bset, size


def check_sibling_repos_by_blobs(bset1, bsize1, bset2, bsize2, ratio):
    iset = bset1.intersection(bset2)
    if not len(iset):
        return False
    isize = 0
    for bhash, bsize in iset:
        isize += bsize
    # Both repos should share at least ratio % of blobs in them
    ratio1 = int(isize / bsize1 * 100)
    logger.debug('isize=%s, bsize1=%s, ratio1=%s', isize, bsize1, ratio1)
    ratio2 = int(isize / bsize2 * 100)
    logger.debug('isize=%s, bsize2=%s ratio2=%s', isize, bsize2, ratio1)
    if ratio1 >= ratio and ratio2 >= ratio:
        return True

    return False


def find_siblings_by_blobs(obstrepo, obstdir, ratio=75):
    siblings = set()
    oset, osize = get_blob_set(obstrepo)
    for srepo in grokmirror.find_all_gitdirs(obstdir, normalize=True, exclude_objstore=False):
        if srepo == obstrepo:
            continue
        logger.debug('Comparing blobs between %s and %s', obstrepo, srepo)
        sset, ssize = get_blob_set(srepo)
        if check_sibling_repos_by_blobs(oset, osize, sset, ssize, ratio):
            logger.info(' siblings: %s and %s', os.path.basename(obstrepo), os.path.basename(srepo))
            siblings.add(srepo)

    return siblings


def merge_siblings(siblings, amap):
    mdest = None
    rcount = 0
    # Who has the most remotes?
    for sibling in set(siblings):
        if sibling not in amap or not len(amap[sibling]):
            # Orphaned sibling, ignore it -- it will get cleaned up
            siblings.remove(sibling)
            continue
        s_remotes = grokmirror.list_repo_remotes(sibling)
        if len(s_remotes) > rcount:
            mdest = sibling
            rcount = len(s_remotes)

    # Migrate all siblings into the repo with most remotes
    siblings.remove(mdest)
    for sibling in siblings:
        logger.info('%s: merging into %s', os.path.basename(sibling), os.path.basename(mdest))
        s_remotes = grokmirror.list_repo_remotes(sibling, withurl=True)
        for virtref, childpath in s_remotes:
            if childpath not in amap[sibling]:
                # The child repo isn't even using us
                args = ['remote', 'remove', virtref]
                grokmirror.run_git_command(sibling, args)
                continue
            logger.info('   moving: %s', childpath)

            success = grokmirror.add_repo_to_objstore(mdest, childpath)
            if not success:
                logger.critical('Could not add %s to %s', childpath, mdest)
                continue

            logger.info('         : fetching into %s', os.path.basename(mdest))
            success = grokmirror.fetch_objstore_repo(mdest, childpath)
            if not success:
                logger.critical('Failed to fetch %s from %s to %s', childpath, os.path.basename(sibling),
                                os.path.basename(mdest))
                continue
            logger.info('         : repointing alternates')
            grokmirror.set_altrepo(childpath, mdest)
            amap[sibling].remove(childpath)
            amap[mdest].add(childpath)
            args = ['remote', 'remove', virtref]
            grokmirror.run_git_command(sibling, args)
            logger.info('         : done')

    return mdest


def check_reclone_error(fullpath, config, errors):
    reclone = None
    toplevel = os.path.realpath(config['core'].get('toplevel'))
    errlist = config['fsck'].get('reclone_on_errors', '').split('\n')
    for line in errors:
        for estring in errlist:
            if line.find(estring) != -1:
                # is this repo used for alternates?
                gitdir = '/' + os.path.relpath(fullpath, toplevel).lstrip('/')
                if grokmirror.is_alt_repo(toplevel, gitdir):
                    logger.critical('\tused for alternates, not requesting auto-reclone')
                    return
                else:
                    reclone = line
                    logger.critical('\trequested auto-reclone')
                break
            if reclone is not None:
                break
    if reclone is None:
        return

    set_repo_reclone(fullpath, reclone)


def get_repo_size(fullpath):
    oi = grokmirror.get_repo_obj_info(fullpath)
    kbsize = 0
    for field in ['size', 'size-pack', 'size-garbage']:
        try:
            kbsize += int(oi[field])
        except (KeyError, ValueError):
            pass
    logger.debug('%s size: %s kb', fullpath, kbsize)
    return kbsize


def get_human_size(kbsize):
    num = kbsize
    for unit in ['Ki', 'Mi', 'Gi']:
        if abs(num) < 1024.0:
            return "%3.2f %sB" % (num, unit)
        num /= 1024.0
    return "%.2f%s TiB" % num


def set_repo_reclone(fullpath, reason):
    rfile = os.path.join(fullpath, 'grokmirror.reclone')
    # Have we already requested a reclone?
    if os.path.exists(rfile):
        logger.debug('Already requested repo reclone for %s', fullpath)
        return

    with open(rfile, 'w') as rfh:
        rfh.write('Requested by grok-fsck due to error: %s' % reason)


def run_git_prune(fullpath, config):
    # WARNING: We assume you've already verified that it's safe to do so
    prune_ok = True
    isprecious = grokmirror.is_precious(fullpath)
    if isprecious:
        set_precious_objects(fullpath, False)

    # We set expire to yesterday in order to avoid race conditions
    # in repositories that are actively being accessed at the time of
    # running the prune job.
    args = ['prune', '--expire=yesterday']
    logger.info('    prune: pruning')
    retcode, output, error = grokmirror.run_git_command(fullpath, args)

    if error:
        warn = remove_ignored_errors(error, config)
        if warn:
            prune_ok = False
            log_errors(fullpath, args, warn)
            check_reclone_error(fullpath, config, warn)

    if isprecious:
        set_precious_objects(fullpath, True)

    return prune_ok


def is_safe_to_prune(fullpath, config):
    if config['fsck'].get('prune', 'yes') != 'yes':
        logger.debug('Pruning disabled in config file')
        return False
    toplevel = os.path.realpath(config['core'].get('toplevel'))
    obstdir = os.path.realpath(config['core'].get('objstore'))
    gitdir = '/' + os.path.relpath(fullpath, toplevel).lstrip('/')
    if grokmirror.is_obstrepo(fullpath, obstdir):
        # We only prune if all repos pointing to us are public
        urls = set(grokmirror.list_repo_remotes(fullpath, withurl=True))
        mine = set([x[1] for x in urls])
        amap = grokmirror.get_altrepo_map(toplevel)
        if mine != amap[fullpath]:
            logger.debug('Cannot prune %s because it is used by non-public repos', gitdir)
            return False
    elif grokmirror.is_alt_repo(toplevel, gitdir):
        logger.debug('Cannot prune %s because it is used as alternates by other repos', gitdir)
        return False

    logger.debug('%s should be safe to prune', gitdir)
    return True


def remove_ignored_errors(output, config):
    ierrors = set([x.strip() for x in config['fsck'].get('ignore_errors', '').split('\n')])
    debug = list()
    warn = list()
    for line in output.split('\n'):
        line = line.strip()
        # ignore any blank linkes
        if not len(line):
            continue
        ignored = False
        for estring in ierrors:
            if line.find(estring) != -1:
                ignored = True
                debug.append(line)
                break
        if not ignored:
            warn.append(line)
    if debug:
        logger.debug('Stderr: %s', '\n'.join(debug))

    return warn


def run_git_repack(fullpath, config, level=1, prune=True):
    # Returns false if we hit any errors on the way
    repack_ok = True
    obstdir = os.path.realpath(config['core'].get('objstore'))
    toplevel = os.path.realpath(config['core'].get('toplevel'))
    gitdir = '/' + os.path.relpath(fullpath, toplevel).lstrip('/')

    if prune:
        # Make sure it's safe to do so
        prune = is_safe_to_prune(fullpath, config)

    if config['fsck'].get('precious', '') == 'always':
        always_precious = True
        set_precious_objects(fullpath, enabled=True)
    else:
        always_precious = False
        set_precious_objects(fullpath, enabled=False)

    set_precious_after = False
    gen_commitgraph = True

    # Figure out what our repack flags should be.
    repack_flags = list()
    rregular = config['fsck'].get('extra_repack_flags', '').split()
    if len(rregular):
        repack_flags += rregular

    full_repack_flags = ['-f', '--pack-kept-objects']
    rfull = config['fsck'].get('extra_repack_flags_full', '').split()
    if len(rfull):
        full_repack_flags += rfull

    if grokmirror.is_obstrepo(fullpath, obstdir):
        set_precious_after = True
        repack_flags.append('-a')
        if not prune and not always_precious:
            repack_flags.append('-k')

    elif grokmirror.is_alt_repo(toplevel, gitdir):
        set_precious_after = True
        if grokmirror.get_altrepo(fullpath):
            gen_commitgraph = False
            logger.warning(' warning : has alternates and is used by others for alternates')
            logger.warning('         : this can cause grandchild corruption')
            repack_flags.append('-A')
            repack_flags.append('-l')
        else:
            repack_flags.append('-a')
            repack_flags.append('-b')
            if not always_precious:
                repack_flags.append('-k')

    elif grokmirror.get_altrepo(fullpath):
        # we are a "child repo"
        gen_commitgraph = False
        repack_flags.append('-l')
        repack_flags.append('-A')
        if prune:
            repack_flags.append('--unpack-unreachable=yesterday')

    else:
        # we have no relationships with other repos
        repack_flags.append('-a')
        repack_flags.append('-b')
        if prune:
            repack_flags.append('--unpack-unreachable=yesterday')

    if level > 1:
        logger.info('   repack: performing a full repack for optimal deltas')
        repack_flags += full_repack_flags

    if not always_precious:
        repack_flags.append('-d')

    # If we have a logs dir, then run reflog expire
    if os.path.isdir(os.path.join(fullpath, 'logs')):
        args = ['reflog', 'expire', '--all', '--stale-fix']
        logger.info('   reflog: expiring reflogs')
        grokmirror.run_git_command(fullpath, args)

    args = ['repack'] + repack_flags
    logger.info('   repack: repacking with "%s"', ' '.join(repack_flags))

    # We always tack on -q
    args.append('-q')

    retcode, output, error = grokmirror.run_git_command(fullpath, args)

    # With newer versions of git, repack may return warnings that are safe to ignore
    # so use the same strategy to weed out things we aren't interested in seeing
    if error:
        warn = remove_ignored_errors(error, config)
        if warn:
            repack_ok = False
            log_errors(fullpath, args, warn)
            check_reclone_error(fullpath, config, warn)

    if not repack_ok:
        # No need to repack refs if repo is broken
        if set_precious_after:
            set_precious_objects(fullpath, enabled=True)
        return False

    if gen_commitgraph and config['fsck'].get('commitgraph', 'yes') == 'yes':
        grokmirror.set_git_config(fullpath, 'core.commitgraph', 'true')
        run_git_commit_graph(fullpath)

    # repacking refs requires a separate command, so run it now
    args = ['pack-refs']
    if level > 1:
        logger.info(' packrefs: repacking all refs')
        args.append('--all')
    else:
        logger.info(' packrefs: repacking refs')
    retcode, output, error = grokmirror.run_git_command(fullpath, args)

    # pack-refs shouldn't return anything, but use the same ignore_errors block
    # to weed out any future potential benign warnings
    if error:
        warn = remove_ignored_errors(error, config)
        if warn:
            repack_ok = False
            log_errors(fullpath, args, warn)
            check_reclone_error(fullpath, config, warn)

    if prune:
        repack_ok = run_git_prune(fullpath, config)

    if set_precious_after:
        set_precious_objects(fullpath, enabled=True)

    return repack_ok


def run_git_fsck(fullpath, config, conn_only=False):
    args = ['fsck', '--no-progress', '--no-dangling', '--no-reflogs']
    obstdir = os.path.realpath(config['core'].get('objstore'))
    # If it's got an obstrepo, always run as connectivity-only
    altrepo = grokmirror.get_altrepo(fullpath)
    if altrepo and grokmirror.is_obstrepo(altrepo, obstdir):
        logger.debug('Repo uses objstore, forcing connectivity-only')
        conn_only = True
    if conn_only:
        args.append('--connectivity-only')
        logger.info('     fsck: running with --connectivity-only')
    else:
        logger.info('     fsck: running full checks')

    retcode, output, error = grokmirror.run_git_command(fullpath, args)
    output = output + '\n' + error

    if output:
        warn = remove_ignored_errors(output, config)
        if warn:
            log_errors(fullpath, args, warn)
            check_reclone_error(fullpath, config, warn)


def run_git_commit_graph(fullpath):
    # Does our version of git support commit-graph?
    if not grokmirror.git_newer_than('2.18.0'):
        logger.debug('Git version too old, not generating commit-graph')
    logger.info('    graph: generating commit-graph')
    args = ['commit-graph', 'write']
    retcode, output, error = grokmirror.run_git_command(fullpath, args)
    if retcode == 0:
        return True

    return False


def set_precious_objects(fullpath, enabled=True):
    # It's better to just set it blindly without checking first,
    # as this results in one fewer shell-out.
    logger.debug('Setting preciousObjects for %s', fullpath)
    if enabled:
        poval = 'true'
    else:
        poval = 'false'
    grokmirror.set_git_config(fullpath, 'extensions.preciousObjects', poval)


def check_precious_objects(fullpath):
    return grokmirror.is_precious(fullpath)


def fsck_mirror(config, force=False, repack_only=False, conn_only=False,
                repack_all_quick=False, repack_all_full=False):

    if repack_all_quick or repack_all_full:
        force = True

    statusfile = config['fsck'].get('statusfile')
    if not statusfile:
        logger.critical('Please define fsck.statusfile in the config')
        return 1

    st_dir = os.path.dirname(statusfile)
    if not os.path.isdir(os.path.dirname(statusfile)):
        logger.critical('Directory %s is absent', st_dir)
        return 1

    # Lock the tree to make sure we only run one instance
    lockfile = os.path.join(st_dir, '.%s.lock' % os.path.basename(statusfile))
    logger.debug('Attempting to obtain lock on %s', lockfile)
    flockh = open(lockfile, 'w')
    try:
        lockf(flockh, LOCK_EX | LOCK_NB)
    except IOError:
        logger.info('Could not obtain exclusive lock on %s', lockfile)
        logger.info('Assuming another process is running.')
        return 0

    manifile = config['core'].get('manifest')
    logger.info('Analyzing %s', manifile)
    grokmirror.manifest_lock(manifile)
    manifest = grokmirror.read_manifest(manifile)

    if os.path.exists(statusfile):
        logger.info('   status: reading %s', statusfile)
        stfh = open(statusfile, 'r')
        # noinspection PyBroadException
        try:
            # Format of the status file:
            #  {
            #    '/full/path/to/repository': {
            #      'lastcheck': 'YYYY-MM-DD' or 'never',
            #      'nextcheck': 'YYYY-MM-DD',
            #      'lastrepack': 'YYYY-MM-DD',
            #      'fingerprint': 'sha-1',
            #      's_elapsed': seconds,
            #      'quick_repack_count': times,
            #    },
            #    ...
            #  }

            status = json.loads(stfh.read())
        except:
            logger.critical('Failed to parse %s', statusfile)
            lockf(flockh, LOCK_UN)
            flockh.close()
            return 1
    else:
        status = dict()

    frequency = config['fsck'].getint('frequency', 30)

    today = datetime.datetime.today()
    todayiso = today.strftime('%F')

    if force:
        # Use randomization for next check, again
        checkdelay = random.randint(1, frequency)
    else:
        checkdelay = frequency

    commitgraph = config['fsck'].getboolean('commitgraph', True)

    # Is our git version new enough to support it?
    if commitgraph and not grokmirror.git_newer_than('2.18.0'):
        logger.info('Git version too old to support commit graphs, disabling')
        config['fsck']['commitgraph'] = 'no'

    # Go through the manifest and compare with status
    toplevel = os.path.realpath(config['core'].get('toplevel'))
    changed = False
    for gitdir in list(manifest):
        fullpath = os.path.join(toplevel, gitdir.lstrip('/'))
        # Does it exist?
        if not os.path.isdir(fullpath):
            # Remove it from manifest and status
            manifest.pop(gitdir)
            try:
                status.pop(fullpath)
            except KeyError:
                pass
            changed = True
            continue

        if fullpath not in status.keys():
            # Newly added repository
            if not force:
                # Randomize next check between now and frequency
                delay = random.randint(0, frequency)
                nextdate = today + datetime.timedelta(days=delay)
                nextcheck = nextdate.strftime('%F')
            else:
                nextcheck = todayiso

            status[fullpath] = {
                'lastcheck': 'never',
                'nextcheck': nextcheck,
                'fingerprint': grokmirror.get_repo_fingerprint(toplevel, gitdir),
            }
            logger.info('%s:', fullpath)
            logger.info('    added: next check on %s', nextcheck)

    if 'manifest' in config:
        pretty = config['manifest'].getboolean('pretty', False)
    else:
        pretty = False

    if changed:
        grokmirror.write_manifest(manifile, manifest, pretty=pretty)

    grokmirror.manifest_unlock(manifile)

    # record newly found repos in the status file
    logger.debug('Updating status file in %s', statusfile)
    with open(statusfile, 'w') as stfh:
        stfh.write(json.dumps(status, indent=2))

    # Go through status and find all repos that need work done on them.
    to_process = set()

    total_checked = 0
    total_elapsed = 0
    space_saved = 0

    cfg_repack = config['fsck'].getboolean('repack', True)
    # Can be "always", which is why we don't getboolean
    cfg_precious = config['fsck'].get('precious', 'yes')

    obstdir = os.path.realpath(config['core'].get('objstore'))
    logger.info('   search: getting parent commit info from all repos, may take a while')
    top_roots, obst_roots = grokmirror.get_rootsets(toplevel, obstdir)
    amap = grokmirror.get_altrepo_map(toplevel)

    fetched_obstrepos = set()
    obst_changes = False
    analyzed = 0
    queued = 0
    logger.info('Analyzing %s (%s repos)', toplevel, len(status))
    stattime = time.time()
    baselines = [x.strip() for x in config['fsck'].get('baselines', '').split('\n')]
    for fullpath in list(status):
        # Give me a status every 5 seconds
        if time.time() - stattime >= 5:
            logger.info('      ---: %s/%s analyzed, %s queued', analyzed, len(status), queued)
            stattime = time.time()
        start_size = get_repo_size(fullpath)
        analyzed += 1
        # We do obstrepos separately below, as logic is different
        if grokmirror.is_obstrepo(fullpath, obstdir):
            logger.debug('Skipping %s (obstrepo)')
            continue

        # Check to make sure it's still in the manifest
        gitdir = fullpath.replace(toplevel, '', 1)
        gitdir = '/' + gitdir.lstrip('/')

        if gitdir not in manifest:
            status.pop(fullpath)
            logger.debug('%s is gone, no longer in manifest', gitdir)
            continue

        # Make sure FETCH_HEAD is pointing to /dev/null
        fetch_headf = os.path.join(fullpath, 'FETCH_HEAD')
        if not os.path.islink(fetch_headf):
            logger.debug('  replacing FETCH_HEAD with symlink to /dev/null')
            try:
                os.unlink(fetch_headf)
            except FileNotFoundError:
                pass
            os.symlink('/dev/null', fetch_headf)

        # Objstore migration routines
        # Are we using objstore?
        altdir = grokmirror.get_altrepo(fullpath)
        is_private = grokmirror.is_private_repo(config, gitdir)
        if grokmirror.is_alt_repo(toplevel, gitdir):
            # Don't prune any repos that are parents -- until migration is fully complete
            m_prune = False
        else:
            m_prune = True

        if not altdir and not os.path.exists(os.path.join(fullpath, 'grokmirror.do-not-objstore')):
            # Do we match any obstdir repos?
            obstrepo = grokmirror.find_best_obstrepo(fullpath, obst_roots, toplevel, baselines)
            if obstrepo:
                obst_changes = True
                # Yes, set ourselves up to be using that obstdir
                logger.info('%s: can use %s', gitdir, os.path.basename(obstrepo))
                grokmirror.set_altrepo(fullpath, obstrepo)
                if not is_private:
                    grokmirror.add_repo_to_objstore(obstrepo, fullpath)
                    # Fetch into the obstrepo
                    logger.info('    fetch: fetching %s', gitdir)
                    grokmirror.fetch_objstore_repo(obstrepo, fullpath)
                    obst_roots[obstrepo] = grokmirror.get_repo_roots(obstrepo, force=True)
                run_git_repack(fullpath, config, level=1, prune=m_prune)
                space_saved += start_size - get_repo_size(fullpath)
            else:
                # Do we have any toplevel siblings?
                obstrepo = None
                my_roots = grokmirror.get_repo_roots(fullpath)
                top_siblings = grokmirror.find_siblings(fullpath, my_roots, top_roots)
                if len(top_siblings):
                    # Am I a private repo?
                    if is_private:
                        # Are there any non-private siblings?
                        for top_sibling in top_siblings:
                            # Are you a private repo?
                            if grokmirror.is_private_repo(config, top_sibling):
                                continue
                            # Great, make an objstore repo out of this sibling
                            obstrepo = grokmirror.setup_objstore_repo(obstdir)
                            logger.info('%s: can use %s', gitdir, os.path.basename(obstrepo))
                            logger.info('     init: new objstore repo %s', os.path.basename(obstrepo))
                            grokmirror.add_repo_to_objstore(obstrepo, top_sibling)
                            # Fetch into the obstrepo
                            logger.info('    fetch: fetching %s', top_sibling)
                            grokmirror.fetch_objstore_repo(obstrepo, top_sibling)
                            obst_roots[obstrepo] = grokmirror.get_repo_roots(obstrepo, force=True)
                            # It doesn't matter if this fails, because repacking is still safe
                            # Other siblings will match in their own due course
                            break
                    else:
                        # Make an objstore repo out of myself
                        obstrepo = grokmirror.setup_objstore_repo(obstdir)
                        logger.info('%s: can use %s', gitdir, os.path.basename(obstrepo))
                        logger.info('     init: new objstore repo %s', os.path.basename(obstrepo))
                        grokmirror.add_repo_to_objstore(obstrepo, fullpath)

                if obstrepo:
                    obst_changes = True
                    # Set alternates to the obstrepo
                    grokmirror.set_altrepo(fullpath, obstrepo)
                    if not is_private:
                        # Fetch into the obstrepo
                        logger.info('    fetch: fetching %s', gitdir)
                        grokmirror.fetch_objstore_repo(obstrepo, fullpath)
                    run_git_repack(fullpath, config, level=1, prune=m_prune)
                    space_saved += start_size - get_repo_size(fullpath)
                    obst_roots[obstrepo] = grokmirror.get_repo_roots(obstrepo, force=True)

        elif not os.path.isdir(altdir):
            logger.critical('  reclone: %s (alternates repo gone)', gitdir)
            set_repo_reclone(fullpath, 'Alternates repository gone')
            continue

        elif altdir.find(obstdir) != 0:
            # We have an alternates repo, but it's not an objstore repo
            # Probably left over from grokmirror-1.x
            # Do we have any matching obstrepos?
            obstrepo = grokmirror.find_best_obstrepo(fullpath, obst_roots, toplevel, baselines)
            if obstrepo:
                logger.info('%s: migrating to %s', gitdir, os.path.basename(obstrepo))
                if altdir not in fetched_obstrepos:
                    # We're already sharing objects with altdir, so no need to check if it's private
                    grokmirror.add_repo_to_objstore(obstrepo, altdir)
                    logger.info('    fetch: fetching %s (previous parent)', os.path.relpath(altdir, toplevel))
                    success = grokmirror.fetch_objstore_repo(obstrepo, altdir)
                    fetched_obstrepos.add(altdir)
                    if success:
                        set_precious_objects(altdir, enabled=False)
                        pre_size = get_repo_size(altdir)
                        run_git_repack(altdir, config, level=1, prune=False)
                        space_saved += pre_size - get_repo_size(altdir)
                    else:
                        logger.critical('Unsuccessful fetching %s into %s', altdir, os.path.basename(obstrepo))
                        obstrepo = None
            else:
                # Make a new obstrepo out of mommy
                obstrepo = grokmirror.setup_objstore_repo(obstdir)
                logger.info('%s: migrating to %s', gitdir, os.path.basename(obstrepo))
                logger.info('     init: new objstore repo %s', os.path.basename(obstrepo))
                grokmirror.add_repo_to_objstore(obstrepo, altdir)
                logger.info('    fetch: fetching %s (previous parent)', os.path.relpath(altdir, toplevel))
                success = grokmirror.fetch_objstore_repo(obstrepo, altdir)
                fetched_obstrepos.add(altdir)
                if success:
                    grokmirror.set_altrepo(altdir, obstrepo)
                    # mommy is no longer precious
                    set_precious_objects(altdir, enabled=False)
                    # Don't prune, because there may be objects others are still borrowing
                    # It can only be pruned once the full migration is completed
                    pre_size = get_repo_size(altdir)
                    run_git_repack(altdir, config, level=1, prune=False)
                    space_saved += pre_size - get_repo_size(altdir)
                else:
                    logger.critical('Unsuccessful fetching %s into %s', altdir, os.path.basename(obstrepo))
                    obstrepo = None

            if obstrepo:
                obst_changes = True
                if not is_private:
                    # Fetch into the obstrepo
                    grokmirror.add_repo_to_objstore(obstrepo, fullpath)
                    logger.info('    fetch: fetching %s', gitdir)
                    if grokmirror.fetch_objstore_repo(obstrepo, fullpath):
                        grokmirror.set_altrepo(fullpath, obstrepo)
                        set_precious_objects(fullpath, enabled=False)
                        run_git_repack(fullpath, config, level=1, prune=m_prune)
                        space_saved += start_size - get_repo_size(fullpath)
                else:
                    # Grab all the objects from the previous parent, since we can't simply
                    # fetch ourselves into the obstrepo (we're private).
                    args = ['repack', '-a']
                    logger.info('    fetch: restoring private repo %s', gitdir)
                    if grokmirror.run_git_command(fullpath, args):
                        grokmirror.set_altrepo(fullpath, obstrepo)
                        set_precious_objects(fullpath, enabled=False)
                        # Now repack ourselves to get rid of any public objects
                        run_git_repack(fullpath, config, level=1, prune=m_prune)

                obst_roots[obstrepo] = grokmirror.get_repo_roots(obstrepo, force=True)

        elif altdir.find(obstdir) == 0 and not is_private:
            # Make sure this repo is properly set up with obstrepo
            # (e.g. it could have been cloned/copied and obstrepo is not tracking it yet)
            obstrepo = altdir
            s_remotes = grokmirror.list_repo_remotes(obstrepo, withurl=True)
            found = False
            for virtref, childpath in s_remotes:
                if childpath == fullpath:
                    found = True
                    break
            if not found:
                # Set it up properly
                grokmirror.add_repo_to_objstore(obstrepo, fullpath)
                logger.info(' reconfig: %s to fetch into %s', gitdir, os.path.basename(obstrepo))

        obj_info = grokmirror.get_repo_obj_info(fullpath)
        try:
            packs = int(obj_info['packs'])
            count_loose = int(obj_info['count'])
        except KeyError:
            logger.warning('Unable to count objects in %s, skipping' % fullpath)
            continue

        schedcheck = datetime.datetime.strptime(status[fullpath]['nextcheck'], '%Y-%m-%d')
        nextcheck = today + datetime.timedelta(days=checkdelay)

        if not cfg_repack:
            # don't look at me if you turned off repack
            logger.debug('Not repacking because repack=no in config')
            repack_level = None
        elif repack_all_full and (count_loose > 0 or packs > 1):
            logger.debug('repack_level=2 due to repack_all_full')
            repack_level = 2
        elif repack_all_quick and count_loose > 0:
            logger.debug('repack_level=1 due to repack_all_quick')
            repack_level = 1
        elif status[fullpath].get('fingerprint') != grokmirror.get_repo_fingerprint(toplevel, gitdir):
            logger.debug('Checking repack level of %s', fullpath)
            repack_level = grokmirror.get_repack_level(obj_info)
        else:
            repack_level = None

        # trigger a level-1 repack if it's regular check time and the fingerprint has changed
        if (not repack_level and schedcheck <= today
                and status[fullpath].get('fingerprint') != grokmirror.get_repo_fingerprint(toplevel, gitdir)):
            status[fullpath]['nextcheck'] = nextcheck.strftime('%F')
            logger.info('     aged: %s (forcing repack)', fullpath)
            repack_level = 1

        # If we're not already repacking the repo, run a prune if we find garbage in it
        if obj_info['garbage'] != '0' and not repack_level and is_safe_to_prune(fullpath, config):
            logger.info('  garbage: %s (%s files, %s KiB)', gitdir, obj_info['garbage'], obj_info['size-garbage'])
            try:
                grokmirror.lock_repo(fullpath, nonblocking=True)
                run_git_prune(fullpath, config)
                grokmirror.unlock_repo(fullpath)
            except IOError:
                pass

        if repack_level and (cfg_precious == 'always' and check_precious_objects(fullpath)):
            # if we have preciousObjects, then we only repack based on the same
            # schedule as fsck.
            logger.debug('preciousObjects is set')
            # for repos with preciousObjects, we use the fsck schedule for repacking
            if schedcheck <= today:
                logger.debug('Time for a full periodic repack of a preciousObjects repo')
                status[fullpath]['nextcheck'] = nextcheck.strftime('%F')
                repack_level = 2
            else:
                logger.debug('Not repacking preciousObjects repo outside of schedule')
                repack_level = None

        if repack_level:
            queued += 1
            to_process.add((fullpath, 'repack', repack_level))
            if repack_level > 1:
                logger.info('   queued: %s (full repack)', fullpath)
            else:
                logger.info('   queued: %s (repack)', fullpath)
        elif repack_only or repack_all_quick or repack_all_full:
            continue
        elif schedcheck <= today or force:
            queued += 1
            to_process.add((fullpath, 'fsck', None))
            logger.info('   queued: %s (fsck)', fullpath)

    logger.info('     done: %s analyzed, %s queued', analyzed, queued)

    if obst_changes:
        # Refresh the alt repo map cache
        amap = grokmirror.get_altrepo_map(toplevel, refresh=True)

    obstrepos = grokmirror.find_all_gitdirs(obstdir, normalize=True, exclude_objstore=False)

    analyzed = 0
    queued = 0
    logger.info('Analyzing %s (%s repos)', obstdir, len(obstrepos))
    objstore_uses_plumbing = config['core'].getboolean('objstore_uses_plumbing', False)
    islandcores = [x.strip() for x in config['fsck'].get('islandcores', '').split('\n')]
    stattime = time.time()
    for obstrepo in obstrepos:
        if time.time() - stattime >= 5:
            logger.info('      ---: %s/%s analyzed, %s queued', analyzed, len(obstrepos), queued)
            stattime = time.time()
        analyzed += 1
        logger.debug('Processing objstore repo: %s', os.path.basename(obstrepo))
        my_roots = grokmirror.get_repo_roots(obstrepo)
        if obstrepo in amap and len(amap[obstrepo]):
            # Is it redundant with any other objstore repos?
            strategy = config['fsck'].get('obstrepo_merge_strategy', 'exact')
            if strategy == 'blobs':
                siblings = find_siblings_by_blobs(obstrepo, obstdir, ratio=75)
            else:
                exact_merge = True
                if strategy == 'loose':
                    exact_merge = False
                siblings = grokmirror.find_siblings(obstrepo, my_roots, obst_roots, exact=exact_merge)
            if len(siblings):
                siblings.add(obstrepo)
                mdest = merge_siblings(siblings, amap)
                obst_changes = True
                if mdest in status:
                    # Force full repack of merged obstrepos
                    status[mdest]['nextcheck'] = todayiso

                # Recalculate my roots
                my_roots = grokmirror.get_repo_roots(obstrepo, force=True)
                obst_roots[obstrepo] = my_roots

        # Not an else, because the previous step may have migrated things
        if obstrepo not in amap or not len(amap[obstrepo]):
            obst_changes = True
            # XXX: Is there a possible race condition here if grok-pull cloned a new repo
            #      while we were migrating this one?
            logger.info('%s: deleting (no longer used by anything)', os.path.basename(obstrepo))
            if obstrepo in amap:
                amap.pop(obstrepo)
            shutil.rmtree(obstrepo)
            continue

        # Record the latest sibling info in the tracking file
        telltale = os.path.join(obstrepo, 'grokmirror.objstore')
        with open(telltale, 'w') as fh:
            fh.write(grokmirror.OBST_PREAMBULE)
            fh.write('\n'.join(sorted(list(amap[obstrepo]))) + '\n')

        my_remotes = grokmirror.list_repo_remotes(obstrepo, withurl=True)
        # Use the first child repo as our "reference" entry in manifest
        refrepo = None
        # Use for the alternateRefsPrefixes value
        baseline_refs = set()
        set_islandcore = False
        new_islandcore = False
        valid_virtrefs = set()
        for virtref, childpath in my_remotes:
            # Is it still relevant?
            if childpath not in amap[obstrepo]:
                # Remove it and let prune take care of it
                grokmirror.remove_from_objstore(obstrepo, childpath)
                logger.info('%s: removed remote %s (no longer used)', os.path.basename(obstrepo), childpath)
                continue
            valid_virtrefs.add(virtref)

            # Does it need fetching?
            fetch = True
            l_fpf = os.path.join(obstrepo, 'grokmirror.%s.fingerprint' % virtref)
            r_fpf = os.path.join(childpath, 'grokmirror.fingerprint')
            try:
                with open(l_fpf) as fh:
                    l_fp = fh.read().strip()
                with open(r_fpf) as fh:
                    r_fp = fh.read().strip()
                if l_fp == r_fp:
                    fetch = False
            except IOError:
                pass

            gitdir = '/' + os.path.relpath(childpath, toplevel)
            if fetch:
                grokmirror.lock_repo(obstrepo, nonblocking=False)
                logger.info('    fetch: %s -> %s', gitdir, os.path.basename(obstrepo))
                success = grokmirror.fetch_objstore_repo(obstrepo, childpath, use_plumbing=objstore_uses_plumbing)
                if not success and objstore_uses_plumbing:
                    # Try using git porcelain
                    grokmirror.fetch_objstore_repo(obstrepo, childpath)
                grokmirror.unlock_repo(obstrepo)

            if gitdir not in manifest:
                continue

            # Do we need to set any alternateRefsPrefixes?
            for baseline in baselines:
                # Does this repo match a baseline
                if fnmatch.fnmatch(gitdir, baseline):
                    baseline_refs.add('refs/virtual/%s/heads/' % virtref)
                    break

            # Do we need to set islandCore?
            if not set_islandcore:
                is_islandcore = False
                for islandcore in islandcores:
                    # Does this repo match a baseline
                    if fnmatch.fnmatch(gitdir, islandcore):
                        is_islandcore = True
                        break
                if is_islandcore:
                    set_islandcore = True
                    # is it already set to that?
                    entries = grokmirror.get_config_from_git(obstrepo, r'pack\.island*')
                    if entries.get('islandcore') != virtref:
                        new_islandcore = True
                        logger.info(' reconfig: %s (islandCore to %s)', os.path.basename(obstrepo), virtref)
                        grokmirror.set_git_config(obstrepo, 'pack.islandCore', virtref)

            if refrepo is None:
                # Legacy "reference=" setting in manifest
                refrepo = gitdir
                manifest[gitdir]['reference'] = None
            else:
                manifest[gitdir]['reference'] = refrepo

            manifest[gitdir]['forkgroup'] = os.path.basename(obstrepo[:-4])

        if len(baseline_refs):
            # sort the list, so we have deterministic value
            br = list(baseline_refs)
            br.sort()
            refpref = ' '.join(br)
            # Go through all remotes and set their alternateRefsPrefixes
            for s_virtref, s_childpath in my_remotes:
                # is it already set to that?
                entries = grokmirror.get_config_from_git(s_childpath, r'core\.alternate*')
                if entries.get('alternaterefsprefixes') != refpref:
                    s_gitdir = '/' + os.path.relpath(s_childpath, toplevel)
                    logger.info(' reconfig: %s (baseline)', s_gitdir)
                    grokmirror.set_git_config(s_childpath, 'core.alternateRefsPrefixes', refpref)
        repack_requested = False
        if os.path.exists(os.path.join(obstrepo, 'grokmirror.repack')):
            repack_requested = True

        # Go through all our refs and find all stale virtrefs
        args = ['for-each-ref', '--format=%(refname)', 'refs/virtual/']
        trimmed_virtrefs = set()
        ecode, out, err = grokmirror.run_git_command(obstrepo, args)
        if ecode == 0 and out:
            for line in out.split('\n'):
                chunks = line.split('/')
                if len(chunks) < 3:
                    # Where did this come from?
                    logger.debug('Weird ref %s in objstore repo %s', line, obstrepo)
                    continue
                virtref = chunks[2]
                if virtref not in valid_virtrefs and virtref not in trimmed_virtrefs:
                    logger.info('     trim: stale virtref %s', virtref)
                    grokmirror.objstore_trim_virtref(obstrepo, virtref)
                    trimmed_virtrefs.add(virtref)

        if obstrepo not in status or new_islandcore or trimmed_virtrefs or repack_requested:
            # We don't use obstrepo fingerprints, so we set it to None
            status[obstrepo] = {
                'lastcheck': 'never',
                'nextcheck': todayiso,
                'fingerprint': None,
            }
            # Always full-repack brand new obstrepos
            repack_level = 2
        else:
            obj_info = grokmirror.get_repo_obj_info(obstrepo)
            repack_level = grokmirror.get_repack_level(obj_info)

        nextcheck = datetime.datetime.strptime(status[obstrepo]['nextcheck'], '%Y-%m-%d')
        if repack_level > 1 and nextcheck > today:
            # Don't do full repacks outside of schedule
            repack_level = 1

        if repack_level:
            queued += 1
            to_process.add((obstrepo, 'repack', repack_level))
            if repack_level > 1:
                logger.info('   queued: %s (full repack)', os.path.basename(obstrepo))
            else:
                logger.info('   queued: %s (repack)', os.path.basename(obstrepo))
        elif repack_only or repack_all_quick or repack_all_full:
            continue
        elif (nextcheck <= today or force) and not repack_only:
            queued += 1
            status[obstrepo]['nextcheck'] = nextcheck.strftime('%F')
            to_process.add((obstrepo, 'fsck', None))
            logger.info('   queued: %s (fsck)', os.path.basename(obstrepo))

    logger.info('     done: %s analyzed, %s queued', analyzed, queued)

    if obst_changes:
        # We keep the same mtime, because the repos themselves haven't changed
        grokmirror.manifest_lock(manifile)
        # Re-read manifest, so we can update reference and forkgroup data
        disk_manifest = grokmirror.read_manifest(manifile)
        # Go through my manifest and update and changes in forkgroup data
        for gitdir in manifest:
            if gitdir not in disk_manifest:
                # What happened here?
                continue
            if 'reference' in manifest[gitdir]:
                disk_manifest[gitdir]['reference'] = manifest[gitdir]['reference']
            if 'forkgroup' in manifest[gitdir]:
                disk_manifest[gitdir]['forkgroup'] = manifest[gitdir]['forkgroup']

        grokmirror.write_manifest(manifile, disk_manifest, pretty=pretty)
        grokmirror.manifest_unlock(manifile)

    if not len(to_process):
        logger.info('No repos need attention.')
        return

    # Delete some vars that are huge for large repo sets -- we no longer need them and the
    # next step will likely eat lots of ram.
    del obst_roots
    del top_roots
    gc.collect()

    logger.info('Processing %s repositories', len(to_process))

    for fullpath, action, repack_level in to_process:
        logger.info('%s:', fullpath)
        start_size = get_repo_size(fullpath)
        checkdelay = frequency if not force else random.randint(1, frequency)
        nextcheck = today + datetime.timedelta(days=checkdelay)

        # Calculate elapsed seconds
        startt = time.time()

        # Wait till the repo is available and lock it for the duration of checks,
        # otherwise there may be false-positives if a mirrored repo is updated
        # in the middle of fsck or repack.
        grokmirror.lock_repo(fullpath, nonblocking=False)
        if action == 'repack':
            if run_git_repack(fullpath, config, repack_level):
                status[fullpath]['lastrepack'] = todayiso
                if repack_level > 1:
                    try:
                        os.unlink(os.path.join(fullpath, 'grokmirror.repack'))
                    except FileNotFoundError:
                        pass

                    status[fullpath]['lastfullrepack'] = todayiso
                    status[fullpath]['lastcheck'] = todayiso
                    status[fullpath]['nextcheck'] = nextcheck.strftime('%F')
                    # Do we need to generate a preload bundle?
                    if config['fsck'].get('preload_bundle_outdir') and grokmirror.is_obstrepo(fullpath, obstdir):
                        gen_preload_bundle(fullpath, config)
                    logger.info('     next: %s', status[fullpath]['nextcheck'])
            else:
                logger.warning('Repacking %s was unsuccessful', fullpath)
                grokmirror.unlock_repo(fullpath)
                continue

        elif action == 'fsck':
            run_git_fsck(fullpath, config, conn_only)
            status[fullpath]['lastcheck'] = todayiso
            status[fullpath]['nextcheck'] = nextcheck.strftime('%F')
            logger.info('     next: %s', status[fullpath]['nextcheck'])

        gitdir = '/' + os.path.relpath(fullpath, toplevel)
        status[fullpath]['fingerprint'] = grokmirror.get_repo_fingerprint(toplevel, gitdir)

        # noinspection PyTypeChecker
        elapsed = int(time.time()-startt)
        status[fullpath]['s_elapsed'] = elapsed

        # We're done with the repo now
        grokmirror.unlock_repo(fullpath)
        total_checked += 1
        total_elapsed += elapsed
        saved = start_size - get_repo_size(fullpath)
        space_saved += saved
        if saved > 0:
            logger.info('     done: %ss, %s saved', elapsed, get_human_size(saved))
        else:
            logger.info('     done: %ss', elapsed)
        if space_saved > 0:
            logger.info('      ---: %s done, %s queued, %s saved', total_checked,
                        len(to_process)-total_checked, get_human_size(space_saved))
        else:
            logger.info('      ---: %s done, %s queued', total_checked, len(to_process)-total_checked)

        # Write status file after each check, so if the process dies, we won't
        # have to recheck all the repos we've already checked
        logger.debug('Updating status file in %s', statusfile)
        with open(statusfile, 'w') as stfh:
            stfh.write(json.dumps(status, indent=2))

    logger.info('Processed %s repos in %0.2fs', total_checked, total_elapsed)

    with open(statusfile, 'w') as stfh:
        stfh.write(json.dumps(status, indent=2))

    lockf(flockh, LOCK_UN)
    flockh.close()


def parse_args():
    import argparse
    # noinspection PyTypeChecker
    op = argparse.ArgumentParser(prog='grok-fsck',
                                 description='Optimize and check mirrored repositories',
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    op.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                    default=False,
                    help='Be verbose and tell us what you are doing')
    op.add_argument('-f', '--force', dest='force',
                    action='store_true', default=False,
                    help='Force immediate run on all repositories')
    op.add_argument('-c', '--config', dest='config',
                    required=True,
                    help='Location of the configuration file')
    op.add_argument('--repack-only', dest='repack_only',
                    action='store_true', default=False,
                    help='Only find and repack repositories that need optimizing')
    op.add_argument('--connectivity-only', dest='conn_only',
                    action='store_true', default=False,
                    help='Only check connectivity when running fsck checks')
    op.add_argument('--repack-all-quick', dest='repack_all_quick',
                    action='store_true', default=False,
                    help='(Assumes --force): Do a quick repack of all repos')
    op.add_argument('--repack-all-full', dest='repack_all_full',
                    action='store_true', default=False,
                    help='(Assumes --force): Do a full repack of all repos')
    op.add_argument('--version', action='version', version=grokmirror.VERSION)

    opts = op.parse_args()

    if opts.repack_all_quick and opts.repack_all_full:
        op.error('Pick either --repack-all-full or --repack-all-quick')

    return opts


def grok_fsck(cfgfile, verbose=False, force=False, repack_only=False, conn_only=False,
              repack_all_quick=False, repack_all_full=False):
    global logger

    config = grokmirror.load_config_file(cfgfile)

    obstdir = config['core'].get('objstore', None)
    if obstdir is None:
        obstdir = os.path.join(config['core'].get('toplevel'), 'objstore')
        config['core']['objstore'] = obstdir

    logfile = config['core'].get('log', None)
    if config['core'].get('loglevel', 'info') == 'debug':
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO

    logger = grokmirror.init_logger('fsck', logfile, loglevel, verbose)

    rh = io.StringIO()
    ch = logging.StreamHandler(stream=rh)
    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    ch.setLevel(logging.CRITICAL)
    logger.addHandler(ch)

    fsck_mirror(config, force, repack_only, conn_only, repack_all_quick, repack_all_full)

    report = rh.getvalue()
    if len(report):
        msg = EmailMessage()
        msg.set_content(report)
        subject = config['fsck'].get('report_subject')
        if not subject:
            import platform
            subject = 'grok-fsck errors on {} ({})'.format(platform.node(), cfgfile)
        msg['Subject'] = subject
        from_addr = config['fsck'].get('report_from', 'root')
        msg['From'] = from_addr
        report_to = config['fsck'].get('report_to', 'root')
        msg['To'] = report_to
        mailhost = config['fsck'].get('report_mailhost', 'localhost')
        s = smtplib.SMTP(mailhost)
        s.send_message(msg)
        s.quit()


def command():
    opts = parse_args()

    return grok_fsck(opts.config, opts.verbose, opts.force, opts.repack_only, opts.conn_only,
                     opts.repack_all_quick, opts.repack_all_full)


if __name__ == '__main__':
    command()