blob: 58fd3e37f6eb9cb993884f872ea5a29dc98ab086 [file] [log] [blame]
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
__author__ = 'Konstantin Ryabitsev <>'
import os
import sys
import argparse
import logging
import subprocess
from fcntl import lockf, LOCK_EX, LOCK_UN
from typing import Optional, Tuple, Dict
from email.utils import formatdate
from collections import OrderedDict
PI_HEAD = 'refs/meta/origins'
logger = logging.getLogger(__name__)
DEFAULT_NAME = 'PI Origin Maker'
DEFAULT_SUBJ = 'Origin commit'
def git_run_command(gitdir: str, args: list, stdin: Optional[bytes] = None,
env: Optional[Dict] = None) -> Tuple[int, bytes, bytes]:
if not env:
env = dict()
if gitdir:
env['GIT_DIR'] = gitdir
args = ['git', '--no-pager'] + args
logger.debug('Running %s', ' '.join(args))
pp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
(output, error) = pp.communicate(input=stdin)
return pp.returncode, output, error
def check_valid_repo(repo: str) -> None:
# check that it exists and has 'objects' and 'refs'
if not os.path.isdir(repo):
raise FileNotFoundError(f'Path does not exist: {repo}')
musts = {'objects', 'refs'}
for must in musts:
if not os.path.exists(os.path.join(repo, must)):
raise FileNotFoundError(f'Path is not a valid bare git repository: {repo}')
def git_write_commit(repo: str, env: dict, c_msg: str, body: bytes, dest: str = 'i') -> None:
# We use git porcelain commands here. We could use pygit2, but this would pull in a fairly
# large external lib for what is effectively 4 commands that we need to run.
# Lock the repository
# The lock shouldn't be held open for very long, so try without a timeout
lockfh = open(os.path.join(repo, 'ezpi.lock'), 'w')
lockf(lockfh, LOCK_EX)
except IOError:
raise RuntimeError('Could not obtain an exclusive lock')
# Create a blob first
ee, out, err = git_run_command(repo, ['hash-object', '-w', '--stdin'], stdin=body)
if ee > 0:
raise RuntimeError(f'Could not create a blob in {repo}: {err.decode()}')
blob = out.strip(b'\n')
# Create a tree object now
treeline = b'100644 blob ' + blob + b'\t' + dest.encode()
# Now mktree
ee, out, err = git_run_command(repo, ['mktree'], stdin=treeline)
if ee > 0:
raise RuntimeError(f'Could not mktree in {repo}: {err.decode()}')
tree = out.decode().strip()
# Find out if we are the first commit or not
ee, out, err = git_run_command(repo, ['rev-parse', f'{PI_HEAD}^0'])
if ee > 0:
args = ['commit-tree', '-m', c_msg, tree]
args = ['commit-tree', '-p', PI_HEAD, '-m', c_msg, tree]
# Commit the tree
ee, out, err = git_run_command(repo, args, env=env)
if ee > 0:
raise RuntimeError(f'Could not commit-tree in {repo}: {err.decode()}')
# Finally, update the ref
commit = out.decode().strip()
ee, out, err = git_run_command(repo, ['update-ref', PI_HEAD, commit])
if ee > 0:
raise RuntimeError(f'Could not update-ref in {repo}: {err.decode()}')
lockf(lockfh, LOCK_UN)
def read_config(cfgfile):
from configparser import ConfigParser
if not os.path.exists(cfgfile):
sys.stderr.write('ERROR: config file %s does not exist' % cfgfile)
# We don't support duplicates right now
piconfig = ConfigParser(strict=False)
return piconfig
def get_pi_repos(mainrepo: str) -> Dict:
res = dict()
at = 0
latest_origins = None
latest_repo = None
while True:
repo = os.path.join(mainrepo, 'git', '%d.git' % at)
logger.debug('Checking current origins in %s', repo)
ec, out, err = git_run_command(repo, ['show', f'{PI_HEAD}:i'])
# If it's blank, then we force it to be written
if not len(out):
res[repo] = ''
latest_repo = repo
latest_origins = out
at += 1
except FileNotFoundError:
if latest_origins is None or latest_repo is None:
logger.debug('Did not find any valid pi repos in %s', mainrepo)
return res
res[latest_repo] = latest_origins.decode()
return res
def make_origins(config, cmdargs):
for section in config.sections():
# We want named sections
if section.find(' ') < 0:
origin = OrderedDict()
origin['infourl'] = cmdargs.infourl
origin['contact'] =
mainrepo = config[section].get('mainrepo')
if not mainrepo:
mainrepo = config[section].get('inboxdir')
if not mainrepo or not os.path.isdir(mainrepo):'%s: mainrepo=%s does not exist', section, mainrepo)
mainrepo = mainrepo.rstrip('/')
if cmdargs.repotop and os.path.dirname(mainrepo) != cmdargs.repotop:'Skipped %s: not directly in %s', mainrepo, cmdargs.repotop)
pirepos = get_pi_repos(mainrepo)
if not len(pirepos):'%s contains no public-inbox repos', mainrepo)
origin['address'] = config[section].get('address')
origin['listid'] = config[section].get('listid')
if not origin['listid']:
origin['listid'] = origin['address'].replace('@', '.')
origin['newsgroup'] = config[section].get('newsgroup')
if not origin['newsgroup']:
origin['newsgroup'] = '.'.join(reversed(origin['listid'].split('.')))
odata_new = '[publicinbox]\n'
for opt, val in origin.items():
odata_new += f'{opt}={val}\n'
for pirepo, odata_old in pirepos.items():
if odata_new != odata_old:
logger.debug('Setting new origins for %s', pirepo)
env = {
'GIT_AUTHOR_DATE': formatdate(),
'GIT_COMMITTER_DATE': formatdate(),
git_write_commit(pirepo, env, DEFAULT_SUBJ, odata_new.encode())'Updated origins for %s', pirepo)
except RuntimeError as ex:'Could not update origins in %s: %s', pirepo, ex)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--pi-config-file', dest='cfgfile',
help='Public-Inbox config file to use')
parser.add_argument('-i', '--infourl', dest='infourl',
help='infourl value')
parser.add_argument('-e', '--contact-email', dest='contact',
default='postmaster <>',
help='contact value')
parser.add_argument('-t', '--repo-top', dest='repotop',
help='Only work on repos in this topdir')
parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', default=False,
help='Quiet operation (cron mode)')
parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=False,
help='Output debug information')
parser.add_argument('-l', '--logfile', dest='logfile',
help='Record activity in this log file')
_cmdargs = parser.parse_args()
_config = read_config(_cmdargs.cfgfile)
if _cmdargs.logfile:
ch = logging.FileHandler(_cmdargs.logfile)
formatter = logging.Formatter(f'[%(asctime)s] %(message)s')
ch = logging.StreamHandler()
formatter = logging.Formatter('%(message)s')
if _cmdargs.quiet:
elif _cmdargs.debug:
make_origins(_config, _cmdargs)