Initial commit
This is the initial commit of "minimal viable product" as startups like
to say. Documentation is going to be forthcoming -- for now we're
running initial burn-in tests and finding the largest bugs.
Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..5e5f255
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,14 @@
+*.swp
+*.pyc
+*.pyo
+*.json
+*.pdf
+test.log
+build/*
+dist/*
+MANIFEST
+.idea
+__pycache__
+*.egg-info
+*.sqlite3
+.venv
diff --git a/default.config.toml b/default.config.toml
new file mode 100644
index 0000000..a490f19
--- /dev/null
+++ b/default.config.toml
@@ -0,0 +1,80 @@
+[db]
+dburl = 'sqlite:////home/user/work/temp/peebz.sqlite3'
+
+[notify]
+neverto = ['*@kernel-bugs.*', 'bugbot@kernel.org']
+never_if_creator = ['bugbot@kernel.org']
+never_if_text_matches = ['*has been marked as a duplicate of*']
+fromaddr = 'Kernel.org Bugbot <bugbot@kernel.org>'
+smtpserver = 'localhost'
+smtpport = 25
+
+[bugzilla]
+apikey = '<omitted>'
+name = 'Kernel.org Bugzilla'
+url = 'https://bugzilla.kernel.org'
+resturl = 'https://bugzilla.kernel.org/rest'
+bugmask = 'https://bugzilla.kernel.org/show_bug.cgi?id={bug_id}'
+commentmask = 'https://bugzilla.kernel.org/show_bug.cgi?id={bug_id}#c{comment_count}'
+attachmask = 'https://bugzilla.kernel.org/attachment.cgi?id={attachment_id}'
+linkmask = 'https://msgid.link/{msgid}'
+subsystem_cf = 'cf_subsystem'
+maintainers_url = 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/MAINTAINERS'
+
+[mimetypes]
+deny = [ 'text/html', 'application/*-signature' ]
+
+[logging]
+logfile = 'peebz.log'
+loglevel = 'info'
+
+[components.'Linux'.'Kernel']
+new_bug_send_notification = true
+pi_query = '(nq:"bugbot on" OR nq:"bugbot assign")'
+pi_must_bz_groups = ['editbugs']
+pi_url = 'https://lore.kernel.org/all/'
+pi_assign_regex = '^bugbot assign to (\S+)'
+bz_new_bugs_quicksearch = 'OPEN flag:bugbot+'
+alwayscc = ['bugs@lists.linux.dev']
+
+[templates]
+parse_bug_intro = '${author} writes:'
+parse_bug_intro_with_count = '${author} replies to comment #${comment_count}:'
+parse_bug_outro = '(via ${msgid_link})'
+
+new_bug_notify = '''
+Hello:
+
+This conversation is now tracked by ${bzname}:
+${bug_url}
+
+There is no need to do anything else, just keep talking.
+'''
+
+new_comment_notify = '''
+${comment_author} writes via ${bzname}:
+
+${comment_text}
+
+View: ${comment_url}
+You can reply to this message to join the discussion.
+'''
+
+new_attachment_notify = '''
+${comment_author} added a new attachment via ${bzname}.
+You can download it by following the link below.
+
+File: ${file_name} (${content_type})
+Size: ${human_size}
+Link: ${attachment_url}
+---
+${summary}
+
+You can reply to this message to join the discussion.
+'''
+
+botsig = '''
+--
+Deet-doot-dot, I am a bot.
+${myname} (${appname} ${appver})
+'''
diff --git a/peebz.sh b/peebz.sh
new file mode 100755
index 0000000..f3809dc
--- /dev/null
+++ b/peebz.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+#
+# Run from a git checkout.
+#
+
+REAL_SCRIPT=$(realpath -e ${BASH_SOURCE[0]})
+SCRIPT_TOP="${SCRIPT_TOP:-$(dirname ${REAL_SCRIPT})}"
+
+exec env PYTHONPATH="${SCRIPT_TOP}" python3 "${SCRIPT_TOP}/peebz/command.py" "${@}"
diff --git a/peebz/__init__.py b/peebz/__init__.py
new file mode 100644
index 0000000..f2d2024
--- /dev/null
+++ b/peebz/__init__.py
@@ -0,0 +1,904 @@
+#!/usr/bin/env python3
+import email
+import email.message
+import email.utils
+import email.parser
+import requests
+import logging
+import base64
+import b4
+import sys
+import uuid
+import datetime
+
+import urllib.parse
+
+import sqlalchemy as sa
+
+from typing import Dict, Tuple, List, Set, Optional
+from fnmatch import fnmatch
+from string import Template
+
+from sqlalchemy.exc import NoSuchTableError
+
+# Policy we use for saving mail locally
+emlpolicy = email.policy.EmailPolicy(utf8=True, cte_type='8bit', max_line_length=None,
+ message_factory=email.message.EmailMessage)
+
+REQSESSION = None
+CONFIG = dict()
+
+__APPNAME__ = 'peebz'
+__VERSION__ = '0.1'
+
+__DBSCHEMA__ = 1
+
+logger = logging.getLogger(__APPNAME__)
+b4.logger = logger
+
+SACONN = None
+SAENGINE = None
+REST_CACHE = dict()
+MAINT_CACHE = dict()
+
+
+def get_msg_from_stdin() -> Optional[email.message.EmailMessage]:
+ if not sys.stdin.isatty():
+ msg = email.parser.BytesParser(policy=emlpolicy).parsebytes(sys.stdin.buffer.read())
+ return msg # noqa
+ return None
+
+
+def get_requests_session():
+ global REQSESSION
+ if REQSESSION is None:
+ REQSESSION = requests.session()
+ REQSESSION.headers.update({'User-Agent': f'{__APPNAME__}/{__VERSION__}'})
+ return REQSESSION
+
+
+def get_config() -> Dict:
+ return CONFIG
+
+
+def get_component_config(product: str, component: str) -> Dict:
+ config = get_config()
+ return config['components'][product][component]
+
+
+def get_template_by_bid(tptname: str, bid: int) -> Template:
+ product, component = bz_get_product_component_by_bid(bid)
+ return get_template_by_product_component(tptname, product, component)
+
+
+def get_template_by_product_component(tptname: str, product: str, component: str) -> Template:
+ config = get_config()
+ try:
+ return Template(config['components'][product][component]['templates'][tptname])
+ except KeyError:
+ pass
+ return Template(config['templates'][tptname])
+
+
+def get_msgid_link(msgid: str) -> str:
+ config = get_config()
+ linkmask = config['bugzilla'].get('linkmask', 'https://lore.kernel.org/{msgid}')
+ return linkmask.format(msgid=msgid)
+
+
+def bz_rest(path: str, payload: Dict = None, params: Dict = None, method: str = 'GET') -> Dict:
+ global REST_CACHE
+ # We only cache GETs without any params
+ if method == 'GET' and not params and path in REST_CACHE:
+ logger.debug('Using cached data')
+ return REST_CACHE[path]
+
+ config = get_config()
+ url = '{BZURL}/{path}'.format(BZURL=config['bugzilla'].get('resturl').rstrip('/'), path=path)
+ logger.debug('Querying url=%s', url)
+ myparams = dict() if params is None else dict(params)
+ myparams['api_key'] = config['bugzilla'].get('apikey', '')
+ ses = get_requests_session()
+ if method == 'GET':
+ res = ses.get(url, params=myparams)
+ elif method == 'POST':
+ res = ses.post(url, params=myparams, json=payload)
+ elif method == 'PUT':
+ res = ses.put(url, params=myparams, json=payload)
+ else:
+ logger.critical('CRITICAL: Unknown method=%s', method)
+ raise RuntimeError('Unknown method %s' % method)
+
+ # logger.debug('res=%s', res.text)
+ if res.status_code == 404:
+ raise LookupError('Bugzilla returned 404: %s' % res.text)
+
+ # for every other error, we just BT for now
+ res.raise_for_status()
+ rdata = res.json()
+ if method == 'GET' and not params:
+ # Cache it
+ REST_CACHE[path] = rdata
+ elif method != 'GET':
+ # We changed something, so nuke our cache
+ logger.debug('Clearing cache')
+ REST_CACHE = dict()
+
+ return rdata
+
+
+def bz_add_atts_to_bug(bid: int, atts: List[Dict]) -> List[int]:
+ aids = list()
+ for att in atts:
+ att['ids'] = [bid]
+ rdata = bz_rest(f'bug/{bid}/attachment', att, method='POST')
+ logger.debug('Created new attachment %s', ', '.join(rdata['ids']))
+ aids += rdata['ids']
+ return aids
+
+
+def bz_add_new_bug(payload: Dict) -> Tuple[int, int]:
+ if 'version' not in payload:
+ payload['version'] = 'unspecified'
+ rdata = bz_rest('bug', payload, method='POST')
+ bid = rdata['id']
+ logger.debug('Created new bug %s', bid)
+ # Apparently, we don't get comment-id info from new bug creation
+ cid = bz_get_cid_by_bid_count(bid)
+ return bid, cid
+
+
+def bz_add_new_comment(bid: int, comment: str) -> int:
+ payload = {
+ 'id': bid,
+ 'comment': comment
+ }
+ rdata = bz_rest(f'bug/{bid}/comment', payload, method='POST')
+ cid = rdata['id']
+ logger.debug('Created new comment %s', cid)
+ return cid
+
+
+def bz_get_bug(bid: int, resolve_dupes=False) -> Dict:
+ if resolve_dupes:
+ bid = bz_dedupe_bid(bid)
+
+ path = f'bug/{bid}'
+ rdata = bz_rest(path)
+ for bdata in rdata['bugs']:
+ if bdata['id'] == bid:
+ return bdata
+ raise RuntimeError('Could not get bug info for %s' % bid)
+
+
+def bz_get_user(username: str) -> Dict:
+ path = f'user/{username}'
+ try:
+ rdata = bz_rest(path)
+ for udata in rdata['users']:
+ if udata['name'] == username:
+ return udata
+ except LookupError:
+ pass
+
+ raise LookupError('Could not get user info for %s' % username)
+
+
+def bz_get_user_groups(username: str) -> Set:
+ udata = bz_get_user(username)
+ return set([x['name'] for x in udata['groups']])
+
+
+def bz_get_cid_by_bid_count(bid: int, count: int = 0) -> int:
+ bdata = bz_rest(f'bug/{bid}/comment')
+ for rbid, rbdata in bdata['bugs'].items():
+ if int(rbid) != bid:
+ continue
+ for comment in rbdata['comments']:
+ if comment['count'] == count:
+ logger.debug('cid for %s/c%s is %s', bid, count, comment['id'])
+ return comment['id']
+ raise LookupError('No cid matching bid=%s count=%s' % (bid, count))
+
+
+def bz_get_count_by_bid_cid(bid: int, cid: int) -> int:
+ bdata = bz_rest(f'bug/{bid}/comment')
+ for rbid, rbdata in bdata['bugs'].items():
+ if int(rbid) != bid:
+ continue
+ for comment in rbdata['comments']:
+ if comment['id'] == cid:
+ logger.debug('count for %s/%s is c%s', bid, cid, comment['count'])
+ return comment['count']
+ raise LookupError('No match for bid=%s cid=%s', bid, cid)
+
+
+def bz_dedupe_bid(bid: int) -> int:
+ bdata = bz_get_bug(bid)
+ if bdata.get('dupe_of'):
+ # Nothing wrong with recursion
+ return bz_dedupe_bid(bdata['dupe_of'])
+ return bid
+
+
+def bz_check_user_allowed(uid: str, product: str, component: str) -> bool:
+ cconf = get_component_config(product, component)
+ mustgroups = cconf.get('pi_must_bz_groups')
+ if mustgroups is None:
+ # No restrictions, anyone can do anything they like
+ return True
+ try:
+ udata = bz_get_user(uid)
+ except LookupError:
+ logger.debug('Could not find user %s in bugzilla', uid)
+ return False
+
+ for mustgroup in mustgroups:
+ for ugroup in udata.get('groups', list()):
+ if ugroup['name'] == mustgroup:
+ logger.debug('%s mustgroup matches %s', uid, mustgroup)
+ return True
+
+ logger.debug('%s not member of %s', uid, mustgroups)
+ return False
+
+
+def bz_assign_bug(bid: int, uid: str) -> None:
+ logger.info('Assigning bug %s to %s', bid, uid)
+ path = f'bug/{bid}'
+ payload = {
+ 'assigned_to': uid,
+ }
+ bz_rest(path, payload=payload, method='PUT')
+
+
+def db_get_query_last_check(product: str, component: str) -> str:
+ key = f'query_{product}_{component}'
+ return db_get_meta_value(key)
+
+
+def db_store_query_last_check(product: str, component: str, last_check: str) -> None:
+ key = f'query_{product}_{component}'
+ return db_store_meta_value(key, last_check)
+
+
+def db_get_msgid_by_bid_cid(bid: int, cid: Optional[int]) -> str:
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_bmap = sa.Table('msgid_bug_mapping', md, autoload=True, autoload_with=engine)
+ if cid:
+ q = sa.select([t_bmap.c.message_id]).where(t_bmap.c.bug_id == bid, t_bmap.c.comment_id == cid)
+ else:
+ # If cid is not defined, we get all mappings and use the lowest cid
+ q = sa.select([t_bmap.c.message_id]).where(t_bmap.c.bug_id == bid).order_by(t_bmap.c.comment_id)
+
+ rp = dbconn.execute(q)
+ fa = rp.fetchall()
+ if len(fa):
+ logger.debug('query results for bid=%s, cid=%s: %s', bid, cid, fa)
+ return fa[0][0]
+
+ raise LookupError('No message-id matching bid=%s, cid=%s' % (bid, cid))
+
+
+def db_get_bid_cid_by_msgid(msgid: str) -> Tuple[int, Optional[int]]:
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_bmap = sa.Table('msgid_bug_mapping', md, autoload=True, autoload_with=engine)
+ logger.debug('Querying db for msgid=%s', msgid)
+ q = sa.select([t_bmap.c.bug_id, t_bmap.c.comment_id]).where(t_bmap.c.message_id == msgid)
+ rp = dbconn.execute(q)
+ fa = rp.fetchall()
+ if not len(fa):
+ raise LookupError('msgid %s not known' % msgid)
+ bid, cid = fa[0]
+ dbid = bz_dedupe_bid(bid)
+ if dbid != bid:
+ cid = None
+ logger.debug(' matching bid=%s, cid=%s', bid, cid)
+ return bid, cid
+
+
+def db_store_msgid_bid_cid(msgid: str, bid: int, cid: int) -> None:
+ msgid = msgid.strip('<>')
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_bmap = sa.Table('msgid_bug_mapping', md, autoload=True, autoload_with=engine)
+ q = sa.insert(t_bmap).values(message_id=msgid, bug_id=bid, comment_id=cid)
+ dbconn.execute(q)
+ logger.info('Created new mapping for %s: %s/%s', msgid, bid, cid)
+
+
+def db_get_recipients(bid: int) -> Set[str]:
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_recip = sa.Table('recipients', md, autoload=True, autoload_with=engine)
+ logger.debug('Querying recipients for bid=%s', bid)
+ q = sa.select([t_recip.c.email]).where(t_recip.c.bug_id == bid)
+ rp = dbconn.execute(q)
+ fa = rp.fetchall()
+ if not len(fa):
+ raise LookupError('bid %s not known' % bid)
+ return set(x[0] for x in fa)
+
+
+def db_store_recipients(bid: int, recipients: Set[str]) -> None:
+ # TODO: add ability to unsubscribe?
+ try:
+ stored = db_get_recipients(bid)
+ except LookupError:
+ stored = set()
+ # Any new ones to store?
+ extras = recipients - stored
+ if not extras:
+ return
+
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_recip = sa.Table('recipients', md, autoload=True, autoload_with=engine)
+ logger.debug('Storing new recipients for bid=%s', bid)
+ for addr in extras:
+ q = sa.insert(t_recip).values(bug_id=bid, email=addr)
+ dbconn.execute(q)
+ logger.debug(' Added %s', addr)
+ return
+
+
+def db_get_meta_value(key: str) -> str:
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_meta = sa.Table('meta', md, autoload=True, autoload_with=engine)
+ q = sa.select([t_meta.c.var_value]).where(t_meta.c.var_key == key)
+ rp = dbconn.execute(q)
+ fa = rp.fetchall()
+ if not len(fa):
+ raise LookupError('meta key %s not known' % key)
+ return fa[0][0]
+
+
+def db_store_meta_value(key: str, value: str) -> None:
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_meta = sa.Table('meta', md, autoload=True, autoload_with=engine)
+ with engine.begin() as dbconn:
+ q = sa.delete(t_meta).where(t_meta.c.var_key == key)
+ dbconn.execute(q)
+ q = sa.insert(t_meta).values(var_key=key, var_value=value)
+ dbconn.execute(q)
+
+
+def db_store_notify_last_check(bid: int, when: str):
+ key = f'notify_bug_{bid}'
+ return db_store_meta_value(key, when)
+
+
+def db_get_notify_last_check(bid: int) -> str:
+ key = f'notify_bug_{bid}'
+ return db_get_meta_value(key)
+
+
+def bz_get_changed_bugs(since: str, include_untracked: bool = False) -> List:
+ logger.debug('Querying for changed bugs since %s', since)
+ params = {
+ 'chfieldfrom': since,
+ 'include_fields': 'id,summary',
+ }
+ rdata = bz_rest('bug', params=params)
+ if include_untracked:
+ return rdata['bugs']
+ bids = tuple([x['id'] for x in rdata['bugs']])
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_bmap = sa.Table('msgid_bug_mapping', md, autoload=True, autoload_with=engine)
+ q = sa.select([t_bmap.c.bug_id]).where(t_bmap.c.bug_id.in_(bids)).distinct()
+ rp = dbconn.execute(q)
+ fa = rp.fetchall()
+ if not fa:
+ return list()
+ tracked = set([x[0] for x in fa])
+ bugs = list()
+ for bdata in rdata['bugs']:
+ if bdata['id'] in tracked:
+ bugs.append(bdata)
+
+ return bugs
+
+
+def bz_quicksearch_bugs(query: str) -> Dict:
+ params = {
+ 'include_fields': 'id,summary',
+ 'quicksearch': query,
+ }
+ return bz_rest('bug', params=params)
+
+
+def bz_get_query_bugs(params: Dict, exclude: Set[int]) -> List[int]:
+ if 'include_fields' not in params:
+ params['include_fields'] = 'id,summary'
+ rdata = bz_rest('bug', params=params)
+ bids = list()
+ for bdata in rdata['bugs']:
+ if bdata['id'] in exclude:
+ continue
+ bids.append(bdata['id'])
+ return bids
+
+
+def get_human_size(size: int, decimals: int = 2) -> str:
+ units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']
+ while True:
+ unit = units.pop(0)
+ if size < 1024.0 or not len(units):
+ break
+ size /= 1024.0
+
+ return f'{size:.{decimals}f} {unit}'
+
+
+def bz_get_attachment_by_aid(aid: int, include_fields: Optional[str] = None) -> Dict:
+ if include_fields is None:
+ rdata = bz_rest(f'bug/attachment/{aid}')
+ else:
+ params = {
+ 'include_fields': include_fields,
+ }
+ rdata = bz_rest(f'bug/attachment/{aid}', params=params)
+
+ for raid, radata in rdata['attachments'].items():
+ if int(raid) == aid:
+ if 'size' in radata:
+ radata['human_size'] = get_human_size(radata['size'])
+ return radata
+
+ raise LookupError('No matching attachment_id found: %s', aid)
+
+
+def bz_get_all_comments_by_bid(bid: int) -> List[Dict]:
+ bdata = bz_rest(f'bug/{bid}/comment')
+ for rbid, rbdata in bdata['bugs'].items():
+ if int(rbid) != bid:
+ continue
+ return rbdata['comments']
+
+
+def bz_get_comments_for_bid_since(bid: int, since: str) -> List[Dict]:
+ params = {
+ 'new_since': since,
+ 'id': bid,
+ }
+ bdata = bz_rest(f'bug/{bid}/comment', params=params)
+ for rbid, rbdata in bdata['bugs'].items():
+ if int(rbid) != bid:
+ continue
+ return rbdata['comments']
+
+
+def bz_get_newest_comments_for_bid(bid: int, include_private: bool = False) -> List[Dict]:
+ try:
+ when = db_get_notify_last_check(bid)
+ except LookupError:
+ # grab the highest cid we know about
+ engine, dbconn = db_get_sa()
+ md = sa.MetaData()
+ t_bmap = sa.Table('msgid_bug_mapping', md, autoload=True, autoload_with=engine)
+ q = sa.select(sa.func.max(t_bmap.c.comment_id)).where(t_bmap.c.bug_id == bid)
+ rp = dbconn.execute(q)
+ mfa = rp.fetchall()
+ c_max = None
+ if mfa:
+ c_max = mfa[0][0]
+ cdatas = bz_get_all_comments_by_bid(bid)
+ comments = list()
+ for cdata in cdatas:
+ if c_max and cdata['id'] <= c_max:
+ continue
+ if not include_private and cdata['is_private']:
+ continue
+ comments.append(cdata)
+ return comments
+
+ logger.debug('Getting newest comments since %s', when)
+ cdatas = bz_get_comments_for_bid_since(bid, when)
+ comments = list()
+ for cdata in cdatas:
+ if not include_private and cdata['is_private']:
+ continue
+ comments.append(cdata)
+ return comments
+
+
+def msg_get_inre_msgids(msg: email.message.EmailMessage) -> List[str]:
+ pairs = list()
+ if msg.get('In-Reply-To'):
+ pairs += email.utils.getaddresses([str(x) for x in msg.get_all('in-reply-to', [])])
+ if msg.get('References'):
+ pairs += email.utils.getaddresses([str(x) for x in msg.get_all('references', [])])
+
+ msgids = list()
+ for pair in pairs:
+ if pair[1] not in msgids:
+ msgids.append(pair[1])
+
+ return msgids
+
+
+def sort_msgs_by_received(msgs: List[email.message.EmailMessage]) -> List[email.message.EmailMessage]:
+ tosort = list()
+ for msg in msgs:
+ latest_rdt = None
+ for rhdr in msg.get_all('Received'):
+ # The received headers are pretty free-form, but generally will end with ; datetimeinfo
+ chunks = rhdr.rsplit(';', maxsplit=1)
+ if len(chunks) < 2:
+ continue
+ rdate = chunks[1].strip()
+ rdt = email.utils.parsedate_tz(rdate)
+ if rdt is None:
+ continue
+ # use the latest rdt
+ if not latest_rdt or latest_rdt < rdt:
+ latest_rdt = rdt
+ latest_rdt = None
+ if not latest_rdt and msg.get('Date'):
+ # Use the Date header as fallback
+ latest_rdt = email.utils.parsedate_to_datetime(msg.get('Date'))
+ if not latest_rdt:
+ logger.debug('Message without a date: %s!', msg.get('Message-ID'))
+ continue
+ tosort.append((latest_rdt, msg))
+
+ sortedmsgs = list()
+ for rdt, msg in sorted(tosort):
+ sortedmsgs.append(msg)
+
+ return sortedmsgs
+
+
+def msg_get_inre_bid_cid(msg: email.message.EmailMessage) -> Tuple[int, int]:
+ msgids = msg_get_inre_msgids(msg)
+ if not msgids:
+ logger.debug('No references in msg')
+ raise LookupError('No references found')
+
+ res = list()
+ for msgid in msgids:
+ try:
+ res.append(db_get_bid_cid_by_msgid(msgid))
+ except LookupError:
+ # Try the next one until we run out
+ pass
+ if not res:
+ logger.debug('No reference matched database records')
+ raise LookupError('Nothing matching in the db')
+
+ if len(res) > 1:
+ # Use the highest cid, which is going to be the latest matching comment
+ res.sort(key=lambda x: x[1], reverse=True)
+ return res[0]
+
+
+def msg_get_recipients(msg: email.message.EmailMessage) -> Set[str]:
+ pairs = list()
+ if msg.get('To'):
+ pairs += email.utils.getaddresses([str(x) for x in msg.get_all('to', [])])
+ if msg.get('Cc'):
+ pairs += email.utils.getaddresses([str(x) for x in msg.get_all('cc', [])])
+ if msg.get('From'):
+ pairs += email.utils.getaddresses([str(x) for x in msg.get_all('from', [])])
+
+ return set([x[1].lower() for x in pairs])
+
+
+def bz_get_product_component_by_bid(bid: int) -> Tuple[str, str]:
+ bdata = bz_get_bug(bid, resolve_dupes=True)
+ return bdata['product'], bdata['component']
+
+
+def get_product_component_by_recipients(recipients: Set[str]) -> Tuple[str, str]:
+ config = get_config()
+ if not config.get('components'):
+ logger.debug('No components found in config')
+ raise LookupError('No components defined in config')
+ for recipient in recipients:
+ logger.debug('Matching %s', recipient)
+ for bz_product, bz_components in config['components'].items():
+ for bz_component, c_config in bz_components.items():
+ if not c_config.get('recipients'):
+ continue
+ for addr in c_config['recipients']:
+ if fnmatch(recipient, addr):
+ logger.debug('Matched %s with product=%s, component=%s', recipient, bz_product, bz_component)
+ return bz_product, bz_component
+
+ raise LookupError('No matches for any recipients')
+
+
+def get_newbug_payload_by_product_component(product: str, component: str) -> Dict:
+ config = get_config()
+ try:
+ payload = config['components'][product][component]['payload']
+ except KeyError:
+ payload = dict()
+ payload['product'] = product
+ payload['component'] = component
+ return payload
+
+
+def msg_get_author(msg: email.message.EmailMessage) -> Tuple[str, str]:
+ author = ('', 'missing@address.local')
+ fh = msg.get('from')
+ if fh:
+ author = email.utils.getaddresses([fh])[0]
+
+ if not author[0]:
+ return 'Zorro Boogs', author[1]
+
+ return author
+
+
+def msg_parse_for_bug(msg: email.message.EmailMessage) -> Tuple[str, Tuple[str, str], str, str, List[Dict]]:
+ msgid = b4.LoreMessage.get_clean_msgid(msg)
+ mp = msg.get_body(preferencelist=('plain',))
+ bbody = mp.get_payload(decode=True)
+ cs = mp.get_content_charset()
+ if not cs:
+ cs = 'utf-8'
+ cpay = bbody.decode(cs, errors='replace')
+ # Strip signature if we find it
+ chunks = cpay.rsplit('-- ', maxsplit=1)
+ cbody = chunks[0]
+ lsub = b4.LoreSubject(msg.get('Subject', ''))
+ subject = lsub.subject
+ atts = msg_get_valid_attachments(msg)
+ author = msg_get_author(msg)
+
+ return msgid, author, subject, cbody, atts
+
+
+def msg_get_valid_attachments(msg: email.message.EmailMessage) -> List[Dict]:
+ # Get all good attachments
+ config = get_config()
+ atts = list()
+ for part in msg.walk():
+ if part.get_content_disposition() != 'attachment':
+ continue
+ ct = part.get_content_type()
+ mts = config.get('mimetypes')
+ allowed = True
+ if mts and 'deny' in mts:
+ for dmt in mts['deny']:
+ if fnmatch(ct, dmt):
+ logger.debug('Skipping denied mime-type attachement: %s', ct)
+ allowed = False
+ break
+ if allowed:
+ databytes = part.get_payload(decode=True)
+ data = base64.b64encode(databytes).decode()
+ filename = part.get_filename()
+ if not filename:
+ filename = 'unnamed.txt'
+ ct = 'text/plain'
+ summary = filename
+ payload = {
+ 'file_name': filename,
+ 'content_type': ct,
+ 'summary': summary,
+ 'data': data,
+ }
+
+ atts.append(payload)
+
+ return atts
+
+
+def get_recipients_by_product_component(product: str, component: str) -> Set[str]:
+ recip = set()
+ config = get_config()
+ try:
+ recip.update(config['notify']['alwayscc'])
+ logger.debug('added global alwayscc: %s', config['notify']['alwayscc'])
+ except KeyError:
+ pass
+ try:
+ recip.update(config['components'][product][component]['alwayscc'])
+ logger.debug('added %s/%s alwayscc: %s', product, component,
+ config['components'][product][component]['alwayscc'])
+ except KeyError:
+ pass
+ return recip
+
+
+def get_recipients_by_subsystem(subsystem: str) -> Set[str]:
+ global MAINT_CACHE
+ if not len(MAINT_CACHE):
+ config = get_config()
+ ses = get_requests_session()
+ murl = config['bugzilla']['maintainers_url']
+ if murl.startswith('file://'):
+ with open(murl.replace('file://', ''), 'r') as fh:
+ mdata = fh.read()
+ else:
+ res = ses.get(config['bugzilla']['maintainers_url'])
+ res.raise_for_status()
+ mdata = res.text
+ lookfor = ['M:', 'L:']
+ prevline = None
+ cur_sub = None
+ for line in mdata.splitlines():
+ if len(line) < 2 or not len(line[0].strip()):
+ cur_sub = None
+ continue
+ if line[:2] not in lookfor:
+ prevline = line
+ continue
+ if not cur_sub:
+ cur_sub = prevline
+ addr = email.utils.parseaddr(line[2:])
+ if cur_sub:
+ if cur_sub not in MAINT_CACHE:
+ MAINT_CACHE[cur_sub] = set()
+ MAINT_CACHE[cur_sub].add(addr[1])
+
+ try:
+ return MAINT_CACHE[subsystem]
+ except KeyError:
+ return set()
+
+
+def get_bug_recipients(bid: int) -> Set[str]:
+ # Get all db-stored recipients
+ # TODO: implement "onlyto"
+ allrecip = set()
+ try:
+ allrecip.update(db_get_recipients(bid))
+ except LookupError:
+ logger.debug('No in-database recipients for bid=%s', bid)
+ # Now get all bug cc recipients
+ bdata = bz_get_bug(bid, resolve_dupes=True)
+ bugr = set(bdata['cc'])
+ bugr.add(bdata['assigned_to'])
+ bugr.add(bdata['creator'])
+ bugr.update(get_recipients_by_product_component(bdata['product'], bdata['component']))
+ allrecip.update(bugr)
+ config = get_config()
+ subsystem_cf = config['bugzilla'].get('subsystem_cf')
+ if subsystem_cf:
+ subr = get_recipients_by_subsystem(bdata[subsystem_cf])
+ allrecip.update(subr)
+
+ # Remove "neverto" addresses
+ for mask in config['notify'].get('neverto', list()):
+ for addr in set(allrecip):
+ if fnmatch(addr, mask):
+ logger.debug('Removed %s because it matched neverto=%s', addr, mask)
+ allrecip.remove(addr)
+ return allrecip
+
+
+def make_msgid(bid: int, cid: Optional[int]) -> str:
+ config = get_config()
+ bzurl = config['bugzilla']['url']
+ bzloc = urllib.parse.urlparse(bzurl)
+ slug = f'b{bid}'
+ if bid and cid:
+ count = bz_get_count_by_bid_cid(bid, cid)
+ slug += f'c{count}'
+
+ msgid = '<%s-%s-%s@%s>' % (datetime.date.today().strftime('%Y%m%d'), slug, uuid.uuid4().hex[:12], bzloc.netloc)
+ return msgid
+
+
+def notify_bug(bid: int, cid: Optional[int], msg: email.message.EmailMessage, inre_cid: Optional[int] = None,
+ dry_run: bool = False) -> str:
+ bdata = bz_get_bug(bid, resolve_dupes=True)
+ config = get_config()
+ if not msg.get('From'):
+ msg['From'] = config['notify'].get('fromaddr')
+ if not msg.get('To'):
+ recipients = get_bug_recipients(bid)
+ msg['To'] = b4.format_addrs([('', x) for x in recipients])
+ if not msg.get('Message-ID'):
+ msg['Message-ID'] = make_msgid(bid, cid)
+ if not msg.get('In-Reply-To'):
+ inre_msgid = None
+ try:
+ inre_msgid = db_get_msgid_by_bid_cid(bid, inre_cid)
+ except LookupError:
+ logger.debug('Could not find msgid matching bid=%s, cid=%s', bid, inre_cid)
+ # Find anything
+ try:
+ inre_msgid = db_get_msgid_by_bid_cid(bid, None)
+ except LookupError:
+ pass
+ if inre_msgid:
+ msg['In-Reply-To'] = f'<{inre_msgid}>'
+ msg['References'] = f'<{inre_msgid}>'
+ if not msg.get('Subject'):
+ msg['Subject'] = 'Re: %s' % bdata['summary']
+ else:
+ msg['Subject'] = bdata['summary']
+
+ msg['X-Bugzilla-Product'] = bdata['product']
+ msg['X-Bugzilla-Component'] = bdata['component']
+ msg['X-Mailer'] = f'{__APPNAME__} {__VERSION__}'
+ # Should we add other X-B headers?
+ # If we have notify.smtpserver and notify.smtpport defined, use that,
+ # otherwise use b4's get_smtp method
+ smtpserver = config['notify'].get('smtpserver')
+ smtpport = int(config['notify'].get('smtpport', '25'))
+ if smtpserver and smtpport:
+ import smtplib
+ smtp = smtplib.SMTP(smtpserver, smtpport)
+ else:
+ smtp, fromaddr = b4.get_smtp(dryrun=dry_run)
+ b4.send_mail(smtp, [msg], fromaddr=config['notify'].get('fromaddr'), dryrun=dry_run)
+
+ return msg.get('Message-ID')
+
+
+def add_bot_signature(body: str) -> str:
+ config = get_config()
+ sigtpt = Template(config['templates'].get('botsig'))
+ sigvals = {
+ 'myname': config['bugzilla'].get('name'),
+ 'appname': __APPNAME__,
+ 'appver': __VERSION__,
+ }
+ body += sigtpt.safe_substitute(sigvals)
+ return body
+
+
+def db_get_sa() -> Tuple[sa.engine.Engine, sa.engine.Connection]:
+ global SACONN, SAENGINE
+ if SACONN is None:
+ config = get_config()
+ try:
+ dburl = config['db']['dburl']
+ except KeyError:
+ raise LookupError('CRITICAL: main.dburl not set in config file')
+
+ db_pool_recycle = int(config['db'].get('dbpoolrecycle', '300'))
+ SAENGINE = sa.create_engine(dburl, pool_recycle=db_pool_recycle)
+ SACONN = SAENGINE.connect()
+ if SAENGINE.driver == 'pysqlite':
+ md = sa.MetaData()
+ try:
+ t_meta = sa.Table('meta', md, autoload=True, autoload_with=SAENGINE)
+ q = sa.select([t_meta.c.var_value]).where(t_meta.c.var_key == 'schema')
+ rp = SACONN.execute(q)
+ dbver = rp.fetchone()[0]
+ # Future logic to upgrade database here
+ logger.debug('dbver=%s', dbver)
+ except NoSuchTableError:
+ db_init_sa_sqlite_db(SAENGINE, SACONN)
+
+ return SAENGINE, SACONN
+
+
+def db_init_sa_sqlite_db(engine: sa.engine.Engine, dbconn: sa.engine.Connection):
+ logger.info('Setting up SQLite database')
+ md = sa.MetaData()
+ meta = sa.Table('meta', md,
+ sa.Column('var_key', sa.Text()),
+ sa.Column('var_value', sa.Text()),
+ )
+ sa.Index('idx_meta_key_value', meta.c.var_key, meta.c.var_value, unique=True)
+ bmap = sa.Table('msgid_bug_mapping', md,
+ sa.Column('row_id', sa.Integer(), primary_key=True),
+ sa.Column('bug_id', sa.Integer(), nullable=False),
+ sa.Column('comment_id', sa.Integer(), nullable=False),
+ sa.Column('message_id', sa.Text(), nullable=False),
+ )
+ sa.Index('idx_msgid_bugid_commentid', bmap.c.message_id, bmap.c.bug_id, bmap.c.comment_id, unique=True)
+ sa.Index('idx_msgid_commentid', bmap.c.message_id, bmap.c.comment_id, unique=True)
+ sa.Index('idx_msgid_bugid', bmap.c.message_id, bmap.c.bug_id, unique=True)
+
+ recip = sa.Table('recipients', md,
+ sa.Column('row_id', sa.Integer(), primary_key=True),
+ sa.Column('bug_id', sa.Integer(), nullable=False),
+ sa.Column('email', sa.Text(), nullable=False),
+ )
+ sa.Index('idx_bugid_email', recip.c.bug_id, recip.c.email, unique=True)
+ md.create_all(engine)
+ q = sa.insert(meta).values(var_key='schema', var_value=str(__DBSCHEMA__))
+ dbconn.execute(q)
diff --git a/peebz/bz2pi.py b/peebz/bz2pi.py
new file mode 100644
index 0000000..b78a648
--- /dev/null
+++ b/peebz/bz2pi.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2023 by the Linux Foundation
+
+import argparse
+import peebz
+import datetime
+import re
+import b4
+
+import email.message
+
+from fnmatch import fnmatch
+
+logger = peebz.logger
+
+
+def process_new_comments(bid: int, dry_run: bool = False):
+ config = peebz.get_config()
+ cdatas = peebz.bz_get_newest_comments_for_bid(bid)
+ for cdata in cdatas:
+ # Check if we've already notified about this bug
+ cid = cdata['id']
+ try:
+ peebz.db_get_msgid_by_bid_cid(bid, cid)
+ logger.debug('Skipping, msgid match for bid=%s, cid=%s', bid, cid)
+ continue
+ except LookupError:
+ pass
+ # Check if the creator is in never_if_creator
+ skip = False
+ for mask in config['notify'].get('never_if_creator', list()):
+ if fnmatch(cdata['creator'], mask):
+ logger.debug('Skipping cid=%s because it matched never_if_creator=%s', cid, mask)
+ skip = True
+ break
+ if skip:
+ continue
+ # Check if text is in never_if_text_matches
+ for mask in config['notify'].get('never_if_text_matches', list()):
+ if fnmatch(cdata['text'], mask):
+ logger.debug('Skipping cid=%s because it matched never_if_text_matches=%s', cid, mask)
+ skip = True
+ break
+ if skip:
+ continue
+ clines = cdata['text'].strip().splitlines()
+ inre_cid = None
+ bodyvals = {
+ 'bzname': config['bugzilla'].get('name'),
+ 'bug_url': config['bugzilla'].get('bugmask', '').format(bug_id=bid),
+ 'comment_url': config['bugzilla'].get('bugmask', '').format(bug_id=bid) + f"#c{cdata['count']}",
+ 'comment_author': cdata['creator'],
+ }
+ if cdata['attachment_id']:
+ logger.info('Processing new attachment for bug_id=%s, comment_id=%s', bid, cid)
+ adata = peebz.bz_get_attachment_by_aid(
+ cdata['attachment_id'],
+ include_fields='file_name,size,content_type,summary,is_patch,is_private',
+ )
+ if adata['is_private']:
+ logger.debug('Skipping attachment marked private')
+ continue
+ bodytpt = peebz.get_template_by_bid('new_attachment_notify', bid)
+ bodyvals.update(adata)
+ bodyvals['attachment_url'] = config['bugzilla'].get('attachmask', '').format(
+ attachment_id=cdata['attachment_id'])
+ else:
+ logger.info('Processing new comment for bug_id=%s, comment_id=%s', bid, cid)
+ fline = clines[0]
+ matches = re.search(r'\(In reply to.*from comment #(\d+)', fline, flags=re.I)
+ if matches:
+ inre_count = int(matches.groups()[0])
+ try:
+ inre_cid = peebz.bz_get_cid_by_bid_count(bid, inre_count)
+ except LookupError:
+ pass
+ bodyvals['comment_text'] = '\n'.join(clines)
+ bodytpt = peebz.get_template_by_bid('new_comment_notify', bid)
+
+ msg = email.message.EmailMessage()
+ msg['Reply-To'] = b4.format_addrs([('', cdata['creator'])])
+ body = bodytpt.safe_substitute(bodyvals)
+ body = peebz.add_bot_signature(body)
+ msg.set_payload(body, charset='utf-8')
+ msgid = peebz.notify_bug(bid, cid, msg, inre_cid=inre_cid, dry_run=dry_run)
+ if msgid and not dry_run:
+ peebz.db_store_msgid_bid_cid(msgid, bid, cid)
+ peebz.db_store_recipients(bid, {cdata['creator']})
+ # TODO: This assumes that comments are always in incremental order
+ lastcheck = cdata['creation_time'].replace('T', ' ').rstrip('Z')
+ peebz.db_store_notify_last_check(bid, lastcheck)
+
+
+def main(cmdargs: argparse.Namespace) -> None:
+ now = datetime.datetime.utcnow()
+ lastrun = now.strftime('%Y-%m-%d %H:%M:%S')
+ try:
+ # Get all new bugs that changed since last run
+ since = peebz.db_get_meta_value('notify_last_run')
+ except LookupError:
+ logger.debug('Got a LookupError, getting everything for the past hour')
+ # Assume it's the first run and get changes for the past hour
+ hourago = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
+ since = hourago.strftime('%Y-%m-%d %H:%M:%S')
+
+ # first, process all changed bugs that we're tracking
+ logger.info('Getting a list of changed bugs since %s', since)
+ buglist = peebz.bz_get_changed_bugs(since)
+ seen = set()
+ if buglist:
+ for bdata in buglist:
+ logger.debug('Looking at %s: %s', bdata['id'], bdata['summary'])
+ bid = bdata['id']
+ seen.add(bid)
+ process_new_comments(bid, dry_run=cmdargs.dry_run)
+ else:
+ logger.info('No changes to any tracked bugs')
+
+ # Now go by product/component and handle new bug queries if defined
+ config = peebz.get_config()
+ for bz_product, bz_components in config['components'].items():
+ for bz_component in bz_components.keys():
+ cconf = peebz.get_component_config(bz_product, bz_component)
+ qs = cconf.get('bz_new_bugs_quicksearch')
+ if not qs:
+ logger.debug('No quicksearch defined for %s/%s', bz_product, bz_component)
+ continue
+ logger.info('Querying matching quicksearch results since %s for %s/%s, qs=%s', since, bz_product,
+ bz_component, qs)
+ params = {
+ 'chfieldfrom': since,
+ 'product': bz_product,
+ 'component': bz_component,
+ 'quicksearch': qs,
+ }
+ buglist = peebz.bz_get_query_bugs(params, exclude=seen)
+ if buglist:
+ logger.info('Processing %s matching quicksearch bugs', len(buglist))
+ for bid in buglist:
+ seen.add(bid)
+ process_new_comments(bid, dry_run=cmdargs.dry_run)
+ else:
+ logger.info('No changed bugs matching these parameters.')
+
+ if not cmdargs.dry_run:
+ peebz.db_store_meta_value(key='notify_last_run', value=lastrun)
diff --git a/peebz/command.py b/peebz/command.py
new file mode 100644
index 0000000..30046bd
--- /dev/null
+++ b/peebz/command.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2020 by the Linux Foundation
+#
+
+import sys
+import argparse
+import b4
+import logging
+import logging.handlers
+import peebz
+
+logger = peebz.logger
+
+
+def cmd_parse(cmdargs):
+ import peebz.parse
+ peebz.parse.main(cmdargs)
+
+
+def cmd_bz2pi(cmdargs):
+ import peebz.bz2pi
+ peebz.bz2pi.main(cmdargs)
+
+
+def cmd_pi2bz(cmdargs):
+ import peebz.pi2bz
+ peebz.pi2bz.main(cmdargs)
+
+
+def cmd_bzdump(cmdargs):
+ import json
+ from pygments import highlight, lexers, formatters
+ if cmdargs.bug_id:
+ rdata = peebz.bz_get_bug(cmdargs.bug_id, resolve_dupes=cmdargs.resolve_dupes)
+ elif cmdargs.username:
+ rdata = peebz.bz_get_user(cmdargs.username)
+ elif cmdargs.quicksearch:
+ rdata = peebz.bz_quicksearch_bugs(query=cmdargs.quicksearch)
+ else:
+ sys.exit(1)
+
+ jdata = json.dumps(rdata, sort_keys=True, indent=4)
+ colorized = highlight(jdata, lexers.JsonLexer(), formatters.TerminalFormatter())
+ print(colorized)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+ # noinspection PyTypeChecker
+ parser = argparse.ArgumentParser(
+ prog='peebz',
+ description='A tool to bridge public-inbox collections with bugzilla',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+ parser.add_argument('--version', action='version', version=peebz.__VERSION__)
+ parser.add_argument('-d', '--debug', action='store_true', default=False,
+ help='Add more debugging info to the output')
+ parser.add_argument('-q', '--quiet', action='store_true', default=False,
+ help='Output critical information only')
+ parser.add_argument('-c', '--config', required=True,
+ help='config.toml to use')
+ parser.add_argument('--dry-run', action='store_true', default=False,
+ help='Dry run, do not make any changes')
+
+ subparsers = parser.add_subparsers(help='sub-command help', dest='subcmd')
+ # parse : process RFC2822 messages passed on stdin (testing mode)
+ sp_parse = subparsers.add_parser('parse', help='Parse messages passed via stdin')
+ sp_parse.add_argument('--product', help='Force this product instead of guessing by recipient')
+ sp_parse.add_argument('--component', help='Force this component instead of guessing by recipient')
+ sp_parse.set_defaults(func=cmd_parse)
+
+ # pi2bz : query public-inbox to find the latest treads that interest us
+ sp_pi2bz = subparsers.add_parser('pi2bz', help='Query public-inbox sources for updates')
+ sp_pi2bz.add_argument('--product', help='Only run queries for this product')
+ sp_pi2bz.add_argument('--component', help='Only run queries for this component')
+ sp_pi2bz.set_defaults(func=cmd_pi2bz)
+
+ # bz2pi: query bugzilla and sends out any mail updates
+ sp_bz2pi = subparsers.add_parser('bz2pi', help='Send emails about bugzilla-originated changes')
+ sp_bz2pi.set_defaults(func=cmd_bz2pi)
+
+ # show : command to show REST raw REST output
+ sp_bzdump = subparsers.add_parser('bzdump', help='Show colorized raw REST output from bugzilla API')
+ sp_bzdump.add_argument('-b', '--bug-id', type=int, help='Bug to show')
+ sp_bzdump.add_argument('-c', '--comment-id', help='Comment to show')
+ sp_bzdump.add_argument('-a', '--attachment-id', help='Attachment to show')
+ sp_bzdump.add_argument('-u', '--username', help='User to show')
+ sp_bzdump.add_argument('-q', '--quicksearch', help='Quicksearch query to run')
+
+ sp_bzdump.add_argument('--resolve-dupes', action='store_true', help='Resolve dupes')
+ sp_bzdump.set_defaults(func=cmd_bzdump)
+
+ return parser
+
+
+def cmd():
+ parser = setup_parser()
+ cmdargs = parser.parse_args()
+ logger.setLevel(logging.DEBUG)
+
+ ch = logging.StreamHandler()
+ formatter = logging.Formatter('%(message)s')
+ ch.setFormatter(formatter)
+
+ if cmdargs.quiet:
+ ch.setLevel(logging.CRITICAL)
+ elif cmdargs.debug:
+ ch.setLevel(logging.DEBUG)
+ else:
+ ch.setLevel(logging.INFO)
+
+ logger.addHandler(ch)
+
+ if 'func' not in cmdargs:
+ parser.print_help()
+ sys.exit(1)
+
+ with open(cmdargs.config, 'rb') as fh:
+ try:
+ import tomllib # noqa
+ peebz.CONFIG = tomllib.load(fh)
+ except ModuleNotFoundError:
+ import tomli # noqa
+ peebz.CONFIG = tomli.load(fh)
+
+ try:
+ logfile = peebz.CONFIG['logging']['logfile']
+ flh = logging.handlers.WatchedFileHandler(logfile)
+ fmt = '[%(process)d] %(asctime)s - ' + cmdargs.subcmd + ': %(message)s'
+ flh.setFormatter(logging.Formatter(fmt))
+ loglevel = peebz.CONFIG['logging'].get('loglevel', 'info')
+ if loglevel == 'debug':
+ flh.setLevel(logging.DEBUG)
+ else:
+ flh.setLevel(logging.INFO)
+ logger.addHandler(flh)
+ except KeyError:
+ # No file logging for you
+ pass
+
+ cmdargs.func(cmdargs)
+
+
+if __name__ == '__main__':
+ import os
+ # noinspection PyBroadException
+ try:
+ if peebz.__VERSION__.endswith('-dev'):
+ base = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+ dotgit = os.path.join(base, '.git')
+ lines = b4.git_get_command_lines(dotgit, ['rev-parse', '--short', 'HEAD'])
+ if lines:
+ peebz.__VERSION__ = '%s-%.5s' % (peebz.__VERSION__, lines[0].strip())
+ except Exception as ex:
+ pass
+ cmd()
diff --git a/peebz/parse.py b/peebz/parse.py
new file mode 100644
index 0000000..07de9e5
--- /dev/null
+++ b/peebz/parse.py
@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2023 by the Linux Foundation
+
+import sys
+import argparse
+import peebz
+import b4
+import re
+
+from typing import Tuple, Dict
+
+import email.message
+import email.utils
+
+logger = peebz.logger
+
+
+def new_bug_notification(bid: int, inre_cid: int, dry_run: bool = False):
+ msg = email.message.EmailMessage()
+ config = peebz.get_config()
+ bodyvals = {
+ 'bzname': config['bugzilla'].get('name'),
+ 'bug_url': config['bugzilla'].get('bugmask', '').format(bug_id=bid),
+ }
+ bodytpt = peebz.get_template_by_bid('new_bug_notify', bid)
+ body = bodytpt.safe_substitute(bodyvals)
+ sigtpt = peebz.get_template_by_bid('botsig', bid)
+ sigvals = {
+ 'myname': config['bugzilla'].get('name'),
+ 'appname': peebz.__APPNAME__,
+ 'appver': peebz.__VERSION__,
+ }
+ body += sigtpt.safe_substitute(sigvals)
+ msg.set_payload(body, charset='utf-8')
+ peebz.notify_bug(bid, None, msg, inre_cid=inre_cid, dry_run=dry_run)
+
+
+def make_bug_desc_from_body(product: str, component: str, body: str, vals: Dict) -> str:
+ if 'comment_count' in vals and vals['comment_count']:
+ tpt_intro = peebz.get_template_by_product_component('parse_bug_intro_with_count', product, component)
+ else:
+ tpt_intro = peebz.get_template_by_product_component('parse_bug_intro', product, component)
+ tpt_outro = peebz.get_template_by_product_component('parse_bug_outro', product, component)
+ desc = ''
+ intro = tpt_intro.safe_substitute(vals)
+ if intro:
+ desc = intro + '\n\n'
+ desc += body.strip()
+ outro = tpt_outro.safe_substitute(vals)
+ if outro:
+ desc += '\n\n' + outro
+ desc += '\n'
+
+ return desc
+
+
+def new_bug_from_msg(msg: email.message.EmailMessage, product: str, component: str,
+ dry_run: bool = False) -> Tuple[int, int]:
+ msgid, author, subject, body, atts = peebz.msg_parse_for_bug(msg)
+ payload = peebz.get_newbug_payload_by_product_component(product, component)
+ summary = re.sub(r'^\s*(Re|Fwd):\s*', '', subject)
+ vals = {
+ 'author': b4.format_addrs([author]),
+ 'msgid_link': peebz.get_msgid_link(msgid),
+ }
+ desc = make_bug_desc_from_body(product, component, body, vals)
+
+ payload['summary'] = summary
+ payload['description'] = desc
+ if not dry_run:
+ bid, cid = peebz.bz_add_new_bug(payload)
+ logger.debug('new bug bid=%s, cid=%s', bid, cid)
+ recipients = peebz.msg_get_recipients(msg)
+ peebz.db_store_recipients(bid, recipients)
+ if atts:
+ peebz.bz_add_atts_to_bug(bid, atts)
+ else:
+ logger.info('--- DRY RUN ---')
+ logger.info('Would have created a new bug in %s/%s:', product, component)
+ logger.info('Summary: %s', payload['summary'])
+ logger.info('Description:')
+ logger.info(payload['description'])
+ bid = cid = None
+
+ return bid, cid
+
+
+def new_comment_from_msg(bid: int, cid: int, msg: email.message.EmailMessage, dry_run: bool = False) -> int:
+ msgid, author, subject, body, atts = peebz.msg_parse_for_bug(msg)
+ vals = {
+ 'author': b4.format_addrs([author]),
+ 'msgid_link': peebz.get_msgid_link(msgid),
+ }
+ if cid:
+ try:
+ vals['comment_count'] = peebz.bz_get_count_by_bid_cid(bid, cid)
+ except LookupError:
+ pass
+ product, component = peebz.bz_get_product_component_by_bid(bid)
+
+ desc = make_bug_desc_from_body(product, component, body, vals)
+
+ if not dry_run:
+ cid = peebz.bz_add_new_comment(bid, desc)
+ recipients = peebz.msg_get_recipients(msg)
+ peebz.db_store_recipients(bid, recipients)
+ if atts:
+ peebz.bz_add_atts_to_bug(bid, atts)
+ else:
+ logger.info('--- DRY RUN ---')
+ logger.info('Would have added this comment to %s', bid)
+ logger.info(desc)
+ cid = None
+ return cid
+
+
+def process_rfc2822(msg: email.message.EmailMessage, product: str, component: str,
+ dry_run: bool = False) -> None:
+ # Ignore any messages that have an X-Bugzilla-Product header,
+ # so we don't get into any loops
+ if msg.get('x-bugzilla-product'):
+ logger.debug('Skipping bugzilla-originating message')
+ return
+
+ cconf = peebz.get_component_config(product, component)
+ # Get the message-id
+ msgid = b4.LoreMessage.get_clean_msgid(msg)
+ try:
+ # If we have this exact msgid, then it's a dupe
+ bid, cid = peebz.db_get_bid_cid_by_msgid(msgid)
+ logger.info('Already recorded as bid=%s, cid=%s', bid, cid)
+ return
+ except LookupError:
+ pass
+
+ # Walk through references and in-reply-to and see if we know any of them
+ bid = cid = None
+ try:
+ bid, cid = peebz.msg_get_inre_bid_cid(msg)
+ except LookupError:
+ pass
+
+ if bid:
+ bdata = peebz.bz_get_bug(bid)
+ if not bdata['is_open']:
+ logger.info('Bug %s is closed, not adding comments', bid)
+ sys.exit(0)
+
+ cid = new_comment_from_msg(bid, cid, msg, dry_run=dry_run)
+ if not dry_run:
+ peebz.db_store_msgid_bid_cid(msgid, bid, cid)
+ else:
+ bid, cid = new_bug_from_msg(msg, product, component, dry_run=dry_run)
+ if not dry_run:
+ peebz.db_store_msgid_bid_cid(msgid, bid, cid)
+ if cconf.get('new_bug_send_notification'):
+ new_bug_notification(bid, cid, dry_run=dry_run)
+
+ # Do we have any assign triggers?
+ assign_re = cconf.get('pi_assign_regex')
+ if assign_re:
+ matches = re.search(assign_re, msg.as_string(), flags=re.I | re.M)
+ if matches:
+ author = peebz.msg_get_author(msg)
+ fromaddr = author[1]
+ if peebz.bz_check_user_allowed(fromaddr, product, component):
+ assign_to = matches.groups()[0]
+ if assign_to == 'me':
+ logger.debug('me=%s', fromaddr)
+ assign_to = fromaddr
+ # Does this user exist?
+ try:
+ peebz.bz_get_user(assign_to)
+ if not dry_run:
+ peebz.bz_assign_bug(bid, assign_to)
+ else:
+ logger.debug('---DRY RUN---')
+ logger.debug('Would have assigned bid=%s to %s', bid, assign_to)
+
+ except LookupError:
+ logger.info('Unable to assign %s to %s: no such user', bid, assign_to)
+
+
+def main(cmdargs: argparse.Namespace) -> None:
+ msg = peebz.get_msg_from_stdin()
+ product = cmdargs.product
+ component = cmdargs.component
+ if not (product and component):
+ recipients = peebz.msg_get_recipients(msg)
+ try:
+ product, component = peebz.get_product_component_by_recipients(recipients)
+ except LookupError as ex:
+ # TODO: fail properly here
+ logger.info(str(ex))
+ sys.exit(1)
+ process_rfc2822(msg, product, component, dry_run=cmdargs.dry_run)
diff --git a/peebz/pi2bz.py b/peebz/pi2bz.py
new file mode 100644
index 0000000..36f07d3
--- /dev/null
+++ b/peebz/pi2bz.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2023 by the Linux Foundation
+
+import argparse
+import peebz
+import peebz.parse
+import b4
+import urllib.parse
+import email.message
+import gzip
+import datetime
+
+from typing import List, Set
+
+logger = peebz.logger
+b4.logger = logger
+# force b4 to use EmailMessage factory
+b4.emlpolicy = peebz.emlpolicy
+
+
+def get_query_results(query_url: str) -> List:
+ loc = urllib.parse.urlparse(query_url)
+ logger.debug('query=%s', query_url)
+ logger.debug('grabbing search results from %s', loc.netloc)
+ session = peebz.get_requests_session()
+ # For the query to retrieve a mbox file, we need to send a POST request
+ resp = session.post(query_url, data='')
+ if resp.status_code == 404:
+ raise LookupError('Nothing matching query=%s', query_url)
+ if resp.status_code != 200:
+ raise LookupError('Server returned an error for %s: %s' % (query_url, resp.status_code))
+ t_mbox = gzip.decompress(resp.content)
+ resp.close()
+ if not len(t_mbox):
+ raise LookupError('Nothing matching query=%s', query_url)
+ return b4.split_and_dedupe_pi_results(t_mbox)
+
+
+def get_sorted_thread(url: str, msgid: str) -> List[email.message.EmailMessage]:
+ loc = urllib.parse.urlparse(url)
+ mbox_url = url.rstrip('/') + '/' + urllib.parse.quote_plus(msgid) + '/t.mbox.gz'
+ logger.debug('mbox_url=%s', mbox_url)
+ logger.debug('grabbing thread from %s', loc.netloc)
+ session = peebz.get_requests_session()
+ resp = session.get(mbox_url)
+ if resp.status_code == 404:
+ raise LookupError('Nothing matching mbox_url=%s', mbox_url)
+ if resp.status_code != 200:
+ raise LookupError('Server returned an error for %s: %s' % (mbox_url, resp.status_code))
+ t_mbox = gzip.decompress(resp.content)
+ resp.close()
+
+ deduped = b4.split_and_dedupe_pi_results(t_mbox)
+ if not deduped:
+ raise LookupError('No messages matching mbox_url=%s' % mbox_url)
+ strict = b4.get_strict_thread(deduped, msgid)
+ return peebz.sort_msgs_by_received(strict)
+
+
+def get_new_msgs(msgs: List[email.message.EmailMessage]) -> List[email.message.EmailMessage]:
+ new_msgs = list()
+ for msg in msgs:
+ msgid = b4.LoreMessage.get_clean_msgid(msg)
+ try:
+ peebz.db_get_bid_cid_by_msgid(msgid)
+ continue
+ except LookupError:
+ new_msgs.append(msg)
+
+ return new_msgs
+
+
+def get_tracked_bug_msgids(product: str, component: str) -> Set[str]:
+ cconf = peebz.get_component_config(product, component)
+ params = {
+ 'include_fields': 'id',
+ 'product': product,
+ 'component': component,
+ 'quicksearch': 'OPEN',
+ 'chfieldfrom': '90d',
+ }
+ params.update(cconf.get('bz_query_params', dict()))
+ rdata = peebz.bz_rest('bug', params=params)
+ msgids = set()
+ for bdata in rdata.get('bugs', list()):
+ bid = bdata['id']
+ try:
+ msgid = peebz.db_get_msgid_by_bid_cid(bid, None)
+ logger.debug('bid=%s is tracked as msgid=%s', bid, msgid)
+ msgids.add(msgid)
+ except LookupError:
+ logger.debug('Not tracking bid=%s', bid)
+
+ return msgids
+
+
+def update_component(product: str, component: str, dry_run: bool = False):
+ logger.info('Running pi2bz for %s/%s, dry_run=%s', product, component, dry_run)
+ cconf = peebz.get_component_config(product, component)
+ tracked = get_tracked_bug_msgids(product, component)
+ url = cconf.get('pi_url').rstrip('/')
+ now = datetime.datetime.utcnow()
+
+ seen_msgids = set()
+ updates = list()
+ if len(tracked):
+ logger.info('Checking for updates in %s tracked threads', len(tracked))
+ for msgid in tracked:
+ try:
+ tmsgs = get_sorted_thread(url, msgid)
+ except LookupError:
+ logger.debug('No results returned for msgid=%s', msgid)
+ continue
+
+ for tmsg in tmsgs:
+ tmsgid = b4.LoreMessage.get_clean_msgid(tmsg)
+ if tmsgid in seen_msgids:
+ logger.debug('Already seen %s', tmsgid)
+ continue
+ seen_msgids.add(tmsgid)
+ try:
+ peebz.db_get_bid_cid_by_msgid(tmsgid)
+ logger.debug('%s has already been processed', tmsgid)
+ continue
+ except LookupError:
+ logger.debug('New message in tracked thread: %s', tmsgid)
+ updates.append(tmsg)
+
+ # Now grab the latest query matches
+ query = cconf.get('pi_query')
+ if query:
+ logger.info('Running query for %s/%s', product, component)
+ try:
+ last_check = peebz.db_get_query_last_check(product, component)
+ query += f' AND dt:{last_check}..'
+ except LookupError:
+ pass
+ qquery = urllib.parse.quote_plus(query)
+ query_url = url.rstrip('/') + f'/?x=m&q={qquery}'
+ lastdt = now.strftime('%Y%m%d%H%M%S')
+ try:
+ msgs = get_query_results(query_url)
+ for msg in msgs:
+ msgid = b4.LoreMessage.get_clean_msgid(msg)
+ if msgid in seen_msgids:
+ logger.debug('Already seen %s', msgid)
+ continue
+
+ # New thing to track!
+ seen_msgids.add(msgid)
+ author = peebz.msg_get_author(msg)
+ fromaddr = author[1]
+ if not peebz.bz_check_user_allowed(fromaddr, product, component):
+ logger.debug('skipping msg %s', msg.get('Subject'))
+ continue
+ # Retrieve and queue up the entire thread
+ try:
+ tmsgs = get_sorted_thread(url, msgid)
+ except LookupError:
+ logger.debug('No results returned for msgid=%s', msgid)
+ continue
+ for tmsg in tmsgs:
+ tmsgid = b4.LoreMessage.get_clean_msgid(tmsg)
+ seen_msgids.add(tmsgid)
+ updates.append(tmsg)
+
+ except LookupError:
+ logger.info('No new results for product=%s, component=%s', product, component)
+
+ if not dry_run:
+ peebz.db_store_query_last_check(product, component, lastdt)
+
+ if not updates:
+ logger.info('No new messages to add to bugzilla for %s/%s', product, component)
+ return
+
+ for msg in updates:
+ logger.debug('Recording %s', msg.get('Subject'))
+ peebz.parse.process_rfc2822(msg, product, component, dry_run=dry_run)
+
+
+def main(cmdargs: argparse.Namespace):
+ config = peebz.get_config()
+ # Iterate all components
+ for bz_product, bz_components in config['components'].items():
+ for bz_component in bz_components.keys():
+ update_component(bz_product, bz_component, dry_run=cmdargs.dry_run)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..cab6f85
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,5 @@
+b4>=0.12.2,<1.0
+Pygments>=2.14.0,<3.0
+requests>=2.28
+sqlalchemy>=1.4,<2.0
+tomli>=2.0; python_version < '3.11'
