blob: 7a60ddc72994e5a78e2234732520fa9be819571d [file] [log] [blame]
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright (C) 2023 by the Linux Foundation
import argparse
import bugspray
import bugspray.parse
import urllib.parse
import datetime
import re
from typing import Set
logger = bugspray.logger
def get_tracked_bug_msgids(product: str, component: str) -> Set[str]:
cconf = bugspray.get_component_config(product, component)
params = {
'include_fields': 'id',
'product': product,
'component': component,
'quicksearch': 'OPEN',
'chfieldfrom': '90d',
}
params.update(cconf.get('bz_query_params', dict()))
rdata = bugspray.bz_rest('bug', params=params)
msgids = set()
for bdata in rdata.get('bugs', list()):
bid = bdata['id']
try:
msgid = bugspray.db_get_msgid_by_bid_cid(bid, None)
logger.debug('bid=%s is tracked as msgid=%s', bid, msgid)
msgids.add(msgid)
except LookupError:
logger.debug('Not tracking bid=%s', bid)
return msgids
def update_component(product: str, component: str, dry_run: bool = False):
logger.info('Running pi2bz for %s/%s, dry_run=%s', product, component, dry_run)
cconf = bugspray.get_component_config(product, component)
tracked = get_tracked_bug_msgids(product, component)
pstitle = cconf.get('pi_search')
pconf = bugspray.get_pi_search(pstitle)
url = pconf.get('url').rstrip('/')
now = datetime.datetime.now(datetime.timezone.utc)
try:
last_check = bugspray.db_get_query_last_check(product, component)
except LookupError:
last_check = None
seen_msgids = set()
updates = list()
if len(tracked):
logger.info('Checking for updates in %s tracked threads', len(tracked))
for msgid in tracked:
try:
tmsgs = bugspray.pi_get_sorted_thread(url, msgid, since=last_check)
logger.info(' %s new updates in: %s', len(tmsgs), msgid)
except LookupError:
logger.debug(' 0 new updates in: %s', msgid)
continue
for tmsg in tmsgs:
tmsgid = bugspray.get_msgid(tmsg)
if tmsgid in seen_msgids:
logger.debug('Already seen %s', tmsgid)
continue
seen_msgids.add(tmsgid)
try:
bugspray.db_get_bid_cid_by_msgid(tmsgid)
logger.debug('%s has already been processed', tmsgid)
continue
except LookupError:
logger.debug('New message in tracked thread: %s', tmsgid)
updates.append(tmsg)
# Now grab the latest query matches
query = pconf.get('query')
if query:
logger.info('Running query for %s/%s', product, component)
if last_check:
query += f' AND dt:{last_check}..'
qquery = urllib.parse.quote_plus(query)
query_url = url.rstrip('/') + f'/?x=m&q={qquery}'
try:
msgs = bugspray.pi_get_query_results(query_url)
for msg in msgs:
msgid = bugspray.get_msgid(msg)
if msgid in seen_msgids:
logger.debug('Already seen %s', msgid)
continue
# New thing to track!
seen_msgids.add(msgid)
author = bugspray.msg_get_author(msg)
fromaddr = author[1]
if not bugspray.bz_check_user_allowed(fromaddr, product, component):
logger.debug('author=%s not allowed, skipping msg %s', fromaddr, msg.get('Subject'))
continue
# Check fine trigger, if configured
trigger_res = pconf.get('trigger_regexes', list())
if trigger_res:
payload = bugspray.msg_get_payload(msg, strip_quoted=True, strip_signature=False)
found = False
for trigger_re in trigger_res:
matches = re.search(trigger_re, payload, flags=re.I | re.M)
if matches:
logger.debug('found trigger_regex: %s', trigger_re)
found = True
break
if not found:
logger.debug('trigger_regexes not found, skipping msg %s', msg.get('Subject'))
continue
# Retrieve and queue up the entire thread
try:
tmsgs = bugspray.pi_get_sorted_thread(url, msgid)
except LookupError:
logger.debug('No results returned for msgid=%s', msgid)
continue
for tmsg in tmsgs:
tmsgid = bugspray.get_msgid(tmsg)
seen_msgids.add(tmsgid)
updates.append(tmsg)
except LookupError:
logger.info('No new results for product=%s, component=%s', product, component)
# TODO: check bug mentions
if not dry_run:
# Give a 10-minute overlap buffer
bufferago = now - datetime.timedelta(minutes=10)
lastdt = bufferago.strftime('%Y%m%d%H%M%S')
bugspray.db_store_query_last_check(product, component, lastdt)
if not updates:
logger.info('No new messages to add to bugzilla for %s/%s', product, component)
return
for msg in updates:
logger.debug('Recording %s', msg.get('Subject'))
bugspray.parse.process_rfc2822(msg, product, component, dry_run=dry_run)
def main(cmdargs: argparse.Namespace):
config = bugspray.get_config()
# Iterate all components
for bz_product, bz_components in config['components'].items():
for bz_component in bz_components.keys():
update_component(bz_product, bz_component, dry_run=cmdargs.dry_run)