| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # SPDX-License-Identifier: GPL-2.0-or-later |
| # Copyright (C) 2023 by the Linux Foundation |
| |
| import argparse |
| import bugspray |
| import bugspray.parse |
| import urllib.parse |
| import datetime |
| import re |
| |
| from typing import Set |
| |
| logger = bugspray.logger |
| |
| |
| def get_tracked_bug_msgids(product: str, component: str) -> Set[str]: |
| cconf = bugspray.get_component_config(product, component) |
| params = { |
| 'include_fields': 'id', |
| 'product': product, |
| 'component': component, |
| 'quicksearch': 'OPEN', |
| 'chfieldfrom': '90d', |
| } |
| params.update(cconf.get('bz_query_params', dict())) |
| rdata = bugspray.bz_rest('bug', params=params) |
| msgids = set() |
| for bdata in rdata.get('bugs', list()): |
| bid = bdata['id'] |
| try: |
| msgid = bugspray.db_get_msgid_by_bid_cid(bid, None) |
| logger.debug('bid=%s is tracked as msgid=%s', bid, msgid) |
| msgids.add(msgid) |
| except LookupError: |
| logger.debug('Not tracking bid=%s', bid) |
| |
| return msgids |
| |
| |
| def update_component(product: str, component: str, dry_run: bool = False): |
| logger.info('Running pi2bz for %s/%s, dry_run=%s', product, component, dry_run) |
| cconf = bugspray.get_component_config(product, component) |
| tracked = get_tracked_bug_msgids(product, component) |
| pstitle = cconf.get('pi_search') |
| pconf = bugspray.get_pi_search(pstitle) |
| |
| url = pconf.get('url').rstrip('/') |
| now = datetime.datetime.now(datetime.timezone.utc) |
| try: |
| last_check = bugspray.db_get_query_last_check(product, component) |
| except LookupError: |
| last_check = None |
| |
| seen_msgids = set() |
| updates = list() |
| if len(tracked): |
| logger.info('Checking for updates in %s tracked threads', len(tracked)) |
| for msgid in tracked: |
| try: |
| tmsgs = bugspray.pi_get_sorted_thread(url, msgid, since=last_check) |
| logger.info(' %s new updates in: %s', len(tmsgs), msgid) |
| except LookupError: |
| logger.debug(' 0 new updates in: %s', msgid) |
| continue |
| |
| for tmsg in tmsgs: |
| tmsgid = bugspray.get_msgid(tmsg) |
| if tmsgid in seen_msgids: |
| logger.debug('Already seen %s', tmsgid) |
| continue |
| seen_msgids.add(tmsgid) |
| try: |
| bugspray.db_get_bid_cid_by_msgid(tmsgid) |
| logger.debug('%s has already been processed', tmsgid) |
| continue |
| except LookupError: |
| logger.debug('New message in tracked thread: %s', tmsgid) |
| updates.append(tmsg) |
| |
| # Now grab the latest query matches |
| query = pconf.get('query') |
| if query: |
| logger.info('Running query for %s/%s', product, component) |
| if last_check: |
| query += f' AND dt:{last_check}..' |
| qquery = urllib.parse.quote_plus(query) |
| query_url = url.rstrip('/') + f'/?x=m&q={qquery}' |
| try: |
| msgs = bugspray.pi_get_query_results(query_url) |
| for msg in msgs: |
| msgid = bugspray.get_msgid(msg) |
| if msgid in seen_msgids: |
| logger.debug('Already seen %s', msgid) |
| continue |
| |
| # New thing to track! |
| seen_msgids.add(msgid) |
| author = bugspray.msg_get_author(msg) |
| fromaddr = author[1] |
| if not bugspray.bz_check_user_allowed(fromaddr, product, component): |
| logger.debug('author=%s not allowed, skipping msg %s', fromaddr, msg.get('Subject')) |
| continue |
| # Check fine trigger, if configured |
| trigger_res = pconf.get('trigger_regexes', list()) |
| if trigger_res: |
| payload = bugspray.msg_get_payload(msg, strip_quoted=True, strip_signature=False) |
| found = False |
| for trigger_re in trigger_res: |
| matches = re.search(trigger_re, payload, flags=re.I | re.M) |
| if matches: |
| logger.debug('found trigger_regex: %s', trigger_re) |
| found = True |
| break |
| |
| if not found: |
| logger.debug('trigger_regexes not found, skipping msg %s', msg.get('Subject')) |
| continue |
| |
| # Retrieve and queue up the entire thread |
| try: |
| tmsgs = bugspray.pi_get_sorted_thread(url, msgid) |
| except LookupError: |
| logger.debug('No results returned for msgid=%s', msgid) |
| continue |
| for tmsg in tmsgs: |
| tmsgid = bugspray.get_msgid(tmsg) |
| seen_msgids.add(tmsgid) |
| updates.append(tmsg) |
| |
| except LookupError: |
| logger.info('No new results for product=%s, component=%s', product, component) |
| |
| # TODO: check bug mentions |
| |
| if not dry_run: |
| # Give a 10-minute overlap buffer |
| bufferago = now - datetime.timedelta(minutes=10) |
| lastdt = bufferago.strftime('%Y%m%d%H%M%S') |
| bugspray.db_store_query_last_check(product, component, lastdt) |
| |
| if not updates: |
| logger.info('No new messages to add to bugzilla for %s/%s', product, component) |
| return |
| |
| for msg in updates: |
| logger.debug('Recording %s', msg.get('Subject')) |
| bugspray.parse.process_rfc2822(msg, product, component, dry_run=dry_run) |
| |
| |
| def main(cmdargs: argparse.Namespace): |
| config = bugspray.get_config() |
| # Iterate all components |
| for bz_product, bz_components in config['components'].items(): |
| for bz_component in bz_components.keys(): |
| update_component(bz_product, bz_component, dry_run=cmdargs.dry_run) |