#!/usr/bin/python -tt
##
# Copyright (C) 2013 by Kernel.org
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#
import os
import time
import datetime
import re

from git import Repo

import requests

from distutils.version import StrictVersion

from pelican import signals, utils

from feedgenerator import Rss201rev2Feed
import json

# Pelican plugin API
def register():
    signals.article_generator_init.connect(kernel_releases_init)
    signals.article_generate_context.connect(fetch_kernel_releases)


def kernel_releases_init(gen):
    gen.releases_instance = KernelReleases(gen)


def fetch_kernel_releases(gen, metadata):
    gen.context['current_releases'] = gen.releases_instance.current_releases
    gen.context['latest_release'] = gen.releases_instance.latest_release

class KernelReleases():
    def __init__(self, generator):
        self.release_tracker = '/var/lib/mirror/release-tracker.json'

        self.rss_path = os.path.join(generator.output_path, 'feeds', 'kdist.xml')
        self.finger_path = os.path.join(generator.output_path, 'finger_banner')
        self.json_path = os.path.join(generator.output_path, 'releases.json')

        self.reqs = requests.Session()

        settings = generator.settings

        GIT_MAINLINE = settings.get('GIT_MAINLINE')
        GIT_STABLE = settings.get('GIT_STABLE')
        GIT_NEXT = settings.get('GIT_NEXT')

        LONGTERM_KERNELS = settings.get('LONGTERM_KERNELS')
        EOL_KERNELS = settings.get('EOL_KERNELS')

        repo = Repo(GIT_MAINLINE)
        tagrefs = self.get_tagref_list(repo)

        rc_reg = re.compile('v.*-rc\d+$')
        mainline_rc = self.find_latest_matching(tagrefs, rc_reg)
        rel_reg = re.compile('v[^-]*\d+$')
        mainline_rel = self.find_latest_matching(tagrefs, rel_reg)

        # if mainline_rel is newer than mainline_rc, ignore mainline_rc
        if mainline_rc[1] < mainline_rel[1]:
            mainline_rc = None

        # Move on to the linux-stable repo
        repo = Repo(GIT_STABLE)
        # ignore any tags older than 12 months
        cutoff = time.time() - 31536000

        # EOL cutoff is 30 days
        eol_cutoff = time.time() - 2592000

        tagrefs = self.get_tagref_list(repo, cutoff)

        # find all tags matching vX.X.X
        rel_reg = re.compile('v\d+\.\d+\.\d+$')
        matched = []
        for tagref in tagrefs:
            if rel_reg.match(tagref.name):
                matched.append(tagref)

        stable = []
        seen = []

        for tagref in matched:
            # does it match a longterm release?
            ignore = False
            for ver in LONGTERM_KERNELS:
                if tagref.name.find(ver+'.') == 1:
                    # this is a long-term release, ignore
                    ignore = True
                    continue
            if ignore:
                continue

            # Drop the final \.\d+ and find the latest matching
            components = tagref.name.split('.')
            regex = re.compile('^' + '\\.'.join(components[:-1]))

            if regex in seen:
                continue

            latest_matching = self.find_latest_matching(matched, regex)
            if latest_matching is not None:
                stable.append(latest_matching)

            seen.append(regex)

        stable = sorted(stable, key=lambda x: StrictVersion(x[0][1:]), reverse=True)

        releases = []

        if mainline_rc is not None:
            releases.append(self.make_release_line(mainline_rc, 'mainline'))

        # look at the last stable and see if mainline_rel fits into it
        latest_stable = stable.pop(0)

        if latest_stable[0].find(mainline_rel[0]) != 0:
            releases.append(self.make_release_line(mainline_rel, 'mainline'))
            latest = self.make_release_line(mainline_rel, 'latest')
        else:
            latest = self.make_release_line(latest_stable, 'latest')
            # if latest stable is newer than latest mainline, but there are no
            # -rc kernels, list the latest mainline in the releases table anyway.
            if mainline_rc is None:
                releases.append(self.make_release_line(mainline_rel, 'mainline'))

        releases.append(self.make_release_line(latest_stable, 'stable'))

        # add other stable kernels and mark EOL accordingly
        for rel in stable:
            iseol = False
            for eolrel in EOL_KERNELS:
                if rel[0].find(eolrel) == 1:
                    iseol = True
                    break

            # Make sure EOL kernels stick around for max 30 days
            if iseol and rel[1] < eol_cutoff:
                continue

            releases.append(self.make_release_line(rel, 'stable', iseol))

        # find latest long-term releases
        for rel in LONGTERM_KERNELS:
            components = rel.split('.')
            regex = re.compile('^v' + '\\.'.join(components) + '\\.\\d+$')
            found = self.find_latest_matching(tagrefs, regex)
            if found is not None:
                iseol = False
                for eolrel in EOL_KERNELS:
                    if found[0].find(eolrel) == 1:
                        iseol = True
                        break

                if iseol and found[1] < eol_cutoff:
                    # Too old to list on the front page
                    continue

                releases.append(self.make_release_line(found, 'longterm', iseol))

        # now find latest tag in linux-next
        repo = Repo(GIT_NEXT)
        tagrefs = self.get_tagref_list(repo, cutoff)

        regex = re.compile('^next-')
        rel = self.find_latest_matching(tagrefs, regex)
        releases.append(self.make_release_line(rel, 'linux-next'))

        self.current_releases = releases
        self.latest_release = latest

        self.generate_rss_feed()
        self.generate_finger_banner()
        self.generate_releases_json()
        self.check_release_tracker()

    def check_url_exists(self, path):
        r = self.reqs.head('https://www.kernel.org/%s' % path)
        return r.status_code == 200

    def check_release_tracker(self):
        if 'PELICAN_DRYRUN' in os.environ.keys():
            print 'PELICAN_DRYRUN found, not doing release tracking'
            return

        import socket
        send_mail = False

        if socket.gethostname() == 'fe-sync.int.kernel.org':
            send_mail = True

        # Load known releases from the release tracker file
        known = []

        if os.path.exists(self.release_tracker):
            try:
                fh = open(self.release_tracker, 'r')
                known = json.load(fh)
                fh.close()
            except:
                # it's better to not announce something than to spam
                # people needlessly.
                send_mail = False

        for chunks in self.current_releases:
            release = chunks[1]
            if release not in known:
                # This appears to be a new release.
                if chunks[5] is None or chunks[6] is None or chunks[7] is None:
                    # Don't announce anything that doesn't have source, patch or sign.
                    continue

                known.append(release)
                if send_mail:
                    self.send_release_email(chunks)

        fh = open(self.release_tracker, 'w')
        json.dump(known, fh, indent=4)
        fh.close()

    def send_release_email(self, chunks):
        (label, release, iseol, timestamp, isodate, source, sign, patch, incr, changelog, gitweb, diffview) = chunks
        if iseol:
            eol = ' (EOL)'
        else:
            eol = ''

        import smtplib
        from email.mime.text import MIMEText
        from email.Utils import COMMASPACE, formatdate, make_msgid

        announce_list = 'linux-kernel-announce@vger.kernel.org'
        smtp_server = 'mail.kernel.org'

        # This allows us to run tests in pre-prod
        if 'ANNOUNCE_LIST' in os.environ.keys():
            announce_list = os.environ['ANNOUNCE_LIST']
        if 'SMTP_SERVER' in os.environ.keys():
            smtp_server = os.environ['SMTP_SERVER']

        body = ("Linux kernel version %s%s has been released. It is available from:\r\n" % (release, eol)
                + "\r\n"
                + "Patch:          https://www.kernel.org/%s\r\n" % patch
                + "Full source:    https://www.kernel.org/%s\r\n" % source
                + "PGP Signature:  https://www.kernel.org/%s\r\n" % sign
                + "\r\n"
                + "-----------------------------------------------------------------------------\r\n"
                + "This is an automatically generated message. To unsubscribe from this list,\r\n"
                + "please send a message to majordomo@vger.kernel.org containing the line:\r\n"
                + "\r\n"
                + "\tunsubscribe linux-kernel-announce <your_email_address>\r\n"
                + "\r\n"
                + "... where <your_email_address> is the email address you used to subscribe\r\n"
                + "to this list.\r\n"
                + "-----------------------------------------------------------------------------\r\n")

        if diffview is not None:
            body += ("\r\n"
                + "You can view the summary of the changes at the following URL:\r\n"
                + "%s\r\n" % diffview)

        msg = MIMEText(body)
        msg['Subject'] = "Linux kernel %s released" % release
        msg['From'] = 'Linux Kernel Distribution System <kdist@linux.kernel.org>'
        msg['To'] = announce_list
        msg['Date'] = formatdate(localtime=True)
        msg['Message-Id'] = make_msgid('kdist.linux.kernel.org')
        msg['X-Linux-Kernel-Version'] = release
        msg['X-Linux-Kernel-Patch-URL'] = "https://www.kernel.org/%s" % patch
        msg['X-Linux-Kernel-Full-URL'] = "https://www.kernel.org/%s" % source

        s = smtplib.SMTP(smtp_server)
        s.sendmail('kdist@linux.kernel.org', [announce_list,], msg.as_string())
        s.quit()

    def generate_releases_json(self):
        # Put release info into a .json file for easy import and parse
        out = {'releases': []}
        for entry in self.current_releases:
            (label, release, iseol, timestamp, isodate, source, sign, patch, incr, changelog, gitweb, diffview) = entry

            if source:
                source = 'https://www.kernel.org/%s' % source
            if sign:
                sign = 'https://www.kernel.org/%s' % sign
            if patch:
                patch = 'https://www.kernel.org/%s' % patch
            if incr:
                incr = 'https://www.kernel.org/%s' % incr
            if changelog:
                changelog = 'https://www.kernel.org/%s' % changelog

            relhash = {
                'moniker': label,
                'version': release,
                'iseol':   iseol,
                'released': {
                    'timestamp': timestamp,
                    'isodate':   isodate,
                },
                'source': source,
                'pgp': sign,
                'patch': {
                    'full': patch,
                    'incremental': incr,
                },
                'changelog': changelog,
                'gitweb': gitweb,
                'diffview': diffview
            }

            out['releases'].append(relhash)

        out['latest_stable'] = {
            'version': self.latest_release[1],
        }

        fh = open(self.json_path, 'w')
        json.dump(out, fh, indent=4)
        fh.close()

    def generate_finger_banner(self):
        # Just a bit of legacy silliness
        contents = ''
        for chunks in self.current_releases:
            label = chunks[0]
            release = chunks[1]
            iseol = chunks[2]
            line = ''

            if label not in ('mainline', 'linux-next'):
                bits = release.split('.')
                bits.pop(-1)
                line = ' ' + '.'.join(bits)

            if iseol:
                eol = ' (EOL)'
            else:
                eol = ''

            leftside = 'The latest %s%s version of the Linux kernel is:' % (label, line)
            contents += '{0:<61} {1}{2}\n'.format(leftside, release, eol)

        fp = open(self.finger_path, 'w')
        fp.write(contents)
        fp.close()

    def generate_rss_feed(self):
        feed = Rss201rev2Feed(
            title='Latest Linux Kernel Versions',
            link='http://www.kernel.org',
            feed_url='http://www.kernel.org/feeds/kdist.xml',
            description='Latest Linux Kernel Versions',
            creator='FTP Admin <ftpadmin@kernel.org>'
        )
        for entry in self.current_releases:
            (label, release, iseol, timestamp, isodate, source, sign, patch, incr, changelog, gitweb, diffview) = entry

            if iseol:
                eol = ' (EOL)'
            else:
                eol = ''

            contents = '''
            <table>
                <tr><th align="right">Version:</th><td><strong>%s%s</strong> (%s)</td></tr>
                <tr><th align="right">Released:</th><td>%s</td></tr>
            ''' % (release, eol, label, isodate)

            if source:
                contents += '''
                <tr><th align="right">Source:</th><td><a href="https://www.kernel.org/%s">linux-%s.tar.xz</a></td></tr>''' % (source, release)

            if sign:
                contents += '''
                <tr><th align="right">PGP Signature:</th><td><a href="https://www.kernel.org/%s">linux-%s.tar.sign</a></td></tr>''' % (sign, release)

            if patch:
                contents += '''
                <tr><th align="right">Patch:</th><td><a href="https://www.kernel.org/%s">patch-%s.xz</a>''' % (patch, release)
                if incr:
                    contents += ''' (<a href="https://www.kernel.org/%s">Incremental</a>)''' % incr
                contents += '''</td></tr>'''

            if changelog:
                contents += '''
                <tr><th align="right">ChangeLog:</th><td><a href="https://www.kernel.org/%s">ChangeLog-%s</a></td></tr>''' % (changelog, release)

            contents += '''
            </table>'''

            feed.add_item(
                title='%s: %s' % (release, label),
                link='http://www.kernel.org/',
                unique_id='kernel.org,%s,%s,%s' % (label, release, isodate),
                description=contents,
                pubdate=datetime.datetime.fromtimestamp(timestamp)
            )

        # We really should be generating after site is done,
        # but I'm too lazy to figure out the plugin hooks for that
        utils.mkdir_p(os.path.dirname(self.rss_path))

        fp = open(self.rss_path, 'w')
        feed.write(fp, 'utf-8')
        fp.close()

    def _get_release_dir_by_version(self, version):
        # some magic here to calculate where the source is
        urlpath = 'pub/linux/kernel'

        if version.find('3.') == 0:
            # This is version 3.x, so will be in /v3.x/
            urlpath += '/v3.x'

        elif version.find('4.') == 0:
            # When Linus feels like releasing a 4.x, it'll be in /v4.x/
            urlpath += '/v4.x'

        elif version.find('2.6') == 0:
            # We're hardcoding ourselves here, but this will rarely change
            urlpath += '/v2.6/longterm'

            if version.find('2.6.32') == 0:
                urlpath += '/v2.6.32'
            else:
                urlpath += '/v2.6.34'

        if version.find('-rc') > 0:
            # This is a testing kernel, so will be in /testing/
            urlpath += '/testing'

        return urlpath

    def _get_source_path_by_version(self, version):
        dirpath = self._get_release_dir_by_version(version)
        path = '%s/linux-%s.tar.xz' % (dirpath, version)

        return path

    def _get_sign_path_by_version(self, version):
        dirpath = self._get_release_dir_by_version(version)
        path = '%s/linux-%s.tar.sign' % (dirpath, version)

        return path

    def _get_patch_path_by_version(self, version):
        dirpath = self._get_release_dir_by_version(version)
        path = '%s/patch-%s.xz' % (dirpath, version)

        return path

    def _get_prev_release(self, release, label):
        # Returns (previous mainline, previous incremental)
        # for 3.17-rc2:  (3.16, 3.17-rc1)
        # for 3.16.2:    (3.16, 3.16.1)
        # for 2.6.32.63: (2.6.32, 2.6.32.62)
        # for 3.17-rc1:  (3.16, None)
        # for 3.17:      (3.16, None)
        # for 3.16.1:    (3.16, None)

        if release.find('next') != -1:
            # Futile for next kernels, as we have to look at actual tags
            return None, None

        bits = release.split('.')

        if len(bits) < 2:
            # What happen? Someone set up us something weird.
            return None, None

        mainline = None
        incremental = None

        if label.find('mainline') == 0:
            # Does the last bit have '-rc' in it?
            if bits[-1].find('-rc') > 0:
                rcbits = bits[-1].split('-rc')
                try:
                    lastbit = int(rcbits.pop(-1))
                    mainlbit = int(rcbits.pop(-1))

                    if lastbit > 1:
                        # We have an incremental
                        prevbit = str(lastbit - 1)
                        prevrc = '%s-rc%s' % (mainlbit, prevbit)
                        newbits = bits[:-1] + [prevrc]
                        incremental = '.'.join(newbits)
                    # Now arrive at mainline
                    if mainlbit == 0:
                        # We're looking at 4.0-rcX, then. Cowardly bail out with only incremental
                        return None, incremental
                    prevbit = str(mainlbit - 1)
                    newbits = bits[:-1] + [prevbit]
                    mainline = '.'.join(newbits)

                except ValueError:
                    pass

                return mainline, incremental

            # First, most unusual case -- a mainline release ending in .0
            try:
                lastbit = int(bits.pop(-1))
                if lastbit == 0:
                    # We can't really figure out what the previous release was based on this info, so
                    # we cowardly pretend it didn't happen
                    return None, None

                prevbit = str(lastbit - 1)
                bits.append(prevbit)
                mainline = '.'.join(bits)
            except ValueError:
                # Not sure what happened, but let's pretend it didn't work
                return None, None
            
            return mainline, incremental

        # Not a mainline kernel, so a simpler logic
        try:
            lastbit = int(bits.pop(-1))
            # Combining the remaining bits gives us mainline
            mainline = '.'.join(bits)

            if lastbit > 1:
                prevbit = str(lastbit - 1)
                bits.append(prevbit)
                incremental = '.'.join(bits)
            else:
                # We don't have a previous incremental in this case
                incremental = None

            return mainline, incremental

        except ValueError:
            # Not sure what happened, but let's pretend it didn't work
            return None, None

        return None, None


    def make_release_line(self, rel, label, iseol=False):
        # drop the leading 'v':
        if rel[0][0] == 'v':
            release = rel[0][1:]
        else:
            release = rel[0]

        timestamp = rel[1]
        isodate = time.strftime('%Y-%m-%d', time.gmtime(rel[1]))

        source = None
        sign = None
        patch = None
        incr = None
        changelog = None
        gitweb = None
        diffview = None

        if release.find('next') != 0:
            # next don't have anything besides a tag

            source = self._get_source_path_by_version(release)
            sign = self._get_sign_path_by_version(release)
            patch = self._get_patch_path_by_version(release)

            prevmainline, previncremental = self._get_prev_release(release, label)

            if label.find('stable') == 0 or label.find('longterm') == 0:
                dirpath = self._get_release_dir_by_version(release)
                changelog = '%s/ChangeLog-%s' % (dirpath, release)
                cgitpath = 'https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git'
                gitweb = ('%s/log/?id=refs/tags/v%s' % (cgitpath, release))

                # incr patches are named incr/3.5.(X-1)-(X).xz
                if previncremental:
                    lastbit = release.split('.')[-1]
                    incr = '%s/incr/patch-%s-%s.xz' % (dirpath, previncremental, lastbit)
                    diffview = '%s/diff/?id=v%s&id2=v%s&dt=2' % (cgitpath, release, previncremental)
                elif prevmainline:
                    # diffview to previous mainline
                    diffview = ('%s/diff/?id=v%s&id2=v%s&dt=2' % (cgitpath, release, prevmainline))

            elif label.find('mainline') == 0:
                cgitpath = 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git'
                gitweb = '%s/log/?id=refs/tags/v%s' % (cgitpath, release)
                if previncremental:
                    diffview = '%s/diff/?id=v%s&id2=v%s&dt=2' % (cgitpath, release, previncremental)
                elif prevmainline:
                    diffview = ('%s/diff/?id=v%s&id2=v%s&dt=2' % (cgitpath, release, prevmainline))

        else:
            gitweb = 'https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/log/?id=refs/tags/%s' % release
            # Not offering a diffview, as it's too hard to calculate
            diffview = None

        # Verify that patch, changelog and incremental patch exist
        if patch and not self.check_url_exists(patch):
            patch = None
        if changelog and not self.check_url_exists(changelog):
            changelog = None
        if incr and not self.check_url_exists(incr):
            incr = None

        # This needs to be refactored into a hash. In my defense,
        # it started with 3 entries.
        return label, release, iseol, timestamp, isodate, source, sign, patch, incr, changelog, gitweb, diffview

    def get_tagref_list(self, repo, cutoff=None):
        tagrefs = []
        for tagref in repo.tags:
            try:
                tdate = tagref.tag.tagged_date
            except:
                # Some of the tags break git-python due to not having an associated
                # commit. Work around this limitation by ignoring those tags.
                continue

            # Is it too old?
            if cutoff is not None and tdate < cutoff:
                continue

            tagrefs.append(tagref)

        return tagrefs

    def _check_tarball_by_tagname(self, tagname):
        # Only check those starting with "v"
        if tagname[0] == 'v':
            # Ignore this check for next-YYYYMMDD kernels
            source = self._get_source_path_by_version(tagname[1:])

            if not self.check_url_exists(source):
                return False

        return True

    def find_latest_matching(self, tagrefs, regex, blacklist=[]):
        current = None

        for tagref in tagrefs:
            # Does it match the regex?
            if not regex.match(tagref.name):
                continue

            if tagref.name in blacklist:
                continue

            tdate = tagref.tag.tagged_date

            # is it older than current?
            if current is None or current.tag.tagged_date < tdate:
                current = tagref

        if current is None:
            return None

        if not self._check_tarball_by_tagname(current.name):
            # Tarball for the latest tag does not exist.
            # Add it to blacklist and try again.
            blacklist.append(current.name)
            return self.find_latest_matching(tagrefs, regex, blacklist)

        return current.name, current.tag.tagged_date