#!/usr/bin/python
# -*- coding: utf-8 -*-

# Make sure tabs expand to 8 spaces in vim
# vim: expandtab

# Copyright: © 2002-2011 Raphaël Hertzog
# Copyright: © 2005 Jeroen van Wolffelaar
# Copyright: © 2007-2009 Stefano Zacchiroli
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

import os.path, sys, string, re, email, common, cPickle, yaml
import xml.dom

try:
    from debian import deb822, debian_support
except: # Required to run on lenny
    from debian_bundle import deb822, debian_support

from config import dir, odir, root
from common import hash_name

def read_font_reviews(fname):
    y = yaml.load(file(fname))
    packages = {} # maps pkg to the font review for it
    for pkg, data in y.iteritems():
        if data.has_key('url'):
            packages[pkg] = data['url']
    return packages

def read_shortdesc(fname):
    global binary2sources
    source_descs = {} # source package -> (binary package -> short description)
    for line in open(fname):
        pkg, shortdesc = line.strip().split("\t", 1)
        if not binary2sources.has_key(pkg):
            continue
        src = binary2sources[pkg]['current']
        if src is None:
            continue
        if not source_descs.has_key(src):
            source_descs[src] = {}
        source_descs[src][pkg] = shortdesc
    return source_descs

def read_bug_summary(fname):
    global binary2sources
    summary = {} # source package -> bug count
    for line in open(fname):
        pkg, count = line.split()
        if not binary2sources.has_key(pkg):
            continue
        src = binary2sources[pkg]['current']
        if src is None:
            continue
        if not summary.has_key(src):
            summary[src] = 0
        summary[src] += int(count)
    return summary

def read_low_threshold_nmu(fname):
    """read low threshold NMU info"""
    emails = []
    if os.path.exists(fname):
        f = open(fname)
        devel_php_RE = \
            re.compile(r'http://qa\.debian\.org/developer\.php\?login=([^\s&]+)')
        word_RE = re.compile(r'^\w+$')
        for line in f.readlines():
            match = devel_php_RE.search(line)
            while match:    # look for several matches on the same line
                email = None
                login = match.group(1)
                if word_RE.match(login):
                    email = login + '@debian.org'
                elif login.find('@') >= 0:
                    email = login
                if email:
                    emails.append(email)
                line = line[match.end():]
                match = devel_php_RE.search(line)
        f.close()
    return emails

def read_transitions(fname):
    y = yaml.load(file(fname))
    packages = {} # maps pkg to the _list_ of transitions they are involved in
    for id, transition in y.iteritems():
        for pkg in transition['packages']:
            if not packages.has_key(pkg):
                packages[pkg] = []
            packages[pkg].append(id)
    return packages

def read_piuparts(fname):
    failures = {}
    if os.path.exists(fname):
        f = open(fname)
        for line in f.readlines():
            try:
                src, outcome = map(lambda s: s.strip(), line.split(':'))
                if outcome == "fail":
                    failures[src] = True
            except ValueError:
                pass
        f.close()
    return failures

def read_lintian_info(fname):
    lintian = {} # maps source pkg names to pairs <errors_no, warnings_no>
    for line in open(fname).readlines():
        try:
            (pkg, errors_no, warnings_no) = line.split()[:3]
            lintian[pkg] = (int(errors_no), int(warnings_no))
        except ValueError:
            continue
    return lintian

# DEHS textual file are line oriented with lines like "field: value"
def read_dehs(fname):
    f = open(fname)
    for line in f.readlines():
	items = map(string.strip, line.split(':'))
	if len(items) == 2:
	    yield items
	else:
	    sys.stderr.write("Incorrect DEHS data: %s" % line)
    f.close()

# read 822 dump of NEW queue content
def read_NEW(fname):
    new_contents = {}
    for stanza in deb822.Sources.iter_paragraphs(file(os.path.join(dir, 'new.822'))):
        if stanza.has_key('source') and stanza.has_key('version') and \
                stanza.has_key('queue'):
            # store only the most recent version in NEW (ignore accepted)
            if stanza['queue'] == 'new' and \
                    (not new_contents.has_key(stanza['source']) or \
                     debian_support.version_compare( \
                                    new_contents[stanza['source']],
                                    stanza['version']) < 0):
                new_contents[stanza['source']] = stanza['version']
    return new_contents

def read_l10n_status(fname):
    l10n = {}
    if os.path.exists(fname):
        f = open(fname)
        for line in f.readlines():
            if line.startswith('#'):
                continue
            pkg, version, trans, url, todo = line.rstrip().split()
            deb_trans, nondeb_trans = trans.strip('()').split(',')
            l10n[pkg] = {'version': version,
                         'deb': deb_trans, 'nondeb': nondeb_trans,
                         'url': url, 'todo': todo}
        f.close()
    return l10n

# Initialization: fill binary <-> source maps
f = open(odir + "/sources_mapping")
source2binaries = cPickle.load(f) # maps a source package to its binaries
binary2sources = cPickle.load(f)  # maps a binary package to its sources
f.close()

# Read all the bugs stats
bugs = {}
f = open(dir + "/bugs.txt")
while 1:
    line = f.readline()
    if not line: break #eof
    line = line.strip()
    (binary, stats) = line.split(None, 1)
    try:
        bugs[binary] = [ string.atoi(i) for i in stats.split() ]
    except:
        sys.stderr.write("Failed to parse bugs.txt stats for %s: %s\n" % (binary, stats))
f.close()

srcbugs = {}
f = open(dir + "/bugs-src.txt")
while 1:
    line = f.readline()
    if not line: break #eof
    line = line.strip()
    (pkg, stats) = line.split(":", 1)
    try:
        srcbugs[pkg] = [ string.atoi(i) for i in stats.replace("(", " ").replace(")", " ").split() ]
    except:
        sys.stderr.write("Failed to parse bugs-src.txt stats for %s: %s\n" % (pkg, stats))
f.close()

gift_bugs = read_bug_summary(os.path.join(dir, 'bugs.gift.txt'))
help_bugs = read_bug_summary(os.path.join(dir, 'bugs.help.txt'))

# Read all the PTS stats
pts = {}
f = open(dir + "/count.txt")
while 1:
    line = f.readline()
    if not line: break #eof
    line = line.strip()
    (binary, stats) = line.split(None, 1)
    pts[binary] = stats
f.close()

# Read the lisf of packages with debcheck problems
debcheck = {}
for dist in ("oldstable", "stable", "testing", "unstable"):
    debcheck[dist] = {}
    f = open(dir + "/debcheck-" + dist)
    while 1:
        line = f.readline()
        if not line: break #eof
        debcheck[dist][line.strip()] = 1
    f.close()

# Read the list of packages with override disparities
override = {}
for dist in ("unstable", "experimental"):
    override[dist] = {}
    f = open(dir + "/override-disparities." + dist)
    for line in f:
        if line[0] != '-':
            source = line.strip()[:-1]
            override[dist][source] = []
        else:
            override[dist][source].append(line.strip()[2:])
    f.close()

# read the package localization status
l10n = read_l10n_status(os.path.join(dir, 'l10n-status.txt'))

# Read the current signature of other.xml files
sigs = {}
if os.path.exists(odir + "/other.sigs"):
    f = open(odir + "/other.sigs", "r")
    sigs = cPickle.load(f)
    f.close()

# Read the wnpp information. [PvR]
wnpp = {}
if os.path.exists(dir + "/wnpp_rm"):
    f = open(dir + "/wnpp_rm")
    while 1:
        line = f.readline()
        if not line: break # eof
        line = line.strip()
        try:
            (package, type, number) = line.split("|")[0].split()
        except:
            #too many badly formatted ITP... disable warning. --RH
            #sys.stderr.write("Ignoring bad line '%s' in wnpp_rm\n" % line)
            pass
        wnpp[package[:-1]] = (type, number)
    f.close()

# Read patches information [FG]
ubuntu_patches = {}
# this can be easily inserted into a new update_patches.py if it becomes too
# heavy to parse the file
if os.path.exists(dir + "/patches.ubuntu"):
    f = open(dir + "/patches.ubuntu")
    for line in f.readlines():
        (package, rel_url) = line.split(' ', 2)
        rel_url = rel_url.strip()
        r = re.search("_(\S+).patch", line)
        if not r:
            continue
        version = r.group(1)
        ubuntu_patches[package] = (version, "http://patches.ubuntu.com/" + rel_url)
    f.close()

ubuntu_versions = {}
if os.path.exists(dir + "/versions.ubuntu"):
    f = open(dir + "/versions.ubuntu")
    for line in f.readlines():
        (package, version) = line.split(' ', 2)
        version = version.strip()
	ubuntu_versions[package] = (version, "https://launchpad.net/ubuntu/+source/" + package)
    f.close()

ubuntu_bugs = {}
ubuntu_bugpatches = {}
if os.path.exists(dir + "/bugs.ubuntu"):
    f = open(dir + "/bugs.ubuntu")
    for line in f.readlines():
        (package, ubugs, upatches) = line.split('|', 3)
        ubugs = ubugs.strip()
        upatches = upatches.strip()
	ubuntu_bugs[package] = (ubugs, "https://bugs.launchpad.net/ubuntu/+source/" + package)
	ubuntu_bugpatches[package] = (upatches, "https://bugs.launchpad.net/ubuntu/+source/" + package + "/+patches")
    f.close()

# write lowThresholdNmu info to a (global, i.e. not per-package) file
#
# XXX this is sub-optimal, as the XSLT rendering of each package page will have
# to read the whole XML file each time. However, this can't be fixed here
# unless we have somewhere an additional map (in the spirit of sources.map)
# mapping source package names to maintainer emails
low_nmu_emails = read_low_threshold_nmu(dir + "/low_threshold_nmu.txt")
f = open(odir + "/low_threshold_nmu.emails.xml", 'w')
f.write("<emails>\n");
f.writelines(map(lambda s: "  <email>%s</email>\n" % s, low_nmu_emails))
f.write("</emails>\n");
f.close()

# read the list of packages involved in transitions
transitions = read_transitions(os.path.join(dir, "transitions.yaml"))

piuparts = read_piuparts(os.path.join(dir, "piuparts-sid.txt"))

new_queue = read_NEW(os.path.join(dir, "new.822"))

# read QA lintian info
lintian = read_lintian_info(os.path.join(dir, "lintian.qa-list.txt"))

# read the list of packages indexed by svnbuildstat.debian.net
svnbuildstat = {}
try:
    f = open(os.path.join(dir, "svnbuildstat_list.txt"))
    for pkgname in map(string.rstrip, f.readlines()):
        svnbuildstat[pkgname] = True
    f.close()
except:
    pass # Silent failure

# read info gathered from dehs.alioth.debian.org
dehs = {}
for pkgname, version in read_dehs(os.path.join(dir, "dehs_out_of_date.txt")):
    if not dehs.has_key(pkgname):
        dehs[pkgname] = {}
    dehs[pkgname]['newer'] = version
for pkgname, msg in read_dehs(os.path.join(dir, "dehs_error.txt")):
    if not dehs.has_key(pkgname):
        dehs[pkgname] = {}
    dehs[pkgname]['error'] = msg

# read list of unfixed security issues
security = {}
for pkgname, count in read_dehs(os.path.join(dir, "security_issues.txt")):
    security[pkgname] = count

# read short descriptions
shortdescs = read_shortdesc(os.path.join(dir, "shortdesc.txt"))

font_reviews = read_font_reviews(os.path.join(dir, "debian-font-review.yaml"))

# Create the XML documents
while 1:
    line = sys.stdin.readline()
    if not line: break #eof
    pkg = line.strip()

    doc = xml.dom.getDOMImplementation('minidom').createDocument(None, "other", None)
    root_elt = doc.documentElement
    hash = hash_name(pkg)

    # Add debcheck availability info
    dc_sig = ""
    elt = doc.createElement("debcheck")
    for dist in ("oldstable", "stable", "testing", "unstable"):
        if debcheck[dist].has_key(pkg):
            elt.setAttribute(dist, "yes")
            dc_sig += "y"
        else:
            elt.setAttribute(dist, "no")
            dc_sig += "n"
    root_elt.appendChild(elt)

    # Add NEW queue versions, if any
    if new_queue.has_key(pkg):
        root_elt.setAttribute("new_version", new_queue[pkg])
        new_queue_sig = 'y'
    else:
        new_queue_sig = 'n'
    
    # Get PTS stats
    elt = doc.createElement("pts")
    elt.setAttribute("count", pts.get(pkg, "0"))
    root_elt.appendChild(elt)

    # Get BTS stats
    elt = doc.createElement("bugs")
    (s_rc, s_rc_m, s_normal, s_normal_m, s_wishlist, s_wishlist_m, s_fixed,
            s_fixed_m, s_patch, s_patch_m) = \
                    srcbugs.get(pkg, [0,0,0,0,0,0,0,0,0,0])
    try:
        binlist = source2binaries[pkg]['merged']
    except:
        binlist = []
    binlist.sort()
    subsig = ""
    for binary in binlist:
        sub_elt = doc.createElement("item")
        sub_elt.setAttribute("name", binary)
        (rc, normal, wishlist, fixed, patch) = bugs.get(binary, [0,0,0,0,0])
        sub_elt.setAttribute("rc", "%d" % rc)
        sub_elt.setAttribute("normal", "%d" % normal)
        sub_elt.setAttribute("wishlist", "%d" % wishlist)
        sub_elt.setAttribute("fixed", "%d" % fixed)
        sub_elt.setAttribute("patch", "%d" % patch)
        all = rc + normal + wishlist + fixed
        sub_elt.setAttribute("all", "%d" % all)
        elt.appendChild(sub_elt)
        if len(subsig):
            subsig = "%s|%d|%d" % (subsig, all, patch)
        else:
            subsig = "%d|%d" % (all, patch)
      
    elt.setAttribute("rc", "%d" % s_rc)
    if s_rc != s_rc_m:
        elt.setAttribute("rc_m", "%d" % s_rc_m)
    elt.setAttribute("normal", "%d" % s_normal)
    if s_normal != s_normal_m:
        elt.setAttribute("normal_m", "%d" % s_normal_m)
    elt.setAttribute("wishlist", "%d" % s_wishlist)
    if s_wishlist != s_wishlist_m:
        elt.setAttribute("wishlist_m", "%d" % s_wishlist_m)
    elt.setAttribute("fixed", "%d" % s_fixed)
    if s_fixed != s_fixed_m:
        elt.setAttribute("fixed_m", "%d" % s_fixed_m)
    elt.setAttribute("patch", "%d" % s_patch)
    if s_patch != s_patch_m:
        elt.setAttribute("patch_m", "%d" % s_patch_m)
    s_all = s_fixed + s_wishlist + s_normal + s_rc
    s_all_m = s_fixed_m + s_wishlist_m + s_normal_m + s_rc_m
    elt.setAttribute("all", "%d" % s_all)
    if s_all != s_all_m:
        elt.setAttribute("all_m", "%d" % s_all_m)
    root_elt.appendChild(elt)
    if gift_bugs.has_key(pkg):
        s_gift = gift_bugs[pkg]
    else:
        s_gift = 0
    elt.setAttribute("gift", str(s_gift))
    if help_bugs.has_key(pkg):
        s_help = help_bugs[pkg]
    else:
        s_help = 0
    elt.setAttribute("help", str(s_help))

    # Get WNPP information. [PvR]
    if wnpp.has_key(pkg):
        (type, number) = wnpp[pkg]
        elt = doc.createElement("wnpp")
        elt.setAttribute("type", type)
        elt.setAttribute("bugnumber", number)
        root_elt.appendChild(elt)
        root_elt.setAttribute("wnpp", "yes")
        wnpp_sig = "%s%s" % (type,number)
    else:
        root_elt.setAttribute("wnpp", "no")
        wnpp_sig = "n"
    
    # Get override info [JvW]
    override_elt = None
    override_sig = []
    for dist in [ 'unstable', 'experimental' ]:
        if override[dist].has_key(pkg):
            if not override_elt: override_elt = doc.createElement("override")
            disparities = override[dist][pkg]
            override_sig.append(disparities)
            elt_g = doc.createElement("group")
            elt_g.setAttribute("suite", dist)
            for disp in disparities:
                elt = doc.createTextNode(disp)
                elt_disp = doc.createElement("disparity")
                elt_disp.appendChild(elt)
                elt_g.appendChild(elt_disp)
            override_elt.appendChild(elt_g)

    if override_elt:
        root_elt.appendChild(override_elt)
        root_elt.setAttribute("override", "yes")
    else:
        root_elt.setAttribute("override", "no")

    # Add Ubuntu information
    if ubuntu_versions.has_key(pkg):
        elt = doc.createElement("ubuntu")
        (version, url) = ubuntu_versions[pkg]
	ubuntu_sig = "ubuntu/ " + version
        elt.setAttribute("version", unicode(version, 'UTF8', 'replace'))
        elt.setAttribute("url", unicode(url, 'UTF8', 'replace'))
    	if ubuntu_bugs.has_key(pkg):
           elt.setAttribute("bugs", "yes")
           elt_bugs = doc.createElement("bugs")
           (count, url) = ubuntu_bugs[pkg]
           ubuntu_sig += "ubuntubugs/" + count
           elt_bugs.setAttribute("count", unicode(count, 'UTF8', 'replace'))
           elt_bugs.setAttribute("url", unicode(url, 'UTF8', 'replace'))
           elt.appendChild(elt_bugs)
       	   if ubuntu_bugpatches[pkg][0] != "0":
              elt.setAttribute("bugpatches", "yes")
              elt_bugpatches = doc.createElement("bugpatches")
              (count, url) = ubuntu_bugpatches[pkg]
              ubuntu_sig += "ubuntubugpatches/" + count
              elt_bugpatches.setAttribute("count", unicode(count, 'UTF8', 'replace'))
              elt_bugpatches.setAttribute("url", unicode(url, 'UTF8', 'replace'))
              elt.appendChild(elt_bugpatches)
	if ubuntu_patches.has_key(pkg):
           elt.setAttribute("patch", "yes")
           elt_patch = doc.createElement("patch")
           (version, url) = ubuntu_patches[pkg]
           ubuntu_sig += "ubuntupatch/" + version
           elt_patch.setAttribute("version", unicode(version, 'UTF8', 'replace'))
           elt_patch.setAttribute("url", unicode(url, 'UTF8', 'replace'))
           elt.appendChild(elt_patch)
        root_elt.appendChild(elt)
        root_elt.setAttribute("ubuntu", "yes")
    else:
        root_elt.setAttribute("ubuntu", "no")
        ubuntu_sig = "n"

    # Get DEHS information
    if dehs.has_key(pkg):
        elt = doc.createElement('dehs')
        root_elt.appendChild(elt)
        root_elt.setAttribute('dehs', 'yes')
        if dehs[pkg].has_key('newer'):
            elt.setAttribute('newer', dehs[pkg]['newer'])
        if dehs[pkg].has_key('error'):
            elt.setAttribute('error', 'yes')
        dehs_sig = 'y'
    else:
        root_elt.setAttribute('dehs', 'no')
        dehs_sig = 'n'

    # add svnbuildstat info
    if svnbuildstat.has_key(pkg):
        elt = doc.createElement("svnbuildstat")
        root_elt.setAttribute("svnbuildstat", "yes")
        root_elt.appendChild(elt)
        svnbuildstat_sig = "y"
    else:
        root_elt.setAttribute("svnbuildstat", "no")
        svnbuildstat_sig = "n"

    # add piuparts info
    if piuparts.has_key(pkg):
        #elt = doc.createElement("piuparts")
        #root_elt.appendChild(elt)
        root_elt.setAttribute("piuparts", "yes")
        piuparts_sig = "y"
    else:
        root_elt.setAttribute("piuparts", "no")
        piuparts_sig = "n"

    # add localization info
    if l10n.has_key(pkg):
        elt = doc.createElement("i18n")
        deb, nondeb = l10n[pkg]['deb'], l10n[pkg]['nondeb']
        elt.setAttribute('deb', deb)
        elt.setAttribute('nondeb', nondeb)
        if not (set([deb, nondeb]) <= set(['-', '100'])):
            # do not show l10n status when fully translated or nothing
            # is translatable
            elt.setAttribute('href', l10n[pkg]['url'])
        if l10n[pkg]['todo'] == '1':
            elt.setAttribute('todo', 'yes')
        root_elt.setAttribute("i18n", "yes")
        root_elt.appendChild(elt)
        i18n_sig = (deb, nondeb, l10n[pkg]['todo'])
    else:
        root_elt.setAttribute("i18n", "no")
        i18n_sig = ('-','-', 0)
	
    # add transitions info
    if transitions.has_key(pkg):
        elt = doc.createElement("transitions")
        for id in transitions[pkg]:
            trans_elt = doc.createElement("transition")
            trans_elt.setAttribute("name", id)
            elt.appendChild(trans_elt)
        root_elt.setAttribute("transitions", "yes")
        root_elt.appendChild(elt)
        transitions_sig = "y"
    else:
        root_elt.setAttribute("transitions", "no")
        transitions_sig = "n"

    # add lintian QA info
    if lintian.has_key(pkg):
        (errs, warns) = lintian[pkg]
        elt = doc.createElement("lintian")
        elt.setAttribute("errors", str(errs))
        elt.setAttribute("warnings", str(warns))
        root_elt.appendChild(elt)
        root_elt.setAttribute("lintian", "yes")
        lintian_sig = (errs, warns)
    else:
        root_elt.setAttribute("lintian", "no")
        lintian_sig = (0, 0)

    # add short descriptions
    elt = doc.createElement("descriptions")
    root_elt.appendChild(elt)
    if shortdescs.has_key(pkg):
        for package, shortdesc in shortdescs[pkg].iteritems():
            desc_elt = doc.createElement("shortdesc")
            elt.appendChild(desc_elt)
            desc_elt.setAttribute("package", package)
            desc_text = doc.createTextNode(unicode(shortdesc, "UTF-8", "replace"))
            desc_elt.appendChild(desc_text)
        shortdesc_sig = str(shortdescs[pkg]).__hash__()
            # XXX hash(str(...)) does not work: WTF?
    else:
        shortdesc_sig = ''.__hash__()

    # Add font review links
    if font_reviews.has_key(pkg):
        elt = doc.createElement("fonts")
        elt.setAttribute('href', font_reviews[pkg])
        root_elt.setAttribute("fonts", "yes")
        root_elt.appendChild(elt)
        fonts_sig = font_reviews[pkg]
    else:
        root_elt.setAttribute("fonts", "no")
        fonts_sig = ''

    # Get security issues
    if security.has_key(pkg):
        root_elt.setAttribute('security', security[pkg])
        sec_sig = security[pkg]
    else:
        root_elt.setAttribute('security', '0')
        sec_sig = '0'

    # TODO: try to do that signature checking before the creation of XML DOM
    # Build the sig and check if anything changed
    sig = (pts.get(pkg, "0"), dc_sig, wnpp_sig, override_sig, dehs_sig,
            ubuntu_sig, s_rc, s_normal, s_wishlist, s_fixed, s_gift, s_help,
            subsig, svnbuildstat_sig, transitions_sig, lintian_sig,
            shortdesc_sig, piuparts_sig, new_queue_sig, i18n_sig,
            fonts_sig, sec_sig)
    if sigs.has_key(pkg) and sig == sigs[pkg] and \
            os.path.isfile("%s/%s/%s/other.xml" % (odir, hash, pkg)):
        continue
    sigs[pkg] = sig

    # Output the data to the XML file
    try:
        f = open("%s/%s/%s/other.xml" % (odir, hash, pkg), "w")
        f.write(doc.toxml(encoding="UTF-8"))
        f.close()
    except Exception, msg:
        sys.stderr.write("Output problem for " + pkg + "/other.xml (%s)\n" %
                msg);

# Store the signatures
f = open(odir + "/other.sigs", "w")
cPickle.dump(sigs, f, 0)
f.close()