#!/usr/bin/python
# -*- coding: utf8 -*-

# Make sure tabs expand to 8 spaces in vim
# vim: expandtab

# Copyright 2002 Raphaël Hertzog
# Copyright 2005 Jeroen van Wolffelaar
# Copyright 2007-2008 Stefano Zacchiroli
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

import os.path, rfc822, sys, string, re, email, common, cPickle, syck
from xml.dom import implementation, ext

from config import dir, odir, root

# Create the source <-> binaries correspondances
source2binaries = {} # maps a source package to its binaries
binary2source = {}   # maps a binary package to its source
f = open(dir + "/sources.map", "r")
while 1:
    line = f.readline();
    if not line: break #eof
    line = line.strip()
    (binary, source) = line.split(None, 1)
    if not source2binaries.has_key(source):
        source2binaries[source] = []
    source2binaries[source].append(binary)
    binary2source[binary] = source
f.close()

def read_shortdesc(fname):
    global binary2source
    source_descs = {} # source package -> (binary package -> short description)
    for line in open(fname):
        pkg, shortdesc = line.strip().split("\t", 1)
        if not binary2source.has_key(pkg):
            continue
        src = binary2source[pkg]
        if not source_descs.has_key(src):
            source_descs[src] = {}
        source_descs[src][pkg] = shortdesc
    return source_descs

def read_bug_summary(fname):
    global binary2source
    summary = {} # source package -> bug count
    for line in open(fname):
        pkg, count = line.split()
        if not binary2source.has_key(pkg):
            continue
        src = binary2source[pkg]
        if not summary.has_key(src):
            summary[src] = 0
        summary[src] += int(count)
    return summary

# Read all the bugs stats
bugs = {}
f = open(dir + "/bugs.txt")
while 1:
    line = f.readline()
    if not line: break #eof
    line = line.strip()
    (binary, stats) = line.split(None, 1)
    bugs[binary] = [ string.atoi(i) for i in stats.split() ]
f.close()

srcbugs = {}
f = open(dir + "/bugs-src.txt")
while 1:
    line = f.readline()
    if not line: break #eof
    line = line.strip()
    (pkg, stats) = line.split(":", 1)
    try:
        srcbugs[pkg] = [ string.atoi(i) for i in stats.replace("(", " ").replace(")", " ").split() ]
    except:
        sys.stderr.write("Failed to parse bugs-src.txt stats for %s: %s" % (pkg, stats))
f.close()

gift_bugs = read_bug_summary(os.path.join(dir, 'bugs.gift.txt'))
help_bugs = read_bug_summary(os.path.join(dir, 'bugs.help.txt'))

# Read all the PTS stats
pts = {}
f = open(dir + "/count.txt")
while 1:
    line = f.readline()
    if not line: break #eof
    line = line.strip()
    (binary, stats) = line.split(None, 1)
    pts[binary] = stats
f.close()

# Read the lisf of packages with debcheck problems
debcheck = {}
for dist in ("stable", "testing", "unstable"):
    debcheck[dist] = {}
    f = open(dir + "/debcheck-" + dist)
    while 1:
        line = f.readline()
        if not line: break #eof
        debcheck[dist][line.strip()] = 1
    f.close()

# Read the list of packages with override disparities
override = {}
for dist in ("unstable", "experimental"):
    override[dist] = {}
    f = open(dir + "/override-disparities." + dist)
    for line in f:
        if line[0] != '-':
            source = line.strip()[:-1]
            override[dist][source] = []
        else:
            override[dist][source].append(line.strip()[2:])
    f.close()

# Read the list of source packages with debconf templates
debconf = {}
# DISABLED until ddtp.debian.org is setup again
#f = open(dir + "/debconf-list")
#while 1:
#    line = f.readline()
#    if not line: break #eof
#    line = line.strip()
#    debconf[line] = 1;
#f.close()

# Read the current signature of other.xml files
sigs = {}
if os.path.exists(odir + "/other.sigs"):
    f = open(odir + "/other.sigs", "r")
    sigs = cPickle.load(f)
    f.close()

# Read the wnpp information. [PvR]
wnpp = {}
if os.path.exists(dir + "/wnpp_rm"):
    f = open(dir + "/wnpp_rm")
    while 1:
        line = f.readline()
        if not line: break # eof
        line = line.strip()
        try:
            (package, type, number) = line.split("|")[0].split()
        except:
            #too many badly formatted ITP... disable warning. --RH
            #sys.stderr.write("Ignoring bad line '%s' in wnpp_rm\n" % line)
            pass
        wnpp[package[:-1]] = (type, number)
    f.close()

# Read patches information [FG]
ubuntu_patches = {}
# this can be easily inserted into a new update_patches.py if it becomes too
# heavy to parse the file
if os.path.exists(dir + "/patches.ubuntu"):
    f = open(dir + "/patches.ubuntu")
    for line in f.readlines():
        (package, rel_url) = line.split(' ', 2)
        rel_url = rel_url.strip()
        r = re.search("_(\S+).patch", line)
        if not r:
            continue
        version = r.group(1)
        ubuntu_patches[package] = (version, "http://patches.ubuntu.com/" + rel_url)
    f.close()

ubuntu_versions = {}
if os.path.exists(dir + "/versions.ubuntu"):
    f = open(dir + "/versions.ubuntu")
    for line in f.readlines():
        (package, version) = line.split(' ', 2)
        version = version.strip()
	ubuntu_versions[package] = (version, "https://launchpad.net/ubuntu/+source/" + package)
    f.close()

ubuntu_bugs = {}
if os.path.exists(dir + "/bugs.ubuntu"):
    f = open(dir + "/bugs.ubuntu")
    for line in f.readlines():
        (package, count) = line.split('|', 2)
        count = count.strip()
	ubuntu_bugs[package] = (count, "https://bugs.launchpad.net/ubuntu/+source/" + package)
    f.close()

# read low threshold NMU infos
def read_low_threshold_nmu(fname):
    emails = []
    if os.path.exists(fname):
        f = open(fname)
        devel_php_RE = \
            re.compile(r'http://qa\.debian\.org/developer\.php\?login=([^\s&]+)')
        word_RE = re.compile(r'^\w+$')
        for line in f.readlines():
            match = devel_php_RE.search(line)
            while match:    # look for several matches on the same line
                email = None
                login = match.group(1)
                if word_RE.match(login):
                    email = login + '@debian.org'
                elif login.find('@') >= 0:
                    email = login
                if email:
                    emails.append(email)
                line = line[match.end():]
                match = devel_php_RE.search(line)
        f.close()
    return emails

# write lowThresholdNmu info to a (global, i.e. not per-package) file
#
# XXX this is sub-optimal, as the XSLT rendering of each package page will have
# to read the whole XML file each time. However, this can't be fixed here
# unless we have somewhere an additional map (in the spirit of sources.map)
# mapping source package names to maintainer emails
low_nmu_emails = read_low_threshold_nmu(dir + "/low_threshold_nmu.txt")
f = open(odir + "/low_threshold_nmu.emails.xml", 'w')
f.write("<emails>\n");
f.writelines(map(lambda s: "  <email>%s</email>\n" % s, low_nmu_emails))
f.write("</emails>\n");
f.close()

def read_transitions(fname):
    f = open(fname)
    markup = f.read()
    f.close()
    yaml = syck.load(markup)
    packages = {} # maps pkg to the _list_ of transitions they are involved in
    for id, transition in yaml.iteritems():
        for pkg in transition['packages']:
            if not packages.has_key(pkg):
                packages[pkg] = []
            packages[pkg].append(id)
    return packages

# read the list of packages involved in transitions
transitions = read_transitions(os.path.join(dir, "transitions.yaml"))

def read_piuparts(fname):
    failures = {}
    if os.path.exists(fname):
        f = open(fname)
        for line in f.readlines():
            try:
                src, outcome, log = line.rstrip().split(None, 2)
                if outcome == "fail":
                    failures[src] = log
            except ValueError:
                pass
        f.close()
    return failures

piuparts = read_piuparts(os.path.join(dir, "piuparts.txt"))

def read_lintian_info(fname):
    lintian = {} # maps source pkg names to pairs <errors_no, warnings_no>
    for line in open(fname).readlines():
        try:
            (pkg, errors_no, warnings_no) = line.split()
            lintian[pkg] = (int(errors_no), int(warnings_no))
        except ValueError:
            continue
    return lintian

# read QA lintian info
lintian = read_lintian_info(os.path.join(dir, "lintian.qa-list.txt"))

# read the list of packages indexed by svnbuildstat.debian.net
svnbuildstat = {}
f = open(os.path.join(dir, "svnbuildstat_list.txt"))
for pkgname in map(string.rstrip, f.readlines()):
    svnbuildstat[pkgname] = True
f.close()

# DEHS textual file are line oriented with lines like "field: value"
def read_dehs(fname):
    f = open(fname)
    for line in f.readlines():
        yield map(string.strip, line.split(':'))
    f.close()

# read info gathered from dehs.alioth.debian.org
dehs = {}
for pkgname, version in read_dehs(os.path.join(dir, "dehs_out_of_date.txt")):
    if not dehs.has_key(pkgname):
        dehs[pkgname] = {}
    dehs[pkgname]['newer'] = version
for pkgname, msg in read_dehs(os.path.join(dir, "dehs_error.txt")):
    if not dehs.has_key(pkgname):
        dehs[pkgname] = {}
    dehs[pkgname]['error'] = msg

# read short descriptions
shortdescs = read_shortdesc(os.path.join(dir, "shortdesc.txt"))

# Create the XML documents
while 1:
    line = sys.stdin.readline()
    if not line: break #eof
    pkg = line.strip()

    doc = implementation.createDocument(None, "other", None)
    root_elt = doc.documentElement
    
    hash = pkg[0]
    if pkg[0:3] == "lib":
        hash = pkg[0:4]

    # Add debcheck availability info
    dc_sig = ""
    elt = doc.createElement("debcheck")
    for dist in ("stable", "testing", "unstable"):
        if debcheck[dist].has_key(pkg):
            elt.setAttribute(dist, "yes")
            dc_sig += "y"
        else:
            elt.setAttribute(dist, "no")
            dc_sig += "n"
    root_elt.appendChild(elt)
    
    # Add debconf templates availibilty info
    if debconf.has_key(pkg):
        root_elt.setAttribute("debconf", "yes")
        dc_sig += "y"
    else:
        root_elt.setAttribute("debconf", "no")
        dc_sig += "n"
	
    # Get PTS stats
    elt = doc.createElement("pts")
    elt.setAttribute("count", pts.get(pkg, "0"))
    root_elt.appendChild(elt)

    # Get BTS stats
    elt = doc.createElement("bugs")
    (s_rc, s_rc_m, s_normal, s_normal_m, s_wishlist, s_wishlist_m, s_fixed,
            s_fixed_m, s_patch, s_patch_m) = \
                    srcbugs.get(pkg, [0,0,0,0,0,0,0,0,0,0])
    binlist = source2binaries.get(pkg, [])
    binlist.sort()
    subsig = ""
    for binary in binlist:
        sub_elt = doc.createElement("item")
        sub_elt.setAttribute("name", binary)
        (rc, normal, wishlist, fixed, patch) = bugs.get(binary, [0,0,0,0,0])
        sub_elt.setAttribute("rc", "%d" % rc)
        sub_elt.setAttribute("normal", "%d" % normal)
        sub_elt.setAttribute("wishlist", "%d" % wishlist)
        sub_elt.setAttribute("fixed", "%d" % fixed)
        sub_elt.setAttribute("patch", "%d" % patch)
        all = rc + normal + wishlist + fixed
        sub_elt.setAttribute("all", "%d" % all)
        elt.appendChild(sub_elt)
        if len(subsig):
            subsig = "%s|%d|%d" % (subsig, all, patch)
        else:
            subsig = "%d|%d" % (all, patch)
      
    elt.setAttribute("rc", "%d" % s_rc)
    if s_rc != s_rc_m:
        elt.setAttribute("rc_m", "%d" % s_rc_m)
    elt.setAttribute("normal", "%d" % s_normal)
    if s_normal != s_normal_m:
        elt.setAttribute("normal_m", "%d" % s_normal_m)
    elt.setAttribute("wishlist", "%d" % s_wishlist)
    if s_wishlist != s_wishlist_m:
        elt.setAttribute("wishlist_m", "%d" % s_wishlist_m)
    elt.setAttribute("fixed", "%d" % s_fixed)
    if s_fixed != s_fixed_m:
        elt.setAttribute("fixed_m", "%d" % s_fixed_m)
    elt.setAttribute("patch", "%d" % s_patch)
    if s_patch != s_patch_m:
        elt.setAttribute("patch_m", "%d" % s_patch_m)
    s_all = s_fixed + s_wishlist + s_normal + s_rc
    s_all_m = s_fixed_m + s_wishlist_m + s_normal_m + s_rc_m
    elt.setAttribute("all", "%d" % s_all)
    if s_all != s_all_m:
        elt.setAttribute("all_m", "%d" % s_all_m)
    root_elt.appendChild(elt)
    if gift_bugs.has_key(pkg):
        s_gift = gift_bugs[pkg]
    else:
        s_gift = 0
    elt.setAttribute("gift", str(s_gift))
    if help_bugs.has_key(pkg):
        s_help = help_bugs[pkg]
    else:
        s_help = 0
    elt.setAttribute("help", str(s_help))

    # Get WNPP information. [PvR]
    if wnpp.has_key(pkg):
        (type, number) = wnpp[pkg]
        elt = doc.createElement("wnpp")
        elt.setAttribute("type", type)
        elt.setAttribute("bugnumber", number)
        root_elt.appendChild(elt)
        root_elt.setAttribute("wnpp", "yes")
        wnpp_sig = "%s%s" % (type,number)
    else:
        root_elt.setAttribute("wnpp", "no")
        wnpp_sig = "n"
    
    # Get override info [JvW]
    override_elt = None
    override_sig = []
    for dist in [ 'unstable', 'experimental' ]:
        if override[dist].has_key(pkg):
            if not override_elt: override_elt = doc.createElement("override")
            disparities = override[dist][pkg]
            override_sig.append(disparities)
            elt_g = doc.createElement("group")
            elt_g.setAttribute("suite", dist)
            for disp in disparities:
                elt = doc.createTextNode(disp)
                elt_disp = doc.createElement("disparity")
                elt_disp.appendChild(elt)
                elt_g.appendChild(elt_disp)
            override_elt.appendChild(elt_g)

    if override_elt:
        root_elt.appendChild(override_elt)
        root_elt.setAttribute("override", "yes")
    else:
        root_elt.setAttribute("override", "no")

    # Add Ubuntu information
    if ubuntu_versions.has_key(pkg):
        elt = doc.createElement("ubuntu")
        (version, url) = ubuntu_versions[pkg]
	ubuntu_sig = "ubuntu/ " + version
        elt.setAttribute("version", unicode(version, 'UTF8', 'replace'))
        elt.setAttribute("url", unicode(url, 'UTF8', 'replace'))
    	if ubuntu_bugs.has_key(pkg):
           elt.setAttribute("bugs", "yes")
           elt_bugs = doc.createElement("bugs")
           (count, url) = ubuntu_bugs[pkg]
           ubuntu_sig += "ubuntubugs/" + count
           elt_bugs.setAttribute("count", unicode(count, 'UTF8', 'replace'))
           elt_bugs.setAttribute("url", unicode(url, 'UTF8', 'replace'))
           elt.appendChild(elt_bugs)
	if ubuntu_patches.has_key(pkg):
           elt.setAttribute("patch", "yes")
           elt_patch = doc.createElement("patch")
           (version, url) = ubuntu_patches[pkg]
           ubuntu_sig += "ubuntupatch/" + version
           elt_patch.setAttribute("version", unicode(version, 'UTF8', 'replace'))
           elt_patch.setAttribute("url", unicode(url, 'UTF8', 'replace'))
           elt.appendChild(elt_patch)
        root_elt.appendChild(elt)
        root_elt.setAttribute("ubuntu", "yes")
    else:
        root_elt.setAttribute("ubuntu", "no")
        ubuntu_sig = "n"

    # Get DEHS information
    if dehs.has_key(pkg):
        elt = doc.createElement('dehs')
        root_elt.appendChild(elt)
        root_elt.setAttribute('dehs', 'yes')
        if dehs[pkg].has_key('newer'):
            elt.setAttribute('newer', dehs[pkg]['newer'])
        if dehs[pkg].has_key('error'):
            elt.setAttribute('error', 'yes')
        dehs_sig = 'y'
    else:
        root_elt.setAttribute('dehs', 'no')
        dehs_sig = 'n'

    # add svnbuildstat info
    if svnbuildstat.has_key(pkg):
        elt = doc.createElement("svnbuildstat")
        root_elt.setAttribute("svnbuildstat", "yes")
        root_elt.appendChild(elt)
        svnbuildstat_sig = "y"
    else:
        root_elt.setAttribute("svnbuildstat", "no")
        svnbuildstat_sig = "n"

    # add piuparts info
    if piuparts.has_key(pkg):
        #elt = doc.createElement("piuparts")
        #root_elt.appendChild(elt)
        root_elt.setAttribute("piuparts", "yes")
        piuparts_sig = "y"
    else:
        root_elt.setAttribute("piuparts", "no")
        piuparts_sig = "n"

    # add transitions info
    if transitions.has_key(pkg):
        elt = doc.createElement("transitions")
        for id in transitions[pkg]:
            trans_elt = doc.createElement("transition")
            trans_elt.setAttribute("name", id)
            elt.appendChild(trans_elt)
        root_elt.setAttribute("transitions", "yes")
        root_elt.appendChild(elt)
        transitions_sig = "y"
    else:
        root_elt.setAttribute("transitions", "no")
        transitions_sig = "n"

    # add lintian QA info
    if lintian.has_key(pkg):
        (errs, warns) = lintian[pkg]
        elt = doc.createElement("lintian")
        elt.setAttribute("errors", str(errs))
        elt.setAttribute("warnings", str(warns))
        root_elt.appendChild(elt)
        root_elt.setAttribute("lintian", "yes")
        lintian_sig = (errs, warns)
    else:
        root_elt.setAttribute("lintian", "no")
        lintian_sig = (0, 0)

    # add short descriptions
    elt = doc.createElement("descriptions")
    root_elt.appendChild(elt)
    if shortdescs.has_key(pkg):
        for package, shortdesc in shortdescs[pkg].iteritems():
            desc_elt = doc.createElement("shortdesc")
            elt.appendChild(desc_elt)
            desc_elt.setAttribute("package", package)
            desc_text = doc.createTextNode(shortdesc)
            desc_elt.appendChild(desc_text)
        shortdesc_sig = str(shortdescs[pkg]).__hash__()
            # XXX hash(str(...)) does not work: WTF?
    else:
        shortdesc_sig = ''.__hash__()

    # TODO: try to do that signature checking before the creation of XML DOM
    # Build the sig and check if anything changed
    sig = (pts.get(pkg, "0"), dc_sig, wnpp_sig, override_sig, dehs_sig,
            ubuntu_sig, s_rc, s_normal, s_wishlist, s_fixed, s_gift, s_help,
            subsig, svnbuildstat_sig, transitions_sig, lintian_sig,
            shortdesc_sig, piuparts_sig)
    if sigs.has_key(pkg) and sig == sigs[pkg] and \
            os.path.isfile("%s/%s/%s/other.xml" % (odir, hash, pkg)):
        continue
    sigs[pkg] = sig

    # Output the data to the XML file
    try:
        f = open("%s/%s/%s/other.xml" % (odir, hash, pkg), "w")
        ext.PrettyPrint(doc, f, "utf-8")
        f.close()
    except Exception, msg:
        sys.stderr.write("Output problem for " + pkg + "/other.xml (%s)\n" %
                msg);

# Store the signatures
f = open(odir + "/other.sigs", "w")
cPickle.dump(sigs, f, 0)
f.close()