| 1 |
jeroen |
1154 |
#!/usr/bin/python
|
| 2 |
|
|
# -*- coding: utf8 -*-
|
| 3 |
hertzog |
344 |
|
| 4 |
|
|
# Make sure tabs expand to 8 spaces in vim
|
| 5 |
|
|
# vim: expandtab
|
| 6 |
|
|
|
| 7 |
jeroen |
1154 |
# Copyright 2002 Raphaël Hertzog
|
| 8 |
jeroen |
947 |
# Copyright 2005 Jeroen van Wolffelaar
|
| 9 |
hertzog |
351 |
# This file is distributed under the terms of the General Public License
|
| 10 |
|
|
# version 2 or (at your option) any later version.
|
| 11 |
|
|
|
| 12 |
hertzog |
357 |
import os.path, rfc822, sys, string, re, email, common, cPickle
|
| 13 |
hertzog |
344 |
from xml.dom import implementation, ext
|
| 14 |
|
|
|
| 15 |
|
|
from config import dir, odir, root
|
| 16 |
|
|
|
| 17 |
|
|
# Create the source -> binaries correspondance
|
| 18 |
|
|
sources = {}
|
| 19 |
hertzog |
812 |
f = open(dir + "/sources.map", "r")
|
| 20 |
hertzog |
344 |
while 1:
|
| 21 |
|
|
line = f.readline();
|
| 22 |
|
|
if not line: break #eof
|
| 23 |
|
|
line = line.strip()
|
| 24 |
|
|
(binary, source) = line.split(None, 1)
|
| 25 |
|
|
if not sources.has_key(source):
|
| 26 |
|
|
sources[source] = []
|
| 27 |
|
|
sources[source].append(binary)
|
| 28 |
|
|
f.close()
|
| 29 |
|
|
|
| 30 |
|
|
# Read all the bugs stats
|
| 31 |
|
|
bugs = {}
|
| 32 |
|
|
f = open(dir + "/bugs.txt")
|
| 33 |
|
|
while 1:
|
| 34 |
|
|
line = f.readline()
|
| 35 |
|
|
if not line: break #eof
|
| 36 |
|
|
line = line.strip()
|
| 37 |
|
|
(binary, stats) = line.split(None, 1)
|
| 38 |
|
|
bugs[binary] = stats.split()
|
| 39 |
|
|
f.close()
|
| 40 |
|
|
|
| 41 |
|
|
# Read all the PTS stats
|
| 42 |
|
|
pts = {}
|
| 43 |
|
|
f = open(dir + "/count.txt")
|
| 44 |
|
|
while 1:
|
| 45 |
|
|
line = f.readline()
|
| 46 |
|
|
if not line: break #eof
|
| 47 |
|
|
line = line.strip()
|
| 48 |
|
|
(binary, stats) = line.split(None, 1)
|
| 49 |
|
|
pts[binary] = stats
|
| 50 |
|
|
f.close()
|
| 51 |
|
|
|
| 52 |
hertzog |
372 |
# Read the lisf of packages with debcheck problems
|
| 53 |
|
|
debcheck = {}
|
| 54 |
|
|
for dist in ("stable", "testing", "unstable"):
|
| 55 |
|
|
debcheck[dist] = {}
|
| 56 |
|
|
f = open(dir + "/debcheck-" + dist)
|
| 57 |
|
|
while 1:
|
| 58 |
|
|
line = f.readline()
|
| 59 |
|
|
if not line: break #eof
|
| 60 |
|
|
debcheck[dist][line.strip()] = 1
|
| 61 |
|
|
f.close()
|
| 62 |
|
|
|
| 63 |
jeroen |
947 |
# Read the list of packages with override disparities
|
| 64 |
|
|
override = {}
|
| 65 |
|
|
for dist in ("unstable", "experimental"):
|
| 66 |
|
|
override[dist] = {}
|
| 67 |
|
|
f = open(dir + "/override-disparities." + dist)
|
| 68 |
|
|
for line in f:
|
| 69 |
|
|
if line[0] != '-':
|
| 70 |
|
|
source = line.strip()[:-1]
|
| 71 |
|
|
override[dist][source] = []
|
| 72 |
|
|
else:
|
| 73 |
|
|
override[dist][source].append(line.strip()[2:])
|
| 74 |
|
|
f.close()
|
| 75 |
|
|
|
| 76 |
hertzog |
479 |
# Read the list of source packages with debconf templates
|
| 77 |
|
|
debconf = {}
|
| 78 |
hertzog |
692 |
# DISABLED until ddtp.debian.org is setup again
|
| 79 |
|
|
#f = open(dir + "/debconf-list")
|
| 80 |
|
|
#while 1:
|
| 81 |
|
|
# line = f.readline()
|
| 82 |
|
|
# if not line: break #eof
|
| 83 |
|
|
# line = line.strip()
|
| 84 |
|
|
# debconf[line] = 1;
|
| 85 |
|
|
#f.close()
|
| 86 |
hertzog |
479 |
|
| 87 |
hertzog |
357 |
# Read the current signature of other.xml files
|
| 88 |
|
|
sigs = {}
|
| 89 |
|
|
if os.path.exists(odir + "/other.sigs"):
|
| 90 |
|
|
f = open(odir + "/other.sigs", "r")
|
| 91 |
|
|
sigs = cPickle.load(f)
|
| 92 |
|
|
f.close()
|
| 93 |
|
|
|
| 94 |
hertzog |
560 |
# Read the wnpp information. [PvR]
|
| 95 |
|
|
wnpp = {}
|
| 96 |
jeroen |
912 |
if os.path.exists(dir + "/wnpp_rm"):
|
| 97 |
|
|
f = open(dir + "/wnpp_rm")
|
| 98 |
hertzog |
560 |
while 1:
|
| 99 |
|
|
line = f.readline()
|
| 100 |
|
|
if not line: break # eof
|
| 101 |
jeroen |
912 |
(package, type, number) = line.split("|")[0].split()
|
| 102 |
|
|
wnpp[package[:-1]] = (type, number)
|
| 103 |
hertzog |
560 |
f.close()
|
| 104 |
|
|
|
| 105 |
hertzog |
643 |
# Read watch information [FG]
|
| 106 |
|
|
watch = {}
|
| 107 |
|
|
if os.path.exists(odir + "/watch_done"):
|
| 108 |
|
|
f = open(odir + "/watch_done")
|
| 109 |
|
|
watch = cPickle.load(f)
|
| 110 |
|
|
f.close()
|
| 111 |
|
|
|
| 112 |
hertzog |
1049 |
# Read patches information [FG]
|
| 113 |
|
|
patches = {}
|
| 114 |
|
|
# this can be easily inserted into a new update_patches.py if it becomes too
|
| 115 |
|
|
# heavy to parse the file
|
| 116 |
|
|
if os.path.exists(dir + "/patches.ubuntu"):
|
| 117 |
|
|
f = open(dir + "/patches.ubuntu")
|
| 118 |
|
|
for line in f.readlines():
|
| 119 |
|
|
(package, rel_url) = line.split(' ', 2)
|
| 120 |
|
|
rel_url = rel_url.strip()
|
| 121 |
|
|
r = re.search("_(\S+).patch", line)
|
| 122 |
|
|
if not r:
|
| 123 |
|
|
continue
|
| 124 |
|
|
version = r.group(1)
|
| 125 |
|
|
patches[package] = {}
|
| 126 |
hertzog |
1355 |
patches[package]['ubuntu'] = (version, "http://patches.ubuntu.com/" + rel_url)
|
| 127 |
zack |
1749 |
f.close()
|
| 128 |
hertzog |
1049 |
|
| 129 |
zack |
1773 |
# read low threshold NMU infos
|
| 130 |
|
|
def read_low_threshold_nmu(fname):
|
| 131 |
|
|
emails = []
|
| 132 |
|
|
if os.path.exists(fname):
|
| 133 |
|
|
f = open(fname)
|
| 134 |
|
|
devel_php_RE = \
|
| 135 |
|
|
re.compile(r'http://qa\.debian\.org/developer\.php\?login=([^\s&]+)')
|
| 136 |
|
|
word_RE = re.compile(r'^\w+$')
|
| 137 |
|
|
for line in f.readlines():
|
| 138 |
|
|
match = devel_php_RE.search(line)
|
| 139 |
|
|
while match: # look for several matches on the same line
|
| 140 |
|
|
email = None
|
| 141 |
|
|
login = match.group(1)
|
| 142 |
|
|
if word_RE.match(login):
|
| 143 |
|
|
email = login + '@debian.org'
|
| 144 |
|
|
elif login.find('@') >= 0:
|
| 145 |
|
|
email = login
|
| 146 |
|
|
if email:
|
| 147 |
|
|
emails.append(email)
|
| 148 |
|
|
line = line[match.end():]
|
| 149 |
|
|
match = devel_php_RE.search(line)
|
| 150 |
|
|
f.close()
|
| 151 |
|
|
return emails
|
| 152 |
|
|
|
| 153 |
|
|
# write lowThresholdNmu info to a (global, i.e. not per-package) file
|
| 154 |
zack |
1800 |
#
|
| 155 |
|
|
# XXX this is sub-optimal, as the XSLT rendering of each package page will have
|
| 156 |
|
|
# to read the whole XML file each time. However, this can't be fixed here
|
| 157 |
|
|
# unless we have somewhere an additional map (in the spirit of sources.map)
|
| 158 |
|
|
# mapping source package names to maintainer emails
|
| 159 |
zack |
1773 |
low_nmu_emails = read_low_threshold_nmu(dir + "/low_threshold_nmu.txt")
|
| 160 |
|
|
f = open(odir + "/low_threshold_nmu.emails.xml", 'w')
|
| 161 |
|
|
f.write("<emails>\n");
|
| 162 |
|
|
f.writelines(map(lambda s: " <email>%s</email>\n" % s, low_nmu_emails))
|
| 163 |
|
|
f.write("</emails>\n");
|
| 164 |
|
|
f.close()
|
| 165 |
|
|
|
| 166 |
zack |
1800 |
# read the list of packages indexed by svnbuildstat.debian.net
|
| 167 |
|
|
svnbuildstat = {}
|
| 168 |
|
|
f = open(os.path.join(dir, "svnbuildstat_list.txt"))
|
| 169 |
|
|
for pkgname in map(string.rstrip, f.readlines()):
|
| 170 |
|
|
svnbuildstat[pkgname] = True
|
| 171 |
|
|
f.close()
|
| 172 |
|
|
|
| 173 |
hertzog |
344 |
# Create the XML documents
|
| 174 |
|
|
while 1:
|
| 175 |
|
|
line = sys.stdin.readline()
|
| 176 |
|
|
if not line: break #eof
|
| 177 |
|
|
pkg = line.strip()
|
| 178 |
|
|
|
| 179 |
|
|
doc = implementation.createDocument(None, "other", None)
|
| 180 |
|
|
root_elt = doc.documentElement
|
| 181 |
|
|
|
| 182 |
|
|
hash = pkg[0]
|
| 183 |
|
|
if pkg[0:3] == "lib":
|
| 184 |
|
|
hash = pkg[0:4]
|
| 185 |
|
|
|
| 186 |
hertzog |
372 |
# Add debcheck availability info
|
| 187 |
|
|
dc_sig = ""
|
| 188 |
|
|
elt = doc.createElement("debcheck")
|
| 189 |
|
|
for dist in ("stable", "testing", "unstable"):
|
| 190 |
|
|
if debcheck[dist].has_key(pkg):
|
| 191 |
|
|
elt.setAttribute(dist, "yes")
|
| 192 |
|
|
dc_sig += "y"
|
| 193 |
|
|
else:
|
| 194 |
|
|
elt.setAttribute(dist, "no")
|
| 195 |
|
|
dc_sig += "n"
|
| 196 |
|
|
root_elt.appendChild(elt)
|
| 197 |
|
|
|
| 198 |
hertzog |
479 |
# Add debconf templates availibilty info
|
| 199 |
|
|
if debconf.has_key(pkg):
|
| 200 |
|
|
root_elt.setAttribute("debconf", "yes")
|
| 201 |
|
|
dc_sig += "y"
|
| 202 |
|
|
else:
|
| 203 |
|
|
root_elt.setAttribute("debconf", "no")
|
| 204 |
|
|
dc_sig += "n"
|
| 205 |
|
|
|
| 206 |
hertzog |
344 |
# Get PTS stats
|
| 207 |
|
|
elt = doc.createElement("pts")
|
| 208 |
|
|
elt.setAttribute("count", pts.get(pkg, "0"))
|
| 209 |
|
|
root_elt.appendChild(elt)
|
| 210 |
|
|
|
| 211 |
|
|
# Get BTS stats
|
| 212 |
|
|
elt = doc.createElement("bugs")
|
| 213 |
hertzog |
1047 |
(s_rc, s_normal, s_wishlist, s_fixed, s_patch) = (0,0,0,0,0)
|
| 214 |
hertzog |
357 |
binlist = sources.get(pkg, [])
|
| 215 |
|
|
binlist.sort()
|
| 216 |
|
|
subsig = ""
|
| 217 |
|
|
for binary in binlist:
|
| 218 |
hertzog |
344 |
sub_elt = doc.createElement("item")
|
| 219 |
|
|
sub_elt.setAttribute("name", binary)
|
| 220 |
hertzog |
1047 |
(rc, normal, wishlist, fixed, patch) = bugs.get(binary, ["0","0","0","0","0"])
|
| 221 |
hertzog |
344 |
sub_elt.setAttribute("rc", rc)
|
| 222 |
|
|
sub_elt.setAttribute("normal", normal)
|
| 223 |
|
|
sub_elt.setAttribute("wishlist", wishlist)
|
| 224 |
|
|
sub_elt.setAttribute("fixed", fixed)
|
| 225 |
hertzog |
1047 |
sub_elt.setAttribute("patch", patch)
|
| 226 |
hertzog |
344 |
rc = string.atoi(rc)
|
| 227 |
|
|
normal = string.atoi(normal)
|
| 228 |
|
|
wishlist = string.atoi(wishlist)
|
| 229 |
|
|
fixed = string.atoi(fixed)
|
| 230 |
hertzog |
1047 |
patch = string.atoi(patch)
|
| 231 |
hertzog |
344 |
all = rc + normal + wishlist + fixed
|
| 232 |
|
|
sub_elt.setAttribute("all", "%d" % all)
|
| 233 |
|
|
elt.appendChild(sub_elt)
|
| 234 |
hertzog |
357 |
if len(subsig):
|
| 235 |
hertzog |
1047 |
subsig = "%s|%d|%d" % (subsig, all, patch)
|
| 236 |
hertzog |
357 |
else:
|
| 237 |
hertzog |
1047 |
subsig = "%d|%d" % (all, patch)
|
| 238 |
hertzog |
344 |
s_rc += rc
|
| 239 |
|
|
s_normal += normal
|
| 240 |
|
|
s_wishlist += wishlist
|
| 241 |
|
|
s_fixed += fixed
|
| 242 |
hertzog |
1047 |
s_patch += patch
|
| 243 |
hertzog |
413 |
if (pkg not in binlist): # Source package needs to be counted too
|
| 244 |
hertzog |
1047 |
(rc, normal, wishlist, fixed, patch) = bugs.get(pkg, ["0","0","0","0","0"])
|
| 245 |
hertzog |
413 |
s_rc += string.atoi(rc)
|
| 246 |
|
|
s_normal += string.atoi(normal)
|
| 247 |
|
|
s_wishlist += string.atoi(wishlist)
|
| 248 |
|
|
s_fixed += string.atoi(fixed)
|
| 249 |
hertzog |
1047 |
s_patch += string.atoi(patch)
|
| 250 |
hertzog |
413 |
|
| 251 |
hertzog |
344 |
elt.setAttribute("rc", "%d" % s_rc)
|
| 252 |
|
|
elt.setAttribute("normal", "%d" % s_normal)
|
| 253 |
|
|
elt.setAttribute("wishlist", "%d" % s_wishlist)
|
| 254 |
|
|
elt.setAttribute("fixed", "%d" % s_fixed)
|
| 255 |
hertzog |
1047 |
elt.setAttribute("patch", "%d" % s_patch)
|
| 256 |
hertzog |
344 |
elt.setAttribute("all", "%d" % (s_fixed + s_wishlist + s_normal + s_rc))
|
| 257 |
|
|
root_elt.appendChild(elt)
|
| 258 |
|
|
|
| 259 |
hertzog |
560 |
# Get WNPP information. [PvR]
|
| 260 |
|
|
if wnpp.has_key(pkg):
|
| 261 |
|
|
(type, number) = wnpp[pkg]
|
| 262 |
|
|
elt = doc.createElement("wnpp")
|
| 263 |
|
|
elt.setAttribute("type", type)
|
| 264 |
|
|
elt.setAttribute("bugnumber", number)
|
| 265 |
|
|
root_elt.appendChild(elt)
|
| 266 |
|
|
root_elt.setAttribute("wnpp", "yes")
|
| 267 |
|
|
wnpp_sig = "%s%s" % (type,number)
|
| 268 |
|
|
else:
|
| 269 |
|
|
root_elt.setAttribute("wnpp", "no")
|
| 270 |
|
|
wnpp_sig = "n"
|
| 271 |
hertzog |
643 |
|
| 272 |
jeroen |
947 |
# Get override info [JvW]
|
| 273 |
|
|
override_elt = None
|
| 274 |
|
|
override_sig = []
|
| 275 |
|
|
for dist in [ 'unstable', 'experimental' ]:
|
| 276 |
|
|
if override[dist].has_key(pkg):
|
| 277 |
|
|
if not override_elt: override_elt = doc.createElement("override")
|
| 278 |
|
|
disparities = override[dist][pkg]
|
| 279 |
|
|
override_sig.append(disparities)
|
| 280 |
|
|
elt_g = doc.createElement("group")
|
| 281 |
|
|
elt_g.setAttribute("suite", dist)
|
| 282 |
|
|
for disp in disparities:
|
| 283 |
|
|
elt = doc.createTextNode(disp)
|
| 284 |
|
|
elt_disp = doc.createElement("disparity")
|
| 285 |
|
|
elt_disp.appendChild(elt)
|
| 286 |
|
|
elt_g.appendChild(elt_disp)
|
| 287 |
jeroen |
950 |
override_elt.appendChild(elt_g)
|
| 288 |
jeroen |
947 |
|
| 289 |
|
|
if override_elt:
|
| 290 |
|
|
root_elt.appendChild(override_elt)
|
| 291 |
|
|
root_elt.setAttribute("override", "yes")
|
| 292 |
|
|
else:
|
| 293 |
|
|
root_elt.setAttribute("override", "no")
|
| 294 |
|
|
|
| 295 |
hertzog |
643 |
# Get watch information [FG]
|
| 296 |
|
|
if watch.has_key(pkg):
|
| 297 |
|
|
new = str(watch[pkg]['new'])
|
| 298 |
|
|
warn = watch[pkg]['warning']
|
| 299 |
|
|
url = watch[pkg]['url']
|
| 300 |
|
|
elt = doc.createElement("watch")
|
| 301 |
hertzog |
812 |
elt.setAttribute("new", unicode(new,'UTF8','replace'))
|
| 302 |
|
|
elt.setAttribute("warning", unicode(warn,'UTF8','replace'))
|
| 303 |
|
|
elt.setAttribute("url", unicode(url,'UTF8','replace'))
|
| 304 |
hertzog |
643 |
root_elt.appendChild(elt)
|
| 305 |
|
|
root_elt.setAttribute("watch", "yes")
|
| 306 |
|
|
watch_sig = new
|
| 307 |
hertzog |
659 |
if warn != '':
|
| 308 |
|
|
watch_sig = watch_sig + ("/W%d" % len(warn))
|
| 309 |
|
|
if url != '':
|
| 310 |
|
|
watch_sig = watch_sig + ("/U%d" % len(url))
|
| 311 |
hertzog |
643 |
else:
|
| 312 |
|
|
root_elt.setAttribute("watch", "no")
|
| 313 |
|
|
watch_sig = "n"
|
| 314 |
|
|
|
| 315 |
hertzog |
1049 |
# Add patches informations [FG]
|
| 316 |
|
|
if patches.has_key(pkg):
|
| 317 |
|
|
elt = doc.createElement("patches")
|
| 318 |
hertzog |
1725 |
patch_sig = ""
|
| 319 |
|
|
distros = patches[pkg].keys()
|
| 320 |
|
|
distros.sort()
|
| 321 |
|
|
for distro in distros:
|
| 322 |
hertzog |
1049 |
child = doc.createElement("item")
|
| 323 |
|
|
(version, url) = patches[pkg][distro]
|
| 324 |
|
|
child.setAttribute("distro", unicode(distro, 'UTF8', 'replace'))
|
| 325 |
|
|
child.setAttribute("version", unicode(version, 'UTF8', 'replace'))
|
| 326 |
|
|
child.setAttribute("url", unicode(url, 'UTF8', 'replace'))
|
| 327 |
|
|
elt.appendChild(child)
|
| 328 |
hertzog |
1725 |
patch_sig += "%s/%s " % (distro, version)
|
| 329 |
hertzog |
1049 |
root_elt.appendChild(elt)
|
| 330 |
|
|
root_elt.setAttribute("patches", "yes")
|
| 331 |
|
|
else:
|
| 332 |
|
|
root_elt.setAttribute("patches", "no")
|
| 333 |
|
|
patch_sig = "n"
|
| 334 |
hertzog |
560 |
|
| 335 |
zack |
1800 |
# add svnbuildstat info
|
| 336 |
|
|
if svnbuildstat.has_key(pkg):
|
| 337 |
|
|
elt = doc.createElement("svnbuildstat")
|
| 338 |
|
|
root_elt.setAttribute("svnbuildstat", "yes")
|
| 339 |
|
|
root_elt.appendChild(elt)
|
| 340 |
|
|
svnbuildstat_sig = "y"
|
| 341 |
|
|
else:
|
| 342 |
|
|
root_elt.setAttribute("svnbuildstat", "no")
|
| 343 |
|
|
svnbuildstat_sig = "n"
|
| 344 |
|
|
|
| 345 |
hertzog |
357 |
# TODO: try to do that signature checking before the creation of XML DOM
|
| 346 |
|
|
# Build the sig and check if anything changed
|
| 347 |
zack |
1800 |
sig = (pts.get(pkg, "0"), dc_sig, wnpp_sig, override_sig, watch_sig,
|
| 348 |
|
|
patch_sig, s_rc, s_normal, s_wishlist, s_fixed, subsig,
|
| 349 |
|
|
svnbuildstat_sig)
|
| 350 |
jeroen |
1033 |
if sigs.has_key(pkg) and sig == sigs[pkg] and \
|
| 351 |
|
|
os.path.isfile("%s/%s/%s/other.xml" % (odir, hash, pkg)):
|
| 352 |
hertzog |
357 |
continue
|
| 353 |
|
|
sigs[pkg] = sig
|
| 354 |
hertzog |
560 |
|
| 355 |
hertzog |
344 |
# Output the data to the XML file
|
| 356 |
hertzog |
812 |
try:
|
| 357 |
|
|
f = open("%s/%s/%s/other.xml" % (odir, hash, pkg), "w")
|
| 358 |
|
|
ext.PrettyPrint(doc, f, "iso-8859-1")
|
| 359 |
|
|
f.close()
|
| 360 |
|
|
except:
|
| 361 |
|
|
sys.stderr.write("Output problem for " + pkg + "/other.xml\n");
|
| 362 |
hertzog |
344 |
|
| 363 |
hertzog |
357 |
# Store the signatures
|
| 364 |
|
|
f = open(odir + "/other.sigs", "w")
|
| 365 |
|
|
cPickle.dump(sigs, f, 0)
|
| 366 |
|
|
f.close()
|
| 367 |
|
|
|
Parent Directory
| 
Revision Log