/[qa]/trunk/pts/www/bin/other_to_xml.py
ViewVC logotype

Contents of /trunk/pts/www/bin/other_to_xml.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1982 - (show annotations) (download) (as text)
Sat Aug 16 04:26:26 2008 UTC (4 years, 9 months ago) by zack
File MIME type: text/x-python
File size: 18053 byte(s)
- use utf8 encoding for intermediate xml files, as package descriptions are
  encoded in utf-8
- more verbose output of output problems
1 #!/usr/bin/python
2 # -*- coding: utf8 -*-
3
4 # Make sure tabs expand to 8 spaces in vim
5 # vim: expandtab
6
7 # Copyright 2002 Raphaƫl Hertzog
8 # Copyright 2005 Jeroen van Wolffelaar
9 # Copyright 2007-2008 Stefano Zacchiroli
10 # This file is distributed under the terms of the General Public License
11 # version 2 or (at your option) any later version.
12
13 import os.path, rfc822, sys, string, re, email, common, cPickle, syck
14 from xml.dom import implementation, ext
15
16 from config import dir, odir, root
17
18 # Create the source <-> binaries correspondances
19 source2binaries = {} # maps a source package to its binaries
20 binary2source = {} # maps a binary package to its source
21 f = open(dir + "/sources.map", "r")
22 while 1:
23 line = f.readline();
24 if not line: break #eof
25 line = line.strip()
26 (binary, source) = line.split(None, 1)
27 if not source2binaries.has_key(source):
28 source2binaries[source] = []
29 source2binaries[source].append(binary)
30 binary2source[binary] = source
31 f.close()
32
33 def read_shortdesc(fname):
34 global binary2source
35 source_descs = {} # source package -> (binary package -> short description)
36 for line in open(fname):
37 pkg, shortdesc = line.strip().split("\t", 1)
38 if not binary2source.has_key(pkg):
39 continue
40 src = binary2source[pkg]
41 if not source_descs.has_key(src):
42 source_descs[src] = {}
43 source_descs[src][pkg] = shortdesc
44 return source_descs
45
46 def read_bug_summary(fname):
47 global binary2source
48 summary = {} # source package -> bug count
49 for line in open(fname):
50 pkg, count = line.split()
51 if not binary2source.has_key(pkg):
52 continue
53 src = binary2source[pkg]
54 if not summary.has_key(src):
55 summary[src] = 0
56 summary[src] += int(count)
57 return summary
58
59 # Read all the bugs stats
60 bugs = {}
61 f = open(dir + "/bugs.txt")
62 while 1:
63 line = f.readline()
64 if not line: break #eof
65 line = line.strip()
66 (binary, stats) = line.split(None, 1)
67 bugs[binary] = [ string.atoi(i) for i in stats.split() ]
68 f.close()
69
70 srcbugs = {}
71 f = open(dir + "/bugs-src.txt")
72 while 1:
73 line = f.readline()
74 if not line: break #eof
75 line = line.strip()
76 (pkg, stats) = line.split(":", 1)
77 srcbugs[pkg] = [ string.atoi(i) for i in stats.replace("(", " ").replace(")", " ").split() ]
78 f.close()
79
80 gift_bugs = read_bug_summary(os.path.join(dir, 'bugs.gift.txt'))
81 help_bugs = read_bug_summary(os.path.join(dir, 'bugs.help.txt'))
82
83 # Read all the PTS stats
84 pts = {}
85 f = open(dir + "/count.txt")
86 while 1:
87 line = f.readline()
88 if not line: break #eof
89 line = line.strip()
90 (binary, stats) = line.split(None, 1)
91 pts[binary] = stats
92 f.close()
93
94 # Read the lisf of packages with debcheck problems
95 debcheck = {}
96 for dist in ("stable", "testing", "unstable"):
97 debcheck[dist] = {}
98 f = open(dir + "/debcheck-" + dist)
99 while 1:
100 line = f.readline()
101 if not line: break #eof
102 debcheck[dist][line.strip()] = 1
103 f.close()
104
105 # Read the list of packages with override disparities
106 override = {}
107 for dist in ("unstable", "experimental"):
108 override[dist] = {}
109 f = open(dir + "/override-disparities." + dist)
110 for line in f:
111 if line[0] != '-':
112 source = line.strip()[:-1]
113 override[dist][source] = []
114 else:
115 override[dist][source].append(line.strip()[2:])
116 f.close()
117
118 # Read the list of source packages with debconf templates
119 debconf = {}
120 # DISABLED until ddtp.debian.org is setup again
121 #f = open(dir + "/debconf-list")
122 #while 1:
123 # line = f.readline()
124 # if not line: break #eof
125 # line = line.strip()
126 # debconf[line] = 1;
127 #f.close()
128
129 # Read the current signature of other.xml files
130 sigs = {}
131 if os.path.exists(odir + "/other.sigs"):
132 f = open(odir + "/other.sigs", "r")
133 sigs = cPickle.load(f)
134 f.close()
135
136 # Read the wnpp information. [PvR]
137 wnpp = {}
138 if os.path.exists(dir + "/wnpp_rm"):
139 f = open(dir + "/wnpp_rm")
140 while 1:
141 line = f.readline()
142 if not line: break # eof
143 line = line.strip()
144 try:
145 (package, type, number) = line.split("|")[0].split()
146 except:
147 #too many badly formatted ITP... disable warning. --RH
148 #sys.stderr.write("Ignoring bad line '%s' in wnpp_rm\n" % line)
149 pass
150 wnpp[package[:-1]] = (type, number)
151 f.close()
152
153 # Read patches information [FG]
154 ubuntu_patches = {}
155 # this can be easily inserted into a new update_patches.py if it becomes too
156 # heavy to parse the file
157 if os.path.exists(dir + "/patches.ubuntu"):
158 f = open(dir + "/patches.ubuntu")
159 for line in f.readlines():
160 (package, rel_url) = line.split(' ', 2)
161 rel_url = rel_url.strip()
162 r = re.search("_(\S+).patch", line)
163 if not r:
164 continue
165 version = r.group(1)
166 ubuntu_patches[package] = (version, "http://patches.ubuntu.com/" + rel_url)
167 f.close()
168
169 ubuntu_versions = {}
170 if os.path.exists(dir + "/versions.ubuntu"):
171 f = open(dir + "/versions.ubuntu")
172 for line in f.readlines():
173 (package, version) = line.split(' ', 2)
174 version = version.strip()
175 ubuntu_versions[package] = (version, "https://launchpad.net/ubuntu/+source/" + package)
176 f.close()
177
178 ubuntu_bugs = {}
179 if os.path.exists(dir + "/bugs.ubuntu"):
180 f = open(dir + "/bugs.ubuntu")
181 for line in f.readlines():
182 (package, count) = line.split('|', 2)
183 count = count.strip()
184 ubuntu_bugs[package] = (count, "https://bugs.launchpad.net/ubuntu/+source/" + package)
185 f.close()
186
187 # read low threshold NMU infos
188 def read_low_threshold_nmu(fname):
189 emails = []
190 if os.path.exists(fname):
191 f = open(fname)
192 devel_php_RE = \
193 re.compile(r'http://qa\.debian\.org/developer\.php\?login=([^\s&]+)')
194 word_RE = re.compile(r'^\w+$')
195 for line in f.readlines():
196 match = devel_php_RE.search(line)
197 while match: # look for several matches on the same line
198 email = None
199 login = match.group(1)
200 if word_RE.match(login):
201 email = login + '@debian.org'
202 elif login.find('@') >= 0:
203 email = login
204 if email:
205 emails.append(email)
206 line = line[match.end():]
207 match = devel_php_RE.search(line)
208 f.close()
209 return emails
210
211 # write lowThresholdNmu info to a (global, i.e. not per-package) file
212 #
213 # XXX this is sub-optimal, as the XSLT rendering of each package page will have
214 # to read the whole XML file each time. However, this can't be fixed here
215 # unless we have somewhere an additional map (in the spirit of sources.map)
216 # mapping source package names to maintainer emails
217 low_nmu_emails = read_low_threshold_nmu(dir + "/low_threshold_nmu.txt")
218 f = open(odir + "/low_threshold_nmu.emails.xml", 'w')
219 f.write("<emails>\n");
220 f.writelines(map(lambda s: " <email>%s</email>\n" % s, low_nmu_emails))
221 f.write("</emails>\n");
222 f.close()
223
224 def read_transitions(fname):
225 f = open(fname)
226 markup = f.read()
227 f.close()
228 yaml = syck.load(markup)
229 packages = {} # maps pkg to the _list_ of transitions they are involved in
230 for id, transition in yaml.iteritems():
231 for pkg in transition['packages']:
232 if not packages.has_key(pkg):
233 packages[pkg] = []
234 packages[pkg].append(id)
235 return packages
236
237 # read the list of packages involved in transitions
238 transitions = read_transitions(os.path.join(dir, "transitions.yaml"))
239
240 def read_lintian_info(fname):
241 lintian = {} # maps source pkg names to pairs <errors_no, warnings_no>
242 for line in open(fname).readlines():
243 try:
244 (pkg, errors_no, warnings_no) = line.split()
245 lintian[pkg] = (int(errors_no), int(warnings_no))
246 except ValueError:
247 continue
248 return lintian
249
250 # read QA lintian info
251 lintian = read_lintian_info(os.path.join(dir, "lintian.qa-list.txt"))
252
253 # read the list of packages indexed by svnbuildstat.debian.net
254 svnbuildstat = {}
255 f = open(os.path.join(dir, "svnbuildstat_list.txt"))
256 for pkgname in map(string.rstrip, f.readlines()):
257 svnbuildstat[pkgname] = True
258 f.close()
259
260 # DEHS textual file are line oriented with lines like "field: value"
261 def read_dehs(fname):
262 f = open(fname)
263 for line in f.readlines():
264 yield map(string.strip, line.split(':'))
265 f.close()
266
267 # read info gathered from dehs.alioth.debian.org
268 dehs = {}
269 for pkgname, version in read_dehs(os.path.join(dir, "dehs_out_of_date.txt")):
270 if not dehs.has_key(pkgname):
271 dehs[pkgname] = {}
272 dehs[pkgname]['newer'] = version
273 for pkgname, msg in read_dehs(os.path.join(dir, "dehs_error.txt")):
274 if not dehs.has_key(pkgname):
275 dehs[pkgname] = {}
276 dehs[pkgname]['error'] = msg
277
278 # read short descriptions
279 shortdescs = read_shortdesc(os.path.join(dir, "shortdesc.txt"))
280
281 # Create the XML documents
282 while 1:
283 line = sys.stdin.readline()
284 if not line: break #eof
285 pkg = line.strip()
286
287 doc = implementation.createDocument(None, "other", None)
288 root_elt = doc.documentElement
289
290 hash = pkg[0]
291 if pkg[0:3] == "lib":
292 hash = pkg[0:4]
293
294 # Add debcheck availability info
295 dc_sig = ""
296 elt = doc.createElement("debcheck")
297 for dist in ("stable", "testing", "unstable"):
298 if debcheck[dist].has_key(pkg):
299 elt.setAttribute(dist, "yes")
300 dc_sig += "y"
301 else:
302 elt.setAttribute(dist, "no")
303 dc_sig += "n"
304 root_elt.appendChild(elt)
305
306 # Add debconf templates availibilty info
307 if debconf.has_key(pkg):
308 root_elt.setAttribute("debconf", "yes")
309 dc_sig += "y"
310 else:
311 root_elt.setAttribute("debconf", "no")
312 dc_sig += "n"
313
314 # Get PTS stats
315 elt = doc.createElement("pts")
316 elt.setAttribute("count", pts.get(pkg, "0"))
317 root_elt.appendChild(elt)
318
319 # Get BTS stats
320 elt = doc.createElement("bugs")
321 (s_rc, s_rc_m, s_normal, s_normal_m, s_wishlist, s_wishlist_m, s_fixed,
322 s_fixed_m, s_patch, s_patch_m) = \
323 srcbugs.get(pkg, [0,0,0,0,0,0,0,0,0,0])
324 binlist = source2binaries.get(pkg, [])
325 binlist.sort()
326 subsig = ""
327 for binary in binlist:
328 sub_elt = doc.createElement("item")
329 sub_elt.setAttribute("name", binary)
330 (rc, normal, wishlist, fixed, patch) = bugs.get(binary, [0,0,0,0,0])
331 sub_elt.setAttribute("rc", "%d" % rc)
332 sub_elt.setAttribute("normal", "%d" % normal)
333 sub_elt.setAttribute("wishlist", "%d" % wishlist)
334 sub_elt.setAttribute("fixed", "%d" % fixed)
335 sub_elt.setAttribute("patch", "%d" % patch)
336 all = rc + normal + wishlist + fixed
337 sub_elt.setAttribute("all", "%d" % all)
338 elt.appendChild(sub_elt)
339 if len(subsig):
340 subsig = "%s|%d|%d" % (subsig, all, patch)
341 else:
342 subsig = "%d|%d" % (all, patch)
343
344 elt.setAttribute("rc", "%d" % s_rc)
345 if s_rc != s_rc_m:
346 elt.setAttribute("rc_m", "%d" % s_rc_m)
347 elt.setAttribute("normal", "%d" % s_normal)
348 if s_normal != s_normal_m:
349 elt.setAttribute("normal_m", "%d" % s_normal_m)
350 elt.setAttribute("wishlist", "%d" % s_wishlist)
351 if s_wishlist != s_wishlist_m:
352 elt.setAttribute("wishlist_m", "%d" % s_wishlist_m)
353 elt.setAttribute("fixed", "%d" % s_fixed)
354 if s_fixed != s_fixed_m:
355 elt.setAttribute("fixed_m", "%d" % s_fixed_m)
356 elt.setAttribute("patch", "%d" % s_patch)
357 if s_patch != s_patch_m:
358 elt.setAttribute("patch_m", "%d" % s_patch_m)
359 s_all = s_fixed + s_wishlist + s_normal + s_rc
360 s_all_m = s_fixed_m + s_wishlist_m + s_normal_m + s_rc_m
361 elt.setAttribute("all", "%d" % s_all)
362 if s_all != s_all_m:
363 elt.setAttribute("all_m", "%d" % s_all_m)
364 root_elt.appendChild(elt)
365 if gift_bugs.has_key(pkg):
366 s_gift = gift_bugs[pkg]
367 else:
368 s_gift = 0
369 elt.setAttribute("gift", str(s_gift))
370 if help_bugs.has_key(pkg):
371 s_help = help_bugs[pkg]
372 else:
373 s_help = 0
374 elt.setAttribute("help", str(s_help))
375
376 # Get WNPP information. [PvR]
377 if wnpp.has_key(pkg):
378 (type, number) = wnpp[pkg]
379 elt = doc.createElement("wnpp")
380 elt.setAttribute("type", type)
381 elt.setAttribute("bugnumber", number)
382 root_elt.appendChild(elt)
383 root_elt.setAttribute("wnpp", "yes")
384 wnpp_sig = "%s%s" % (type,number)
385 else:
386 root_elt.setAttribute("wnpp", "no")
387 wnpp_sig = "n"
388
389 # Get override info [JvW]
390 override_elt = None
391 override_sig = []
392 for dist in [ 'unstable', 'experimental' ]:
393 if override[dist].has_key(pkg):
394 if not override_elt: override_elt = doc.createElement("override")
395 disparities = override[dist][pkg]
396 override_sig.append(disparities)
397 elt_g = doc.createElement("group")
398 elt_g.setAttribute("suite", dist)
399 for disp in disparities:
400 elt = doc.createTextNode(disp)
401 elt_disp = doc.createElement("disparity")
402 elt_disp.appendChild(elt)
403 elt_g.appendChild(elt_disp)
404 override_elt.appendChild(elt_g)
405
406 if override_elt:
407 root_elt.appendChild(override_elt)
408 root_elt.setAttribute("override", "yes")
409 else:
410 root_elt.setAttribute("override", "no")
411
412 # Add Ubuntu information
413 if ubuntu_versions.has_key(pkg):
414 elt = doc.createElement("ubuntu")
415 (version, url) = ubuntu_versions[pkg]
416 ubuntu_sig = "ubuntu/ " + version
417 elt.setAttribute("version", unicode(version, 'UTF8', 'replace'))
418 elt.setAttribute("url", unicode(url, 'UTF8', 'replace'))
419 if ubuntu_bugs.has_key(pkg):
420 elt.setAttribute("bugs", "yes")
421 elt_bugs = doc.createElement("bugs")
422 (count, url) = ubuntu_bugs[pkg]
423 ubuntu_sig += "ubuntubugs/" + count
424 elt_bugs.setAttribute("count", unicode(count, 'UTF8', 'replace'))
425 elt_bugs.setAttribute("url", unicode(url, 'UTF8', 'replace'))
426 elt.appendChild(elt_bugs)
427 if ubuntu_patches.has_key(pkg):
428 elt.setAttribute("patch", "yes")
429 elt_patch = doc.createElement("patch")
430 (version, url) = ubuntu_patches[pkg]
431 ubuntu_sig += "ubuntupatch/" + version
432 elt_patch.setAttribute("version", unicode(version, 'UTF8', 'replace'))
433 elt_patch.setAttribute("url", unicode(url, 'UTF8', 'replace'))
434 elt.appendChild(elt_patch)
435 root_elt.appendChild(elt)
436 root_elt.setAttribute("ubuntu", "yes")
437 else:
438 root_elt.setAttribute("ubuntu", "no")
439 ubuntu_sig = "n"
440
441 # Get DEHS information
442 if dehs.has_key(pkg):
443 elt = doc.createElement('dehs')
444 root_elt.appendChild(elt)
445 root_elt.setAttribute('dehs', 'yes')
446 if dehs[pkg].has_key('newer'):
447 elt.setAttribute('newer', dehs[pkg]['newer'])
448 if dehs[pkg].has_key('error'):
449 elt.setAttribute('error', 'yes')
450 dehs_sig = 'y'
451 else:
452 root_elt.setAttribute('dehs', 'no')
453 dehs_sig = 'n'
454
455 # add svnbuildstat info
456 if svnbuildstat.has_key(pkg):
457 elt = doc.createElement("svnbuildstat")
458 root_elt.setAttribute("svnbuildstat", "yes")
459 root_elt.appendChild(elt)
460 svnbuildstat_sig = "y"
461 else:
462 root_elt.setAttribute("svnbuildstat", "no")
463 svnbuildstat_sig = "n"
464
465 # add transitions info
466 if transitions.has_key(pkg):
467 elt = doc.createElement("transitions")
468 for id in transitions[pkg]:
469 trans_elt = doc.createElement("transition")
470 trans_elt.setAttribute("name", id)
471 elt.appendChild(trans_elt)
472 root_elt.setAttribute("transitions", "yes")
473 root_elt.appendChild(elt)
474 transitions_sig = "y"
475 else:
476 root_elt.setAttribute("transitions", "no")
477 transitions_sig = "n"
478
479 # add lintian QA info
480 if lintian.has_key(pkg):
481 (errs, warns) = lintian[pkg]
482 elt = doc.createElement("lintian")
483 elt.setAttribute("errors", str(errs))
484 elt.setAttribute("warnings", str(warns))
485 root_elt.appendChild(elt)
486 root_elt.setAttribute("lintian", "yes")
487 lintian_sig = (errs, warns)
488 else:
489 root_elt.setAttribute("lintian", "no")
490 lintian_sig = (0, 0)
491
492 # add short descriptions
493 elt = doc.createElement("descriptions")
494 root_elt.appendChild(elt)
495 if shortdescs.has_key(pkg):
496 for package, shortdesc in shortdescs[pkg].iteritems():
497 desc_elt = doc.createElement("shortdesc")
498 elt.appendChild(desc_elt)
499 desc_elt.setAttribute("package", package)
500 desc_text = doc.createTextNode(shortdesc)
501 desc_elt.appendChild(desc_text)
502 shortdesc_sig = str(shortdescs[pkg]).__hash__()
503 # XXX hash(str(...)) does not work: WTF?
504 else:
505 shortdesc_sig = ''.__hash__()
506
507 # TODO: try to do that signature checking before the creation of XML DOM
508 # Build the sig and check if anything changed
509 sig = (pts.get(pkg, "0"), dc_sig, wnpp_sig, override_sig, dehs_sig,
510 ubuntu_sig, s_rc, s_normal, s_wishlist, s_fixed, s_gift, s_help,
511 subsig, svnbuildstat_sig, transitions_sig, lintian_sig,
512 shortdesc_sig)
513 if sigs.has_key(pkg) and sig == sigs[pkg] and \
514 os.path.isfile("%s/%s/%s/other.xml" % (odir, hash, pkg)):
515 continue
516 sigs[pkg] = sig
517
518 # Output the data to the XML file
519 try:
520 f = open("%s/%s/%s/other.xml" % (odir, hash, pkg), "w")
521 ext.PrettyPrint(doc, f, "utf-8")
522 f.close()
523 except Exception, msg:
524 sys.stderr.write("Output problem for " + pkg + "/other.xml (%s)\n" %
525 msg);
526
527 # Store the signatures
528 f = open(odir + "/other.sigs", "w")
529 cPickle.dump(sigs, f, 0)
530 f.close()
531

Properties

Name Value
svn:eol-style native
svn:executable *
svn:keywords Author Date Id Revision

  ViewVC Help
Powered by ViewVC 1.1.5