/[qa]/trunk/pts/www/bin/other_to_xml.py
ViewVC logotype

Contents of /trunk/pts/www/bin/other_to_xml.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1882 - (hide annotations) (download) (as text)
Fri Jun 6 10:42:02 2008 UTC (4 years, 11 months ago) by hertzog
File MIME type: text/x-python
File size: 13497 byte(s)
Disable warning, it happens too often.
1 jeroen 1154 #!/usr/bin/python
2     # -*- coding: utf8 -*-
3 hertzog 344
4     # Make sure tabs expand to 8 spaces in vim
5     # vim: expandtab
6    
7 jeroen 1154 # Copyright 2002 Raphaƫl Hertzog
8 jeroen 947 # Copyright 2005 Jeroen van Wolffelaar
9 zack 1820 # Copyright 2007-2008 Stefano Zacchiroli
10 hertzog 351 # This file is distributed under the terms of the General Public License
11     # version 2 or (at your option) any later version.
12    
13 zack 1866 import os.path, rfc822, sys, string, re, email, common, cPickle, syck
14 hertzog 344 from xml.dom import implementation, ext
15    
16     from config import dir, odir, root
17    
18     # Create the source -> binaries correspondance
19     sources = {}
20 hertzog 812 f = open(dir + "/sources.map", "r")
21 hertzog 344 while 1:
22     line = f.readline();
23     if not line: break #eof
24     line = line.strip()
25     (binary, source) = line.split(None, 1)
26     if not sources.has_key(source):
27     sources[source] = []
28     sources[source].append(binary)
29     f.close()
30    
31     # Read all the bugs stats
32     bugs = {}
33     f = open(dir + "/bugs.txt")
34     while 1:
35     line = f.readline()
36     if not line: break #eof
37     line = line.strip()
38     (binary, stats) = line.split(None, 1)
39     bugs[binary] = stats.split()
40     f.close()
41    
42     # Read all the PTS stats
43     pts = {}
44     f = open(dir + "/count.txt")
45     while 1:
46     line = f.readline()
47     if not line: break #eof
48     line = line.strip()
49     (binary, stats) = line.split(None, 1)
50     pts[binary] = stats
51     f.close()
52    
53 hertzog 372 # Read the lisf of packages with debcheck problems
54     debcheck = {}
55     for dist in ("stable", "testing", "unstable"):
56     debcheck[dist] = {}
57     f = open(dir + "/debcheck-" + dist)
58     while 1:
59     line = f.readline()
60     if not line: break #eof
61     debcheck[dist][line.strip()] = 1
62     f.close()
63    
64 jeroen 947 # Read the list of packages with override disparities
65     override = {}
66     for dist in ("unstable", "experimental"):
67     override[dist] = {}
68     f = open(dir + "/override-disparities." + dist)
69     for line in f:
70     if line[0] != '-':
71     source = line.strip()[:-1]
72     override[dist][source] = []
73     else:
74     override[dist][source].append(line.strip()[2:])
75     f.close()
76    
77 hertzog 479 # Read the list of source packages with debconf templates
78     debconf = {}
79 hertzog 692 # DISABLED until ddtp.debian.org is setup again
80     #f = open(dir + "/debconf-list")
81     #while 1:
82     # line = f.readline()
83     # if not line: break #eof
84     # line = line.strip()
85     # debconf[line] = 1;
86     #f.close()
87 hertzog 479
88 hertzog 357 # Read the current signature of other.xml files
89     sigs = {}
90     if os.path.exists(odir + "/other.sigs"):
91     f = open(odir + "/other.sigs", "r")
92     sigs = cPickle.load(f)
93     f.close()
94    
95 hertzog 560 # Read the wnpp information. [PvR]
96     wnpp = {}
97 jeroen 912 if os.path.exists(dir + "/wnpp_rm"):
98     f = open(dir + "/wnpp_rm")
99 hertzog 560 while 1:
100     line = f.readline()
101     if not line: break # eof
102 hertzog 1843 line = line.strip()
103     try:
104     (package, type, number) = line.split("|")[0].split()
105     except:
106 hertzog 1882 #too many badly formatted ITP... disable warning. --RH
107     #sys.stderr.write("Ignoring bad line '%s' in wnpp_rm\n" % line)
108 hertzog 1843 pass
109 jeroen 912 wnpp[package[:-1]] = (type, number)
110 hertzog 560 f.close()
111    
112 hertzog 1049 # Read patches information [FG]
113     patches = {}
114     # this can be easily inserted into a new update_patches.py if it becomes too
115     # heavy to parse the file
116     if os.path.exists(dir + "/patches.ubuntu"):
117     f = open(dir + "/patches.ubuntu")
118     for line in f.readlines():
119     (package, rel_url) = line.split(' ', 2)
120     rel_url = rel_url.strip()
121     r = re.search("_(\S+).patch", line)
122     if not r:
123     continue
124     version = r.group(1)
125     patches[package] = {}
126 hertzog 1355 patches[package]['ubuntu'] = (version, "http://patches.ubuntu.com/" + rel_url)
127 zack 1749 f.close()
128 hertzog 1049
129 zack 1773 # read low threshold NMU infos
130     def read_low_threshold_nmu(fname):
131     emails = []
132     if os.path.exists(fname):
133     f = open(fname)
134     devel_php_RE = \
135     re.compile(r'http://qa\.debian\.org/developer\.php\?login=([^\s&]+)')
136     word_RE = re.compile(r'^\w+$')
137     for line in f.readlines():
138     match = devel_php_RE.search(line)
139     while match: # look for several matches on the same line
140     email = None
141     login = match.group(1)
142     if word_RE.match(login):
143     email = login + '@debian.org'
144     elif login.find('@') >= 0:
145     email = login
146     if email:
147     emails.append(email)
148     line = line[match.end():]
149     match = devel_php_RE.search(line)
150     f.close()
151     return emails
152    
153     # write lowThresholdNmu info to a (global, i.e. not per-package) file
154 zack 1800 #
155     # XXX this is sub-optimal, as the XSLT rendering of each package page will have
156     # to read the whole XML file each time. However, this can't be fixed here
157     # unless we have somewhere an additional map (in the spirit of sources.map)
158     # mapping source package names to maintainer emails
159 zack 1773 low_nmu_emails = read_low_threshold_nmu(dir + "/low_threshold_nmu.txt")
160     f = open(odir + "/low_threshold_nmu.emails.xml", 'w')
161     f.write("<emails>\n");
162     f.writelines(map(lambda s: " <email>%s</email>\n" % s, low_nmu_emails))
163     f.write("</emails>\n");
164     f.close()
165    
166 zack 1866 def read_transitions(fname):
167     f = open(fname)
168     markup = f.read()
169     f.close()
170     yaml = syck.load(markup)
171     packages = {} # maps pkg to the _list_ of transitions they are involved in
172     for id, transition in yaml.iteritems():
173     for pkg in transition['packages']:
174     if not packages.has_key(pkg):
175     packages[pkg] = []
176     packages[pkg].append(id)
177     return packages
178    
179     # read the list of packages involved in transitions
180     transitions = read_transitions(os.path.join(dir, "transitions.yaml"))
181    
182 zack 1800 # read the list of packages indexed by svnbuildstat.debian.net
183     svnbuildstat = {}
184     f = open(os.path.join(dir, "svnbuildstat_list.txt"))
185     for pkgname in map(string.rstrip, f.readlines()):
186     svnbuildstat[pkgname] = True
187     f.close()
188    
189 zack 1823 # DEHS textual file are line oriented with lines like "field: value"
190     def read_dehs(fname):
191     f = open(fname)
192     for line in f.readlines():
193     yield map(string.strip, line.split(':'))
194     f.close()
195    
196     # read info gathered from dehs.alioth.debian.org
197     dehs = {}
198     for pkgname, version in read_dehs(os.path.join(dir, "dehs_out_of_date.txt")):
199     if not dehs.has_key(pkgname):
200     dehs[pkgname] = {}
201     dehs[pkgname]['newer'] = version
202     for pkgname, msg in read_dehs(os.path.join(dir, "dehs_error.txt")):
203     if not dehs.has_key(pkgname):
204     dehs[pkgname] = {}
205     dehs[pkgname]['error'] = msg
206    
207 hertzog 344 # Create the XML documents
208     while 1:
209     line = sys.stdin.readline()
210     if not line: break #eof
211     pkg = line.strip()
212    
213     doc = implementation.createDocument(None, "other", None)
214     root_elt = doc.documentElement
215    
216     hash = pkg[0]
217     if pkg[0:3] == "lib":
218     hash = pkg[0:4]
219    
220 hertzog 372 # Add debcheck availability info
221     dc_sig = ""
222     elt = doc.createElement("debcheck")
223     for dist in ("stable", "testing", "unstable"):
224     if debcheck[dist].has_key(pkg):
225     elt.setAttribute(dist, "yes")
226     dc_sig += "y"
227     else:
228     elt.setAttribute(dist, "no")
229     dc_sig += "n"
230     root_elt.appendChild(elt)
231    
232 hertzog 479 # Add debconf templates availibilty info
233     if debconf.has_key(pkg):
234     root_elt.setAttribute("debconf", "yes")
235     dc_sig += "y"
236     else:
237     root_elt.setAttribute("debconf", "no")
238     dc_sig += "n"
239    
240 hertzog 344 # Get PTS stats
241     elt = doc.createElement("pts")
242     elt.setAttribute("count", pts.get(pkg, "0"))
243     root_elt.appendChild(elt)
244    
245     # Get BTS stats
246     elt = doc.createElement("bugs")
247 hertzog 1047 (s_rc, s_normal, s_wishlist, s_fixed, s_patch) = (0,0,0,0,0)
248 hertzog 357 binlist = sources.get(pkg, [])
249     binlist.sort()
250     subsig = ""
251     for binary in binlist:
252 hertzog 344 sub_elt = doc.createElement("item")
253     sub_elt.setAttribute("name", binary)
254 hertzog 1047 (rc, normal, wishlist, fixed, patch) = bugs.get(binary, ["0","0","0","0","0"])
255 hertzog 344 sub_elt.setAttribute("rc", rc)
256     sub_elt.setAttribute("normal", normal)
257     sub_elt.setAttribute("wishlist", wishlist)
258     sub_elt.setAttribute("fixed", fixed)
259 hertzog 1047 sub_elt.setAttribute("patch", patch)
260 hertzog 344 rc = string.atoi(rc)
261     normal = string.atoi(normal)
262     wishlist = string.atoi(wishlist)
263     fixed = string.atoi(fixed)
264 hertzog 1047 patch = string.atoi(patch)
265 hertzog 344 all = rc + normal + wishlist + fixed
266     sub_elt.setAttribute("all", "%d" % all)
267     elt.appendChild(sub_elt)
268 hertzog 357 if len(subsig):
269 hertzog 1047 subsig = "%s|%d|%d" % (subsig, all, patch)
270 hertzog 357 else:
271 hertzog 1047 subsig = "%d|%d" % (all, patch)
272 hertzog 344 s_rc += rc
273     s_normal += normal
274     s_wishlist += wishlist
275     s_fixed += fixed
276 hertzog 1047 s_patch += patch
277 hertzog 413 if (pkg not in binlist): # Source package needs to be counted too
278 hertzog 1047 (rc, normal, wishlist, fixed, patch) = bugs.get(pkg, ["0","0","0","0","0"])
279 hertzog 413 s_rc += string.atoi(rc)
280     s_normal += string.atoi(normal)
281     s_wishlist += string.atoi(wishlist)
282     s_fixed += string.atoi(fixed)
283 hertzog 1047 s_patch += string.atoi(patch)
284 hertzog 413
285 hertzog 344 elt.setAttribute("rc", "%d" % s_rc)
286     elt.setAttribute("normal", "%d" % s_normal)
287     elt.setAttribute("wishlist", "%d" % s_wishlist)
288     elt.setAttribute("fixed", "%d" % s_fixed)
289 hertzog 1047 elt.setAttribute("patch", "%d" % s_patch)
290 hertzog 344 elt.setAttribute("all", "%d" % (s_fixed + s_wishlist + s_normal + s_rc))
291     root_elt.appendChild(elt)
292    
293 hertzog 560 # Get WNPP information. [PvR]
294     if wnpp.has_key(pkg):
295     (type, number) = wnpp[pkg]
296     elt = doc.createElement("wnpp")
297     elt.setAttribute("type", type)
298     elt.setAttribute("bugnumber", number)
299     root_elt.appendChild(elt)
300     root_elt.setAttribute("wnpp", "yes")
301     wnpp_sig = "%s%s" % (type,number)
302     else:
303     root_elt.setAttribute("wnpp", "no")
304     wnpp_sig = "n"
305 hertzog 643
306 jeroen 947 # Get override info [JvW]
307     override_elt = None
308     override_sig = []
309     for dist in [ 'unstable', 'experimental' ]:
310     if override[dist].has_key(pkg):
311     if not override_elt: override_elt = doc.createElement("override")
312     disparities = override[dist][pkg]
313     override_sig.append(disparities)
314     elt_g = doc.createElement("group")
315     elt_g.setAttribute("suite", dist)
316     for disp in disparities:
317     elt = doc.createTextNode(disp)
318     elt_disp = doc.createElement("disparity")
319     elt_disp.appendChild(elt)
320     elt_g.appendChild(elt_disp)
321 jeroen 950 override_elt.appendChild(elt_g)
322 jeroen 947
323     if override_elt:
324     root_elt.appendChild(override_elt)
325     root_elt.setAttribute("override", "yes")
326     else:
327     root_elt.setAttribute("override", "no")
328    
329 hertzog 1049 # Add patches informations [FG]
330     if patches.has_key(pkg):
331     elt = doc.createElement("patches")
332 hertzog 1725 patch_sig = ""
333     distros = patches[pkg].keys()
334     distros.sort()
335     for distro in distros:
336 hertzog 1049 child = doc.createElement("item")
337     (version, url) = patches[pkg][distro]
338     child.setAttribute("distro", unicode(distro, 'UTF8', 'replace'))
339     child.setAttribute("version", unicode(version, 'UTF8', 'replace'))
340     child.setAttribute("url", unicode(url, 'UTF8', 'replace'))
341     elt.appendChild(child)
342 hertzog 1725 patch_sig += "%s/%s " % (distro, version)
343 hertzog 1049 root_elt.appendChild(elt)
344     root_elt.setAttribute("patches", "yes")
345     else:
346     root_elt.setAttribute("patches", "no")
347     patch_sig = "n"
348 hertzog 560
349 zack 1823 # Get DEHS information
350     if dehs.has_key(pkg):
351     elt = doc.createElement('dehs')
352     root_elt.appendChild(elt)
353     root_elt.setAttribute('dehs', 'yes')
354     if dehs[pkg].has_key('newer'):
355     elt.setAttribute('newer', dehs[pkg]['newer'])
356     if dehs[pkg].has_key('error'):
357     elt.setAttribute('error', 'yes')
358     dehs_sig = 'y'
359     else:
360     root_elt.setAttribute('dehs', 'no')
361     dehs_sig = 'n'
362    
363 zack 1800 # add svnbuildstat info
364     if svnbuildstat.has_key(pkg):
365     elt = doc.createElement("svnbuildstat")
366     root_elt.setAttribute("svnbuildstat", "yes")
367     root_elt.appendChild(elt)
368     svnbuildstat_sig = "y"
369     else:
370     root_elt.setAttribute("svnbuildstat", "no")
371     svnbuildstat_sig = "n"
372    
373 zack 1866 # add transitions info
374     if transitions.has_key(pkg):
375     elt = doc.createElement("transitions")
376     for id in transitions[pkg]:
377     trans_elt = doc.createElement("transition")
378     trans_elt.setAttribute("name", id)
379     elt.appendChild(trans_elt)
380     root_elt.setAttribute("transitions", "yes")
381     root_elt.appendChild(elt)
382     transitions_sig = "y"
383     else:
384     root_elt.setAttribute("transitions", "no")
385     transitions_sig = "n"
386    
387 hertzog 357 # TODO: try to do that signature checking before the creation of XML DOM
388     # Build the sig and check if anything changed
389 zack 1823 sig = (pts.get(pkg, "0"), dc_sig, wnpp_sig, override_sig, dehs_sig,
390 zack 1800 patch_sig, s_rc, s_normal, s_wishlist, s_fixed, subsig,
391 zack 1866 svnbuildstat_sig, transitions_sig)
392 jeroen 1033 if sigs.has_key(pkg) and sig == sigs[pkg] and \
393     os.path.isfile("%s/%s/%s/other.xml" % (odir, hash, pkg)):
394 hertzog 357 continue
395     sigs[pkg] = sig
396 hertzog 560
397 hertzog 344 # Output the data to the XML file
398 hertzog 812 try:
399     f = open("%s/%s/%s/other.xml" % (odir, hash, pkg), "w")
400     ext.PrettyPrint(doc, f, "iso-8859-1")
401     f.close()
402     except:
403     sys.stderr.write("Output problem for " + pkg + "/other.xml\n");
404 hertzog 344
405 hertzog 357 # Store the signatures
406     f = open(odir + "/other.sigs", "w")
407     cPickle.dump(sigs, f, 0)
408     f.close()
409    

Properties

Name Value
svn:eol-style native
svn:executable *
svn:keywords Author Date Id Revision

  ViewVC Help
Powered by ViewVC 1.1.5