/[qa]/trunk/pts/www/bin/sources_to_xml.py
ViewVC logotype

Diff of /trunk/pts/www/bin/sources_to_xml.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 344 by hertzog, Mon Aug 12 17:20:36 2002 UTC revision 1090 by hertzog, Sat Sep 10 17:50:51 2005 UTC
# Line 3  Line 3 
3  # Make sure tabs expand to 8 spaces in vim  # Make sure tabs expand to 8 spaces in vim
4  # vim: expandtab  # vim: expandtab
5    
6  import os.path, rfc822, sys, string, re  # Copyright 2002 Raphaël Hertzog
7    # This file is distributed under the terms of the General Public License
8    # version 2 or (at your option) any later version.
9    
10    import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
11  from xml.dom import implementation, ext  from xml.dom import implementation, ext
12    
13  from config import dir, odir  from config import dir, odir
14    
15  """if os.path.isdir("incoming"):  # address_from_string takes an address in RFC822 format
16      dir = "incoming"  # and turns it into a tuple of the form (real name, email).
17  elif os.path.isdir("../incoming"):  # The difference with email.Utils.parseaddr  and rfc822.parseaddr
18      dir = "../incoming"  # is that this routine allows unquoted comma's to appear in the real name
19  else:  # (in violation of RFC822). This is a hack to allow a Maintainer field to
20      dir = "/home/rhertzog/shared/paquets/debian/pts/incoming"  # be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
21    def address_from_string(content):
22  odir = dir + "/../base" """      hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
23        (name, mail) = email.Utils.parseaddr(hacked_content)
24  def add_maintainer_info(child, content, doc):      return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))
25      (name, email) = rfc822.parseaddr(content)  
26      text = doc.createTextNode(unicode(name,'iso-8859-1')) # Take care of non-ascii  # addresses_from_string takes a string with addresses in RFC822 format
27    # and changes it into a list of tuples of the form (real name, email).
28    # Just as address_from_string, it tries to be forgiving about unquoted
29    # commas in addresses. [PvR]
30    def addresses_from_string(content):
31        pattern = re.compile("([^>]),")
32        hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
33        msg = email.message_from_string("Header: " + hacked_content)
34        hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
35        list = map(lambda p:
36                   map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
37                   hacked_list)
38        return list
39    
40    def add_maintainer_info(child, name, mail, doc):
41        text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
42      item_elt = doc.createElement("name")      item_elt = doc.createElement("name")
43      item_elt.appendChild(text)      item_elt.appendChild(text)
44      child.appendChild(item_elt)      child.appendChild(item_elt)
45      text = doc.createTextNode(email)      text = doc.createTextNode(mail)
46      item_elt = doc.createElement("email")      item_elt = doc.createElement("email")
47      item_elt.appendChild(text)      item_elt.appendChild(text)
48      child.appendChild(item_elt)      child.appendChild(item_elt)
49    
50  def update_sources_info(m, dist):  def update_sources_info(m, dist):
51      """Update the XML information with the given Message (Package entry)"""      """Update the XML information with the given Message (Package entry)"""
52      global odir      global odir, old_done, new_done, new_dist_map
     # Make sure the directory exists  
53      package = m["Package"]      package = m["Package"]
54      hash = package[0]      hash = package[0]
55      if package[0:3] == "lib":      if package[0:3] == "lib":
56          hash = package[0:4]          hash = package[0:4]
57        # Check if the work has already been done
58        key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
59        new_done[key] = 1
60        new_dist_map["%s_%s" % (m["Package"], dist)] = 1
61        if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
62            return
63        # Make sure the directory exists
64      if not os.path.isdir(odir + "/" + hash + "/" + package):      if not os.path.isdir(odir + "/" + hash + "/" + package):
65          if not os.path.isdir(odir + "/" + hash):          if not os.path.isdir(odir + "/" + hash):
66              os.mkdir(odir + "/" + hash)              os.mkdir(odir + "/" + hash)
# Line 44  def update_sources_info(m, dist): Line 69  def update_sources_info(m, dist):
69      doc = implementation.createDocument(None, None, None)      doc = implementation.createDocument(None, None, None)
70      root = doc.createElement("source")      root = doc.createElement("source")
71      doc.appendChild(root)      doc.appendChild(root)
72      root.setAttribute("release", dist);      root.setAttribute("release", dist)
73      if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):      if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
74          root.setAttribute("nmu", "yes")          root.setAttribute("nmu", "yes")
75      for tag in m.keys():      for tag in m.keys():
# Line 52  def update_sources_info(m, dist): Line 77  def update_sources_info(m, dist):
77          root.appendChild(child)          root.appendChild(child)
78          if tag == "binary" or tag[0:5] == "build":          if tag == "binary" or tag[0:5] == "build":
79              for item in re.split(", ?", m[tag]):              for item in re.split(", ?", m[tag]):
80                  text = doc.createTextNode(item)                  # Take care of non-ascii, prevents troubles...
81                    text = doc.createTextNode(unicode(item,'ISO-8859-1','replace'))
82                  item_elt = doc.createElement("item")                  item_elt = doc.createElement("item")
83                  item_elt.appendChild(text)                  item_elt.appendChild(text)
84                  child.appendChild(item_elt)                  child.appendChild(item_elt)
85          elif tag == "maintainer":          elif tag == "maintainer":
86              add_maintainer_info(child, m[tag], doc)              (name, mail) = address_from_string(m[tag])
87                add_maintainer_info(child, name, mail, doc)
88          elif tag == "uploaders":          elif tag == "uploaders":
89              for item in re.split(", ?", m[tag]):              uploaders = addresses_from_string(m[tag])
90                for item in uploaders:
91                  item_elt = doc.createElement("item")                  item_elt = doc.createElement("item")
92                  add_maintainer_info(item_elt, item, doc)                  (name,mail) = item
93                    add_maintainer_info(item_elt, name, mail, doc)
94                  child.appendChild(item_elt)                  child.appendChild(item_elt)
95          elif tag == "files":          elif tag == "files":
96              for line in string.split(m[tag], "\n"):              for line in string.split(m[tag], "\n"):
# Line 81  def update_sources_info(m, dist): Line 110  def update_sources_info(m, dist):
110      # Print the DOM object to a file      # Print the DOM object to a file
111      try:      try:
112          f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")          f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
113          ext.PrettyPrint(doc, f, "iso-8859-1")          ext.PrettyPrint(doc, f, "UTF-8")
114          f.close()          f.close()
115      except:      except:
116          sys.stderr.write("Output problem for" + m["package"] + "\n");          sys.stderr.write("Output problem for " + m["package"] + "\n");
117    
118  def treat_sources_file(f, dist):  def treat_sources_file(f, dist):
119      """Scan the given Sources file and treat each Package entry"""      """Scan the given Sources file and treat each Package entry"""
# Line 97  def treat_sources_file(f, dist): Line 126  def treat_sources_file(f, dist):
126          except EOFError:          except EOFError:
127              break              break
128    
129    # Load the list of sources generated the last time
130    old_done = {}
131    new_done = {}
132    new_dist_map = {}
133    if os.path.exists(odir + "/sources_done"):
134        f = open(odir + "/sources_done", "r")
135        old_done = cPickle.load(f)
136        f.close()
137    
138  for comp in ["main", "contrib", "non-free"]:  for comp in ["main", "contrib", "non-free"]:
139      for dist in ["stable", "testing", "unstable"]:      for dist in ["oldstable", "stable", "testing", "unstable"]:
140          f = open(dir + "/Sources_%s_%s" % (dist, comp), "r")          f = open(dir + "/Sources_%s_%s" % (dist, comp), "r")
141          treat_sources_file(f, dist)          treat_sources_file(f, dist)
142          f.close()          f.close()
         f = open(dir + "/Sources-nonus_%s_%s" % (dist, comp), "r")  
         treat_sources_file(f, dist)  
         f.close()  
143      # Experimental      # Experimental
144      f = open(dir + "/Sources-experimental_" + comp, "r")      f = open(dir + "/Sources-experimental_" + comp, "r")
145      treat_sources_file(f,  "experimental")      treat_sources_file(f,  "experimental")
146      f.close()      f.close()
147        # s-p-u and t-p-u
148        f = open(dir + "/Sources-spu_%s" % comp, "r")
149        treat_sources_file(f, "s-p-u")
150        f.close()
151        f = open(dir + "/Sources-tpu_%s" % comp, "r")
152        treat_sources_file(f, "t-p-u")
153        f.close()
154        # security
155        f = open(dir + "/Sources-security-oldstable_%s" % comp, "r")
156        treat_sources_file(f, "oldstable-security")
157        f.close()
158        f = open(dir + "/Sources-security-stable_%s" % comp, "r")
159        treat_sources_file(f, "stable-security")
160        f.close()
161        #f = open(dir + "/Sources-security-testing_%s" % comp, "r")
162        #treat_sources_file(f, "testing-security")
163        #f.close()
164        f = open(dir + "/Sources-secure-testing_%s" % comp, "r")
165        treat_sources_file(f, "secure-testing")
166        f.close()
167        f = open(dir + "/Sources-volatile_%s" % comp, "r")
168        treat_sources_file(f, "volatile")
169        f.close()
170    
171    # Store the list of sources generated
172    f = open(odir + "/sources_done", "w")
173    cPickle.dump(new_done, f, 0)
174    f.close()
175    
176    # Scan the old package/distribution that existed and check if they
177    # still exist ... if they don't, remove the associated xml file.
178    for key in old_done.keys():
179        (p, v, d) = key.split("_", 2)
180        if not new_dist_map.has_key("%s_%s" % (p, d)):
181            hash = p[0]
182            if p[0:3] == "lib":
183                hash = p[0:4]
184            filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
185            filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
186            if os.path.exists(filename):
187                os.unlink(filename)
188                f = open(filenamerebuild, "w")
189                f.close()
190    
191    # We're done
Legend:
Removed from v.344  
changed lines
  Added in v.1090
  ViewVC Help
Powered by ViewVC 1.1.5