| 1 |
#!/usr/bin/python
|
| 2 |
# -*- coding: utf8 -*-
|
| 3 |
|
| 4 |
# vim: expandtab
|
| 5 |
|
| 6 |
# Copyright 2002 Raphaël Hertzog
|
| 7 |
# This file is distributed under the terms of the General Public License
|
| 8 |
# version 2 or (at your option) any later version.
|
| 9 |
|
| 10 |
import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
|
| 11 |
from xml.dom import implementation, ext
|
| 12 |
|
| 13 |
from config import dir, odir
|
| 14 |
from common import vcs_table
|
| 15 |
|
| 16 |
# address_from_string takes an address in RFC822 format
|
| 17 |
# and turns it into a tuple of the form (real name, email).
|
| 18 |
# The difference with email.Utils.parseaddr and rfc822.parseaddr
|
| 19 |
# is that this routine allows unquoted comma's to appear in the real name
|
| 20 |
# (in violation of RFC822). This is a hack to allow a Maintainer field to
|
| 21 |
# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
|
| 22 |
def address_from_string(content):
|
| 23 |
hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
|
| 24 |
(name, mail) = email.Utils.parseaddr(hacked_content)
|
| 25 |
return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))
|
| 26 |
|
| 27 |
# addresses_from_string takes a string with addresses in RFC822 format
|
| 28 |
# and changes it into a list of tuples of the form (real name, email).
|
| 29 |
# Just as address_from_string, it tries to be forgiving about unquoted
|
| 30 |
# commas in addresses. [PvR]
|
| 31 |
def addresses_from_string(content):
|
| 32 |
pattern = re.compile("([^>]),")
|
| 33 |
hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
|
| 34 |
msg = email.message_from_string("Header: " + hacked_content)
|
| 35 |
hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
|
| 36 |
list = map(lambda p:
|
| 37 |
map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
|
| 38 |
hacked_list)
|
| 39 |
return list
|
| 40 |
|
| 41 |
def add_maintainer_info(child, name, mail, doc):
|
| 42 |
text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
|
| 43 |
item_elt = doc.createElement("name")
|
| 44 |
item_elt.appendChild(text)
|
| 45 |
child.appendChild(item_elt)
|
| 46 |
text = doc.createTextNode(mail)
|
| 47 |
item_elt = doc.createElement("email")
|
| 48 |
item_elt.appendChild(text)
|
| 49 |
child.appendChild(item_elt)
|
| 50 |
|
| 51 |
def update_sources_info(m, dist):
|
| 52 |
"""Update the XML information with the given Message (Package entry)"""
|
| 53 |
global odir, old_done, new_done, new_dist_map
|
| 54 |
package = m["Package"]
|
| 55 |
hash = package[0]
|
| 56 |
if package[0:3] == "lib":
|
| 57 |
hash = package[0:4]
|
| 58 |
# Check if the work has already been done
|
| 59 |
key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
|
| 60 |
new_done[key] = 1
|
| 61 |
new_dist_map["%s_%s" % (m["Package"], dist)] = 1
|
| 62 |
if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
|
| 63 |
return
|
| 64 |
# Make sure the directory exists
|
| 65 |
if not os.path.isdir(odir + "/" + hash + "/" + package):
|
| 66 |
if not os.path.isdir(odir + "/" + hash):
|
| 67 |
os.mkdir(odir + "/" + hash)
|
| 68 |
os.mkdir(odir + "/" + hash + "/" + package)
|
| 69 |
# Create the XML DOM object
|
| 70 |
doc = implementation.createDocument(None, None, None)
|
| 71 |
root = doc.createElement("source")
|
| 72 |
doc.appendChild(root)
|
| 73 |
root.setAttribute("release", dist)
|
| 74 |
if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
|
| 75 |
root.setAttribute("nmu", "yes")
|
| 76 |
for tag in m.keys():
|
| 77 |
child = doc.createElement(tag)
|
| 78 |
root.appendChild(child)
|
| 79 |
if tag == "binary" or tag[0:5] == "build":
|
| 80 |
for item in re.split(", ?", m[tag]):
|
| 81 |
# Take care of non-ascii, prevents troubles...
|
| 82 |
text = doc.createTextNode(unicode(item,'ISO-8859-1','replace'))
|
| 83 |
item_elt = doc.createElement("item")
|
| 84 |
item_elt.appendChild(text)
|
| 85 |
child.appendChild(item_elt)
|
| 86 |
elif tag == "maintainer":
|
| 87 |
(name, mail) = address_from_string(m[tag])
|
| 88 |
add_maintainer_info(child, name, mail, doc)
|
| 89 |
elif tag == "uploaders":
|
| 90 |
uploaders = addresses_from_string(m[tag])
|
| 91 |
for item in uploaders:
|
| 92 |
item_elt = doc.createElement("item")
|
| 93 |
(name,mail) = item
|
| 94 |
add_maintainer_info(item_elt, name, mail, doc)
|
| 95 |
child.appendChild(item_elt)
|
| 96 |
elif tag == "files":
|
| 97 |
for line in string.split(m[tag], "\n"):
|
| 98 |
item_elt = doc.createElement("item")
|
| 99 |
child.appendChild(item_elt)
|
| 100 |
line = line.strip()
|
| 101 |
fields = ["md5sum", "size", "filename"]
|
| 102 |
values = string.split(line)
|
| 103 |
for i in range(len(fields)):
|
| 104 |
new_elt = doc.createElement(fields[i])
|
| 105 |
text = doc.createTextNode(values[i])
|
| 106 |
new_elt.appendChild(text)
|
| 107 |
item_elt.appendChild(new_elt)
|
| 108 |
else:
|
| 109 |
text = doc.createTextNode(m[tag])
|
| 110 |
child.appendChild(text)
|
| 111 |
# now compute derived information and store it in the xml
|
| 112 |
# add 'parsed' version of VCS info
|
| 113 |
if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs-':
|
| 114 |
if tag[0] == 'x':
|
| 115 |
prefix_len = 6
|
| 116 |
else:
|
| 117 |
prefix_len = 4
|
| 118 |
# e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
|
| 119 |
# would be 'browser' for 'vcs-browser' (to be handled specially)
|
| 120 |
repos_elt = list(root.getElementsByTagName('repository'))
|
| 121 |
if repos_elt: # reuse existing repository element
|
| 122 |
repos_elt = repos_elt[0] # invariant: at most 1 repository elt
|
| 123 |
else: # create a new repository element
|
| 124 |
repos_elt = doc.createElement('repository')
|
| 125 |
root.appendChild(repos_elt)
|
| 126 |
vcs_elt = doc.createElement('vcs')
|
| 127 |
kind = tag[prefix_len:]
|
| 128 |
if vcs_table.has_key(kind):
|
| 129 |
kind = vcs_table[kind][0]
|
| 130 |
vcs_elt.setAttribute('kind', kind)
|
| 131 |
vcs_elt.setAttribute('url', m[tag])
|
| 132 |
repos_elt.appendChild(vcs_elt)
|
| 133 |
# Print the DOM object to a file
|
| 134 |
try:
|
| 135 |
f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
|
| 136 |
ext.PrettyPrint(doc, f, "UTF-8")
|
| 137 |
f.close()
|
| 138 |
except:
|
| 139 |
sys.stderr.write("Output problem for " + m["package"] + "\n");
|
| 140 |
|
| 141 |
def treat_sources_file(f, dist):
|
| 142 |
"""Scan the given Sources file and treat each Package entry"""
|
| 143 |
while 1:
|
| 144 |
try:
|
| 145 |
m = rfc822.Message(f)
|
| 146 |
if len(m) == 0: #eof
|
| 147 |
break
|
| 148 |
update_sources_info(m, dist)
|
| 149 |
except EOFError:
|
| 150 |
break
|
| 151 |
|
| 152 |
# Load the list of sources generated the last time
|
| 153 |
old_done = {}
|
| 154 |
new_done = {}
|
| 155 |
new_dist_map = {}
|
| 156 |
if os.path.exists(odir + "/sources_done"):
|
| 157 |
f = open(odir + "/sources_done", "r")
|
| 158 |
old_done = cPickle.load(f)
|
| 159 |
f.close()
|
| 160 |
|
| 161 |
for comp in ["main", "contrib", "non-free"]:
|
| 162 |
for dist in ["oldstable", "stable", "testing", "unstable", "experimental"]:
|
| 163 |
f = open(dir + "/Sources_%s_%s" % (dist, comp), "r")
|
| 164 |
treat_sources_file(f, dist)
|
| 165 |
f.close()
|
| 166 |
# s-p-u and t-p-u
|
| 167 |
f = open(dir + "/Sources-spu_%s" % comp, "r")
|
| 168 |
treat_sources_file(f, "s-p-u")
|
| 169 |
f.close()
|
| 170 |
f = open(dir + "/Sources-tpu_%s" % comp, "r")
|
| 171 |
treat_sources_file(f, "t-p-u")
|
| 172 |
f.close()
|
| 173 |
# security
|
| 174 |
f = open(dir + "/Sources-security-oldstable_%s" % comp, "r")
|
| 175 |
treat_sources_file(f, "oldstable-security")
|
| 176 |
f.close()
|
| 177 |
f = open(dir + "/Sources-security-stable_%s" % comp, "r")
|
| 178 |
treat_sources_file(f, "stable-security")
|
| 179 |
f.close()
|
| 180 |
#f = open(dir + "/Sources-security-testing_%s" % comp, "r")
|
| 181 |
#treat_sources_file(f, "testing-security")
|
| 182 |
#f.close()
|
| 183 |
f = open(dir + "/Sources-volatile_%s" % comp, "r")
|
| 184 |
treat_sources_file(f, "volatile")
|
| 185 |
f.close()
|
| 186 |
|
| 187 |
# Store the list of sources generated
|
| 188 |
f = open(odir + "/sources_done", "w")
|
| 189 |
cPickle.dump(new_done, f, 0)
|
| 190 |
f.close()
|
| 191 |
|
| 192 |
# Scan the old package/distribution that existed and check if they
|
| 193 |
# still exist ... if they don't, remove the associated xml file.
|
| 194 |
for key in old_done.keys():
|
| 195 |
(p, v, d) = key.split("_", 2)
|
| 196 |
if not new_dist_map.has_key("%s_%s" % (p, d)):
|
| 197 |
hash = p[0]
|
| 198 |
if p[0:3] == "lib":
|
| 199 |
hash = p[0:4]
|
| 200 |
filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
|
| 201 |
filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
|
| 202 |
if os.path.exists(filename):
|
| 203 |
os.unlink(filename)
|
| 204 |
f = open(filenamerebuild, "w")
|
| 205 |
f.close()
|
| 206 |
|
| 207 |
# We're done
|
Parent Directory
| 
Revision Log