www/bin/sources_to_xml.py

#!/usr/bin/python
# -*- coding: utf8 -*-

# vim: expandtab

# Copyright 2002 Raphaël Hertzog
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
from xml.dom import implementation, ext

from config import dir, odir
from common import vcs_table

# address_from_string takes an address in RFC822 format 
# and turns it into a tuple of the form (real name, email).
# The difference with email.Utils.parseaddr  and rfc822.parseaddr 
# is that this routine allows unquoted comma's to appear in the real name
# (in violation of RFC822). This is a hack to allow a Maintainer field to
# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
def address_from_string(content):
    hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
    (name, mail) = email.Utils.parseaddr(hacked_content)
    return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))

# addresses_from_string takes a string with addresses in RFC822 format
# and changes it into a list of tuples of the form (real name, email). 
# Just as address_from_string, it tries to be forgiving about unquoted
# commas in addresses. [PvR]
def addresses_from_string(content):
    pattern = re.compile("([^>]),")
    hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
    msg = email.message_from_string("Header: " + hacked_content)
    hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
    list = map(lambda p:
               map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
               hacked_list)
    return list

def add_maintainer_info(child, name, mail, doc):
    text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
    item_elt = doc.createElement("name")
    item_elt.appendChild(text)
    child.appendChild(item_elt)
    text = doc.createTextNode(mail)
    item_elt = doc.createElement("email")
    item_elt.appendChild(text)
    child.appendChild(item_elt)

def update_sources_info(m, dist):
    """Update the XML information with the given Message (Package entry)"""
    global odir, old_done, new_done, new_dist_map
    package = m["Package"]
    hash = package[0]
    if package[0:3] == "lib":
        hash = package[0:4]
    # Check if the work has already been done
    key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
    new_done[key] = 1
    new_dist_map["%s_%s" % (m["Package"], dist)] = 1
    if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
        return
    # Make sure the directory exists
    if not os.path.isdir(odir + "/" + hash + "/" + package):
        if not os.path.isdir(odir + "/" + hash):
            os.mkdir(odir + "/" + hash)
        os.mkdir(odir + "/" + hash + "/" + package)
    # Create the XML DOM object
    doc = implementation.createDocument(None, None, None)
    root = doc.createElement("source")
    doc.appendChild(root)
    root.setAttribute("release", dist)
    if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
        root.setAttribute("nmu", "yes")
    for tag in m.keys():
        child = doc.createElement(tag)
        root.appendChild(child)
        if tag == "binary" or tag[0:5] == "build":
            for item in re.split(", ?", m[tag]):
                # Take care of non-ascii, prevents troubles...
                text = doc.createTextNode(unicode(item,'ISO-8859-1','replace')) 
                item_elt = doc.createElement("item")
                item_elt.appendChild(text)
                child.appendChild(item_elt)
        elif tag == "maintainer":
            (name, mail) = address_from_string(m[tag])
            add_maintainer_info(child, name, mail, doc)
        elif tag == "uploaders":
            uploaders = addresses_from_string(m[tag])
            for item in uploaders:
                item_elt = doc.createElement("item")
                (name,mail) = item
                add_maintainer_info(item_elt, name, mail, doc)
                child.appendChild(item_elt)
        elif tag == "files":
            for line in string.split(m[tag], "\n"):
                item_elt = doc.createElement("item")
                child.appendChild(item_elt)
                line = line.strip()
                fields = ["md5sum", "size", "filename"]
                values = string.split(line)
                for i in range(len(fields)):
                    new_elt = doc.createElement(fields[i])
                    text = doc.createTextNode(values[i])
                    new_elt.appendChild(text)
                    item_elt.appendChild(new_elt)
        else:
            text = doc.createTextNode(m[tag])
            child.appendChild(text)
        # now compute derived information and store it in the xml
            # add 'parsed' version of VCS info
        if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs':
            if tag[0] == 'x':
                prefix_len = 6
            else:
                prefix_len = 4
            # e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
            if vcs_table.has_key(tag[prefix_len:]):
                repos_elt = doc.createElement('repository')
                root.appendChild(repos_elt)
                kind_elt = doc.createElement('kind')
                repos_elt.appendChild(kind_elt)
                (name, vcsurl) = vcs_table[tag[prefix_len:]]
                kind_elt.setAttribute('href', vcsurl)
                name_txt = doc.createTextNode(name)
                kind_elt.appendChild(name_txt)
                url_elt = doc.createElement('url')
                repos_elt.appendChild(url_elt)
                url_txt = doc.createTextNode(m[tag])
                url_elt.appendChild(url_txt)
    # Print the DOM object to a file
    try:
        f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
        ext.PrettyPrint(doc, f, "UTF-8")
        f.close()
    except:
        sys.stderr.write("Output problem for " + m["package"] + "\n");

def treat_sources_file(f, dist):
    """Scan the given Sources file and treat each Package entry"""
    while 1:
        try:
            m = rfc822.Message(f)
            if len(m) == 0: #eof
                break
            update_sources_info(m, dist)
        except EOFError:
            break

# Load the list of sources generated the last time
old_done = {}
new_done = {}
new_dist_map = {}
if os.path.exists(odir + "/sources_done"):
    f = open(odir + "/sources_done", "r")
    old_done = cPickle.load(f)
    f.close()

for comp in ["main", "contrib", "non-free"]:
    for dist in ["oldstable", "stable", "testing", "unstable", "experimental"]:
        f = open(dir + "/Sources_%s_%s" % (dist, comp), "r")
        treat_sources_file(f, dist) 
        f.close()
    # s-p-u and t-p-u
    f = open(dir + "/Sources-spu_%s" % comp, "r")
    treat_sources_file(f, "s-p-u") 
    f.close()
    f = open(dir + "/Sources-tpu_%s" % comp, "r")
    treat_sources_file(f, "t-p-u") 
    f.close()
    # security
    f = open(dir + "/Sources-security-oldstable_%s" % comp, "r")
    treat_sources_file(f, "oldstable-security") 
    f.close()
    f = open(dir + "/Sources-security-stable_%s" % comp, "r")
    treat_sources_file(f, "stable-security") 
    f.close()
    #f = open(dir + "/Sources-security-testing_%s" % comp, "r")
    #treat_sources_file(f, "testing-security") 
    #f.close()
    f = open(dir + "/Sources-volatile_%s" % comp, "r")
    treat_sources_file(f, "volatile") 
    f.close()

# Store the list of sources generated
f = open(odir + "/sources_done", "w")
cPickle.dump(new_done, f, 0)
f.close()

# Scan the old package/distribution that existed and check if they
# still exist ... if they don't, remove the associated xml file.
for key in old_done.keys():
    (p, v, d) = key.split("_", 2)
    if not new_dist_map.has_key("%s_%s" % (p, d)):
        hash = p[0]
        if p[0:3] == "lib":
            hash = p[0:4]
        filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
        filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
        if os.path.exists(filename):
            os.unlink(filename)
            f = open(filenamerebuild, "w")
            f.close()

# We're done
1	#!/usr/bin/python
2	# -- coding: utf8 --
3
4	# vim: expandtab
5
6	# Copyright 2002 Raphaël Hertzog
7	# This file is distributed under the terms of the General Public License
8	# version 2 or (at your option) any later version.
9
10	import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
11	from xml.dom import implementation, ext
12
13	from config import dir, odir
14	from common import vcs_table
15
16	# address_from_string takes an address in RFC822 format
17	# and turns it into a tuple of the form (real name, email).
18	# The difference with email.Utils.parseaddr and rfc822.parseaddr
19	# is that this routine allows unquoted comma's to appear in the real name
20	# (in violation of RFC822). This is a hack to allow a Maintainer field to
21	# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
22	def address_from_string(content):
23	hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
24	(name, mail) = email.Utils.parseaddr(hacked_content)
25	return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))
26
27	# addresses_from_string takes a string with addresses in RFC822 format
28	# and changes it into a list of tuples of the form (real name, email).
29	# Just as address_from_string, it tries to be forgiving about unquoted
30	# commas in addresses. [PvR]
31	def addresses_from_string(content):
32	pattern = re.compile("([^>]),")
33	hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
34	msg = email.message_from_string("Header: " + hacked_content)
35	hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
36	list = map(lambda p:
37	map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
38	hacked_list)
39	return list
40
41	def add_maintainer_info(child, name, mail, doc):
42	text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
43	item_elt = doc.createElement("name")
44	item_elt.appendChild(text)
45	child.appendChild(item_elt)
46	text = doc.createTextNode(mail)
47	item_elt = doc.createElement("email")
48	item_elt.appendChild(text)
49	child.appendChild(item_elt)
50
51	def update_sources_info(m, dist):
52	"""Update the XML information with the given Message (Package entry)"""
53	global odir, old_done, new_done, new_dist_map
54	package = m["Package"]
55	hash = package[0]
56	if package[0:3] == "lib":
57	hash = package[0:4]
58	# Check if the work has already been done
59	key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
60	new_done[key] = 1
61	new_dist_map["%s_%s" % (m["Package"], dist)] = 1
62	if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
63	return
64	# Make sure the directory exists
65	if not os.path.isdir(odir + "/" + hash + "/" + package):
66	if not os.path.isdir(odir + "/" + hash):
67	os.mkdir(odir + "/" + hash)
68	os.mkdir(odir + "/" + hash + "/" + package)
69	# Create the XML DOM object
70	doc = implementation.createDocument(None, None, None)
71	root = doc.createElement("source")
72	doc.appendChild(root)
73	root.setAttribute("release", dist)
74	if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
75	root.setAttribute("nmu", "yes")
76	for tag in m.keys():
77	child = doc.createElement(tag)
78	root.appendChild(child)
79	if tag == "binary" or tag[0:5] == "build":
80	for item in re.split(", ?", m[tag]):
81	# Take care of non-ascii, prevents troubles...
82	text = doc.createTextNode(unicode(item,'ISO-8859-1','replace'))
83	item_elt = doc.createElement("item")
84	item_elt.appendChild(text)
85	child.appendChild(item_elt)
86	elif tag == "maintainer":
87	(name, mail) = address_from_string(m[tag])
88	add_maintainer_info(child, name, mail, doc)
89	elif tag == "uploaders":
90	uploaders = addresses_from_string(m[tag])
91	for item in uploaders:
92	item_elt = doc.createElement("item")
93	(name,mail) = item
94	add_maintainer_info(item_elt, name, mail, doc)
95	child.appendChild(item_elt)
96	elif tag == "files":
97	for line in string.split(m[tag], "\n"):
98	item_elt = doc.createElement("item")
99	child.appendChild(item_elt)
100	line = line.strip()
101	fields = ["md5sum", "size", "filename"]
102	values = string.split(line)
103	for i in range(len(fields)):
104	new_elt = doc.createElement(fields[i])
105	text = doc.createTextNode(values[i])
106	new_elt.appendChild(text)
107	item_elt.appendChild(new_elt)
108	else:
109	text = doc.createTextNode(m[tag])
110	child.appendChild(text)
111	# now compute derived information and store it in the xml
112	# add 'parsed' version of VCS info
113	if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs':
114	if tag[0] == 'x':
115	prefix_len = 6
116	else:
117	prefix_len = 4
118	# e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
119	if vcs_table.has_key(tag[prefix_len:]):
120	repos_elt = doc.createElement('repository')
121	root.appendChild(repos_elt)
122	kind_elt = doc.createElement('kind')
123	repos_elt.appendChild(kind_elt)
124	(name, vcsurl) = vcs_table[tag[prefix_len:]]
125	kind_elt.setAttribute('href', vcsurl)
126	name_txt = doc.createTextNode(name)
127	kind_elt.appendChild(name_txt)
128	url_elt = doc.createElement('url')
129	repos_elt.appendChild(url_elt)
130	url_txt = doc.createTextNode(m[tag])
131	url_elt.appendChild(url_txt)
132	# Print the DOM object to a file
133	try:
134	f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
135	ext.PrettyPrint(doc, f, "UTF-8")
136	f.close()
137	except:
138	sys.stderr.write("Output problem for " + m["package"] + "\n");
139
140	def treat_sources_file(f, dist):
141	"""Scan the given Sources file and treat each Package entry"""
142	while 1:
143	try:
144	m = rfc822.Message(f)
145	if len(m) == 0: #eof
146	break
147	update_sources_info(m, dist)
148	except EOFError:
149	break
150
151	# Load the list of sources generated the last time
152	old_done = {}
153	new_done = {}
154	new_dist_map = {}
155	if os.path.exists(odir + "/sources_done"):
156	f = open(odir + "/sources_done", "r")
157	old_done = cPickle.load(f)
158	f.close()
159
160	for comp in ["main", "contrib", "non-free"]:
161	for dist in ["oldstable", "stable", "testing", "unstable", "experimental"]:
162	f = open(dir + "/Sources_%s_%s" % (dist, comp), "r")
163	treat_sources_file(f, dist)
164	f.close()
165	# s-p-u and t-p-u
166	f = open(dir + "/Sources-spu_%s" % comp, "r")
167	treat_sources_file(f, "s-p-u")
168	f.close()
169	f = open(dir + "/Sources-tpu_%s" % comp, "r")
170	treat_sources_file(f, "t-p-u")
171	f.close()
172	# security
173	f = open(dir + "/Sources-security-oldstable_%s" % comp, "r")
174	treat_sources_file(f, "oldstable-security")
175	f.close()
176	f = open(dir + "/Sources-security-stable_%s" % comp, "r")
177	treat_sources_file(f, "stable-security")
178	f.close()
179	#f = open(dir + "/Sources-security-testing_%s" % comp, "r")
180	#treat_sources_file(f, "testing-security")
181	#f.close()
182	f = open(dir + "/Sources-volatile_%s" % comp, "r")
183	treat_sources_file(f, "volatile")
184	f.close()
185
186	# Store the list of sources generated
187	f = open(odir + "/sources_done", "w")
188	cPickle.dump(new_done, f, 0)
189	f.close()
190
191	# Scan the old package/distribution that existed and check if they
192	# still exist ... if they don't, remove the associated xml file.
193	for key in old_done.keys():
194	(p, v, d) = key.split("_", 2)
195	if not new_dist_map.has_key("%s_%s" % (p, d)):
196	hash = p[0]
197	if p[0:3] == "lib":
198	hash = p[0:4]
199	filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
200	filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
201	if os.path.exists(filename):
202	os.unlink(filename)
203	f = open(filenamerebuild, "w")
204	f.close()
205
206	# We're done
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision