www/bin/sources_to_xml.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

# vim: expandtab

# Copyright 2002 Raphaël Hertzog
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
from xml.dom import implementation, ext

from config import dir, odir
from common import vcs_table, hash_name, DpkgVersion

# address_from_string takes an address in RFC822 format 
# and turns it into a tuple of the form (real name, email).
# The difference with email.Utils.parseaddr  and rfc822.parseaddr 
# is that this routine allows unquoted comma's to appear in the real name
# (in violation of RFC822). This is a hack to allow a Maintainer field to
# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
def address_from_string(content):
    hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
    (name, mail) = email.Utils.parseaddr(hacked_content)
    return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))

# addresses_from_string takes a string with addresses in RFC822 format
# and changes it into a list of tuples of the form (real name, email). 
# Just as address_from_string, it tries to be forgiving about unquoted
# commas in addresses. [PvR]
def addresses_from_string(content):
    pattern = re.compile("([^>]),")
    hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
    msg = email.message_from_string("Header: " + hacked_content)
    hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
    list = map(lambda p:
               map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
               hacked_list)
    return list

def add_maintainer_info(child, name, mail, doc):
    text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
    item_elt = doc.createElement("name")
    item_elt.appendChild(text)
    child.appendChild(item_elt)
    text = doc.createTextNode(mail)
    item_elt = doc.createElement("email")
    item_elt.appendChild(text)
    child.appendChild(item_elt)

def update_sources_info(m, dist):
    """Update the XML information with the given Message (Package entry)"""
    global odir, old_done, new_done
    package = m["Package"]
    version = DpkgVersion(m["Version"])
    hash = hash_name(package)
    xml_ok = os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist))
    # Check if the work has already been done
    key = "%s_%s" % (m["Package"], dist)
    # Skip (duplicate) old entries
    if old_done.has_key(key) and version < old_done[key] and xml_ok:
        return
    if new_done.has_key(key) and version <= new_done[key] and xml_ok:
        return
    new_done[key] = version
    # Don't redo the work already done
    if old_done.has_key(key) and old_done[key] == version and xml_ok:
        return
    # Make sure the directory exists
    if not os.path.isdir(odir + "/" + hash + "/" + package):
        if not os.path.isdir(odir + "/" + hash):
            os.mkdir(odir + "/" + hash)
        os.mkdir(odir + "/" + hash + "/" + package)
    # Create the XML DOM object
    doc = implementation.createDocument(None, None, None)
    root = doc.createElement("source")
    doc.appendChild(root)
    root.setAttribute("release", dist)
    if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
        root.setAttribute("nmu", "yes")
    for tag in m.keys():
        child = doc.createElement(tag)
        root.appendChild(child)
        if tag == "binary" or tag[0:5] == "build":
            for item in re.split(",[ \n]*", m[tag]):
                # Take care of non-ascii, prevents troubles...
                text = doc.createTextNode(unicode(item,'ISO-8859-1','replace')) 
                item_elt = doc.createElement("item")
                item_elt.appendChild(text)
                child.appendChild(item_elt)
        elif tag == "maintainer":
            (name, mail) = address_from_string(m[tag])
            add_maintainer_info(child, name, mail, doc)
        elif tag == "uploaders":
            uploaders = addresses_from_string(m[tag])
            for item in uploaders:
                item_elt = doc.createElement("item")
                (name,mail) = item
                add_maintainer_info(item_elt, name, mail, doc)
                child.appendChild(item_elt)
        elif tag == "files":
            for line in string.split(m[tag], "\n"):
                item_elt = doc.createElement("item")
                child.appendChild(item_elt)
                line = line.strip()
                fields = ["md5sum", "size", "filename"]
                values = string.split(line)
                for i in range(len(fields)):
                    new_elt = doc.createElement(fields[i])
                    text = doc.createTextNode(values[i])
                    new_elt.appendChild(text)
                    item_elt.appendChild(new_elt)
        else:
            text = doc.createTextNode(m[tag])
            child.appendChild(text)
        # now compute derived information and store it in the xml
            # add 'parsed' version of VCS info
        if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs-':
            if tag[0] == 'x':
                prefix_len = 6
            else:
                prefix_len = 4
            # e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
            # would be 'browser' for 'vcs-browser' (to be handled specially)
            repos_elt = list(root.getElementsByTagName('repository'))
            if repos_elt:   # reuse existing repository element
                repos_elt = repos_elt[0] # invariant: at most 1 repository elt
            else:   # create a new repository element
                repos_elt = doc.createElement('repository')
                root.appendChild(repos_elt)
            vcs_elt = doc.createElement('vcs')
            kind = tag[prefix_len:]
            if vcs_table.has_key(kind):
                kind = vcs_table[kind][0]
            vcs_elt.setAttribute('kind', kind)
            vcs_elt.setAttribute('url', m[tag])
            repos_elt.appendChild(vcs_elt)
    # Print the DOM object to a file
    try:
        f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
        ext.PrettyPrint(doc, f, "UTF-8")
        f.close()
    except:
        sys.stderr.write("Output problem for " + m["package"] + "\n");

def treat_sources_file(fname, dist):
    """Scan the given Sources file and treat each Package entry"""
    f = open(fname, "r")
    while 1:
        try:
            m = rfc822.Message(f)
            if len(m) == 0: #eof
                break
            update_sources_info(m, dist)
        except EOFError:
            break
    f.close()

# Load the list of sources generated the last time
old_done = {}
new_done = {}
if os.path.exists(odir + "/sources_done"):
    f = open(odir + "/sources_done", "r")
    old_done = cPickle.load(f)
    f.close()

distros = ['oldstable', 'stable', 'testing', 'unstable', 'experimental']
distros.extend(['stable-proposed-updates', 'testing-proposed-updates'])
distros.extend(['security-oldstable', 'security-stable', 'security-testing'])
distros.extend(['volatile', 'mentors'])

for distro in distros:
    for comp in ['main', 'contrib', 'non-free']:
        treat_sources_file(os.path.join(dir, 'Sources-%s_%s' % (distro, comp)),
            distro)

# Store the list of sources generated
f = open(odir + "/sources_done", "w")
cPickle.dump(new_done, f, 0)
f.close()

# Scan the old package/distribution that existed and check if they
# still exist ... if they don't, remove the associated xml file.
for key in old_done.keys():
    (p, d) = key.split("_", 1)
    if not new_done.has_key(key):
        hash = hash_name(p)
        filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
        filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
        if os.path.exists(filename):
            os.unlink(filename)
            f = open(filenamerebuild, "w")
            f.close()

# We're done
1	#!/usr/bin/python
2	# -- coding: utf-8 --
3
4	# vim: expandtab
5
6	# Copyright 2002 Raphaël Hertzog
7	# This file is distributed under the terms of the General Public License
8	# version 2 or (at your option) any later version.
9
10	import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
11	from xml.dom import implementation, ext
12
13	from config import dir, odir
14	from common import vcs_table, hash_name, DpkgVersion
15
16	# address_from_string takes an address in RFC822 format
17	# and turns it into a tuple of the form (real name, email).
18	# The difference with email.Utils.parseaddr and rfc822.parseaddr
19	# is that this routine allows unquoted comma's to appear in the real name
20	# (in violation of RFC822). This is a hack to allow a Maintainer field to
21	# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
22	def address_from_string(content):
23	hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
24	(name, mail) = email.Utils.parseaddr(hacked_content)
25	return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))
26
27	# addresses_from_string takes a string with addresses in RFC822 format
28	# and changes it into a list of tuples of the form (real name, email).
29	# Just as address_from_string, it tries to be forgiving about unquoted
30	# commas in addresses. [PvR]
31	def addresses_from_string(content):
32	pattern = re.compile("([^>]),")
33	hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
34	msg = email.message_from_string("Header: " + hacked_content)
35	hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
36	list = map(lambda p:
37	map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
38	hacked_list)
39	return list
40
41	def add_maintainer_info(child, name, mail, doc):
42	text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
43	item_elt = doc.createElement("name")
44	item_elt.appendChild(text)
45	child.appendChild(item_elt)
46	text = doc.createTextNode(mail)
47	item_elt = doc.createElement("email")
48	item_elt.appendChild(text)
49	child.appendChild(item_elt)
50
51	def update_sources_info(m, dist):
52	"""Update the XML information with the given Message (Package entry)"""
53	global odir, old_done, new_done
54	package = m["Package"]
55	version = DpkgVersion(m["Version"])
56	hash = hash_name(package)
57	xml_ok = os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist))
58	# Check if the work has already been done
59	key = "%s_%s" % (m["Package"], dist)
60	# Skip (duplicate) old entries
61	if old_done.has_key(key) and version < old_done[key] and xml_ok:
62	return
63	if new_done.has_key(key) and version <= new_done[key] and xml_ok:
64	return
65	new_done[key] = version
66	# Don't redo the work already done
67	if old_done.has_key(key) and old_done[key] == version and xml_ok:
68	return
69	# Make sure the directory exists
70	if not os.path.isdir(odir + "/" + hash + "/" + package):
71	if not os.path.isdir(odir + "/" + hash):
72	os.mkdir(odir + "/" + hash)
73	os.mkdir(odir + "/" + hash + "/" + package)
74	# Create the XML DOM object
75	doc = implementation.createDocument(None, None, None)
76	root = doc.createElement("source")
77	doc.appendChild(root)
78	root.setAttribute("release", dist)
79	if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
80	root.setAttribute("nmu", "yes")
81	for tag in m.keys():
82	child = doc.createElement(tag)
83	root.appendChild(child)
84	if tag == "binary" or tag[0:5] == "build":
85	for item in re.split(",[ \n]*", m[tag]):
86	# Take care of non-ascii, prevents troubles...
87	text = doc.createTextNode(unicode(item,'ISO-8859-1','replace'))
88	item_elt = doc.createElement("item")
89	item_elt.appendChild(text)
90	child.appendChild(item_elt)
91	elif tag == "maintainer":
92	(name, mail) = address_from_string(m[tag])
93	add_maintainer_info(child, name, mail, doc)
94	elif tag == "uploaders":
95	uploaders = addresses_from_string(m[tag])
96	for item in uploaders:
97	item_elt = doc.createElement("item")
98	(name,mail) = item
99	add_maintainer_info(item_elt, name, mail, doc)
100	child.appendChild(item_elt)
101	elif tag == "files":
102	for line in string.split(m[tag], "\n"):
103	item_elt = doc.createElement("item")
104	child.appendChild(item_elt)
105	line = line.strip()
106	fields = ["md5sum", "size", "filename"]
107	values = string.split(line)
108	for i in range(len(fields)):
109	new_elt = doc.createElement(fields[i])
110	text = doc.createTextNode(values[i])
111	new_elt.appendChild(text)
112	item_elt.appendChild(new_elt)
113	else:
114	text = doc.createTextNode(m[tag])
115	child.appendChild(text)
116	# now compute derived information and store it in the xml
117	# add 'parsed' version of VCS info
118	if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs-':
119	if tag[0] == 'x':
120	prefix_len = 6
121	else:
122	prefix_len = 4
123	# e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
124	# would be 'browser' for 'vcs-browser' (to be handled specially)
125	repos_elt = list(root.getElementsByTagName('repository'))
126	if repos_elt: # reuse existing repository element
127	repos_elt = repos_elt[0] # invariant: at most 1 repository elt
128	else: # create a new repository element
129	repos_elt = doc.createElement('repository')
130	root.appendChild(repos_elt)
131	vcs_elt = doc.createElement('vcs')
132	kind = tag[prefix_len:]
133	if vcs_table.has_key(kind):
134	kind = vcs_table[kind][0]
135	vcs_elt.setAttribute('kind', kind)
136	vcs_elt.setAttribute('url', m[tag])
137	repos_elt.appendChild(vcs_elt)
138	# Print the DOM object to a file
139	try:
140	f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
141	ext.PrettyPrint(doc, f, "UTF-8")
142	f.close()
143	except:
144	sys.stderr.write("Output problem for " + m["package"] + "\n");
145
146	def treat_sources_file(fname, dist):
147	"""Scan the given Sources file and treat each Package entry"""
148	f = open(fname, "r")
149	while 1:
150	try:
151	m = rfc822.Message(f)
152	if len(m) == 0: #eof
153	break
154	update_sources_info(m, dist)
155	except EOFError:
156	break
157	f.close()
158
159	# Load the list of sources generated the last time
160	old_done = {}
161	new_done = {}
162	if os.path.exists(odir + "/sources_done"):
163	f = open(odir + "/sources_done", "r")
164	old_done = cPickle.load(f)
165	f.close()
166
167	distros = ['oldstable', 'stable', 'testing', 'unstable', 'experimental']
168	distros.extend(['stable-proposed-updates', 'testing-proposed-updates'])
169	distros.extend(['security-oldstable', 'security-stable', 'security-testing'])
170	distros.extend(['volatile', 'mentors'])
171
172	for distro in distros:
173	for comp in ['main', 'contrib', 'non-free']:
174	treat_sources_file(os.path.join(dir, 'Sources-%s_%s' % (distro, comp)),
175	distro)
176
177	# Store the list of sources generated
178	f = open(odir + "/sources_done", "w")
179	cPickle.dump(new_done, f, 0)
180	f.close()
181
182	# Scan the old package/distribution that existed and check if they
183	# still exist ... if they don't, remove the associated xml file.
184	for key in old_done.keys():
185	(p, d) = key.split("_", 1)
186	if not new_done.has_key(key):
187	hash = hash_name(p)
188	filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
189	filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
190	if os.path.exists(filename):
191	os.unlink(filename)
192	f = open(filenamerebuild, "w")
193	f.close()
194
195	# We're done
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision