www/bin/sources_to_xml.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

# vim: expandtab

# Copyright 2002 Raphaël Hertzog
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
from xml.dom import implementation, ext

from config import dir, odir
from common import vcs_table, hash_name

# address_from_string takes an address in RFC822 format 
# and turns it into a tuple of the form (real name, email).
# The difference with email.Utils.parseaddr  and rfc822.parseaddr 
# is that this routine allows unquoted comma's to appear in the real name
# (in violation of RFC822). This is a hack to allow a Maintainer field to
# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
def address_from_string(content):
    hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
    (name, mail) = email.Utils.parseaddr(hacked_content)
    return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))

# addresses_from_string takes a string with addresses in RFC822 format
# and changes it into a list of tuples of the form (real name, email). 
# Just as address_from_string, it tries to be forgiving about unquoted
# commas in addresses. [PvR]
def addresses_from_string(content):
    pattern = re.compile("([^>]),")
    hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
    msg = email.message_from_string("Header: " + hacked_content)
    hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
    list = map(lambda p:
               map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
               hacked_list)
    return list

def add_maintainer_info(child, name, mail, doc):
    text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
    item_elt = doc.createElement("name")
    item_elt.appendChild(text)
    child.appendChild(item_elt)
    text = doc.createTextNode(mail)
    item_elt = doc.createElement("email")
    item_elt.appendChild(text)
    child.appendChild(item_elt)

def update_sources_info(m, dist):
    """Update the XML information with the given Message (Package entry)"""
    global odir, old_done, new_done, new_dist_map
    package = m["Package"]
    hash = hash_name(package)
    # Check if the work has already been done
    key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
    new_done[key] = 1
    new_dist_map["%s_%s" % (m["Package"], dist)] = 1
    if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
        return
    # Make sure the directory exists
    if not os.path.isdir(odir + "/" + hash + "/" + package):
        if not os.path.isdir(odir + "/" + hash):
            os.mkdir(odir + "/" + hash)
        os.mkdir(odir + "/" + hash + "/" + package)
    # Create the XML DOM object
    doc = implementation.createDocument(None, None, None)
    root = doc.createElement("source")
    doc.appendChild(root)
    root.setAttribute("release", dist)
    if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
        root.setAttribute("nmu", "yes")
    for tag in m.keys():
        child = doc.createElement(tag)
        root.appendChild(child)
        if tag == "binary" or tag[0:5] == "build":
            for item in re.split(", ?", m[tag]):
                # Take care of non-ascii, prevents troubles...
                text = doc.createTextNode(unicode(item,'ISO-8859-1','replace')) 
                item_elt = doc.createElement("item")
                item_elt.appendChild(text)
                child.appendChild(item_elt)
        elif tag == "maintainer":
            (name, mail) = address_from_string(m[tag])
            add_maintainer_info(child, name, mail, doc)
        elif tag == "uploaders":
            uploaders = addresses_from_string(m[tag])
            for item in uploaders:
                item_elt = doc.createElement("item")
                (name,mail) = item
                add_maintainer_info(item_elt, name, mail, doc)
                child.appendChild(item_elt)
        elif tag == "files":
            for line in string.split(m[tag], "\n"):
                item_elt = doc.createElement("item")
                child.appendChild(item_elt)
                line = line.strip()
                fields = ["md5sum", "size", "filename"]
                values = string.split(line)
                for i in range(len(fields)):
                    new_elt = doc.createElement(fields[i])
                    text = doc.createTextNode(values[i])
                    new_elt.appendChild(text)
                    item_elt.appendChild(new_elt)
        else:
            text = doc.createTextNode(m[tag])
            child.appendChild(text)
        # now compute derived information and store it in the xml
            # add 'parsed' version of VCS info
        if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs-':
            if tag[0] == 'x':
                prefix_len = 6
            else:
                prefix_len = 4
            # e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
            # would be 'browser' for 'vcs-browser' (to be handled specially)
            repos_elt = list(root.getElementsByTagName('repository'))
            if repos_elt:   # reuse existing repository element
                repos_elt = repos_elt[0] # invariant: at most 1 repository elt
            else:   # create a new repository element
                repos_elt = doc.createElement('repository')
                root.appendChild(repos_elt)
            vcs_elt = doc.createElement('vcs')
            kind = tag[prefix_len:]
            if vcs_table.has_key(kind):
                kind = vcs_table[kind][0]
            vcs_elt.setAttribute('kind', kind)
            vcs_elt.setAttribute('url', m[tag])
            repos_elt.appendChild(vcs_elt)
    # Print the DOM object to a file
    try:
        f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
        ext.PrettyPrint(doc, f, "UTF-8")
        f.close()
    except:
        sys.stderr.write("Output problem for " + m["package"] + "\n");

def treat_sources_file(fname, dist):
    """Scan the given Sources file and treat each Package entry"""
    f = open(fname, "r")
    while 1:
        try:
            m = rfc822.Message(f)
            if len(m) == 0: #eof
                break
            update_sources_info(m, dist)
        except EOFError:
            break
    f.close()

# Load the list of sources generated the last time
old_done = {}
new_done = {}
new_dist_map = {}
if os.path.exists(odir + "/sources_done"):
    f = open(odir + "/sources_done", "r")
    old_done = cPickle.load(f)
    f.close()

distros = ['oldstable', 'stable', 'testing', 'unstable', 'experimental']
distros.extend(['stable-proposed-updates', 'testing-proposed-updates'])
distros.extend(['security-oldstable', 'security-stable'])
distros.extend(['volatile', 'mentors'])

for distro in distros:
    for comp in ['main', 'contrib', 'non-free']:
        treat_sources_file(os.path.join(dir, 'Sources-%s_%s' % (distro, comp)),
            distro)

# Store the list of sources generated
f = open(odir + "/sources_done", "w")
cPickle.dump(new_done, f, 0)
f.close()

# Scan the old package/distribution that existed and check if they
# still exist ... if they don't, remove the associated xml file.
for key in old_done.keys():
    (p, v, d) = key.split("_", 2)
    if not new_dist_map.has_key("%s_%s" % (p, d)):
        hash = hash_name(p)
        filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
        filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
        if os.path.exists(filename):
            os.unlink(filename)
            f = open(filenamerebuild, "w")
            f.close()

# We're done
1	#!/usr/bin/python
2	# -- coding: utf-8 --
3
4	# vim: expandtab
5
6	# Copyright 2002 Raphaël Hertzog
7	# This file is distributed under the terms of the General Public License
8	# version 2 or (at your option) any later version.
9
10	import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
11	from xml.dom import implementation, ext
12
13	from config import dir, odir
14	from common import vcs_table, hash_name
15
16	# address_from_string takes an address in RFC822 format
17	# and turns it into a tuple of the form (real name, email).
18	# The difference with email.Utils.parseaddr and rfc822.parseaddr
19	# is that this routine allows unquoted comma's to appear in the real name
20	# (in violation of RFC822). This is a hack to allow a Maintainer field to
21	# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
22	def address_from_string(content):
23	hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
24	(name, mail) = email.Utils.parseaddr(hacked_content)
25	return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))
26
27	# addresses_from_string takes a string with addresses in RFC822 format
28	# and changes it into a list of tuples of the form (real name, email).
29	# Just as address_from_string, it tries to be forgiving about unquoted
30	# commas in addresses. [PvR]
31	def addresses_from_string(content):
32	pattern = re.compile("([^>]),")
33	hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
34	msg = email.message_from_string("Header: " + hacked_content)
35	hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
36	list = map(lambda p:
37	map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
38	hacked_list)
39	return list
40
41	def add_maintainer_info(child, name, mail, doc):
42	text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
43	item_elt = doc.createElement("name")
44	item_elt.appendChild(text)
45	child.appendChild(item_elt)
46	text = doc.createTextNode(mail)
47	item_elt = doc.createElement("email")
48	item_elt.appendChild(text)
49	child.appendChild(item_elt)
50
51	def update_sources_info(m, dist):
52	"""Update the XML information with the given Message (Package entry)"""
53	global odir, old_done, new_done, new_dist_map
54	package = m["Package"]
55	hash = hash_name(package)
56	# Check if the work has already been done
57	key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
58	new_done[key] = 1
59	new_dist_map["%s_%s" % (m["Package"], dist)] = 1
60	if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
61	return
62	# Make sure the directory exists
63	if not os.path.isdir(odir + "/" + hash + "/" + package):
64	if not os.path.isdir(odir + "/" + hash):
65	os.mkdir(odir + "/" + hash)
66	os.mkdir(odir + "/" + hash + "/" + package)
67	# Create the XML DOM object
68	doc = implementation.createDocument(None, None, None)
69	root = doc.createElement("source")
70	doc.appendChild(root)
71	root.setAttribute("release", dist)
72	if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
73	root.setAttribute("nmu", "yes")
74	for tag in m.keys():
75	child = doc.createElement(tag)
76	root.appendChild(child)
77	if tag == "binary" or tag[0:5] == "build":
78	for item in re.split(", ?", m[tag]):
79	# Take care of non-ascii, prevents troubles...
80	text = doc.createTextNode(unicode(item,'ISO-8859-1','replace'))
81	item_elt = doc.createElement("item")
82	item_elt.appendChild(text)
83	child.appendChild(item_elt)
84	elif tag == "maintainer":
85	(name, mail) = address_from_string(m[tag])
86	add_maintainer_info(child, name, mail, doc)
87	elif tag == "uploaders":
88	uploaders = addresses_from_string(m[tag])
89	for item in uploaders:
90	item_elt = doc.createElement("item")
91	(name,mail) = item
92	add_maintainer_info(item_elt, name, mail, doc)
93	child.appendChild(item_elt)
94	elif tag == "files":
95	for line in string.split(m[tag], "\n"):
96	item_elt = doc.createElement("item")
97	child.appendChild(item_elt)
98	line = line.strip()
99	fields = ["md5sum", "size", "filename"]
100	values = string.split(line)
101	for i in range(len(fields)):
102	new_elt = doc.createElement(fields[i])
103	text = doc.createTextNode(values[i])
104	new_elt.appendChild(text)
105	item_elt.appendChild(new_elt)
106	else:
107	text = doc.createTextNode(m[tag])
108	child.appendChild(text)
109	# now compute derived information and store it in the xml
110	# add 'parsed' version of VCS info
111	if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs-':
112	if tag[0] == 'x':
113	prefix_len = 6
114	else:
115	prefix_len = 4
116	# e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
117	# would be 'browser' for 'vcs-browser' (to be handled specially)
118	repos_elt = list(root.getElementsByTagName('repository'))
119	if repos_elt: # reuse existing repository element
120	repos_elt = repos_elt[0] # invariant: at most 1 repository elt
121	else: # create a new repository element
122	repos_elt = doc.createElement('repository')
123	root.appendChild(repos_elt)
124	vcs_elt = doc.createElement('vcs')
125	kind = tag[prefix_len:]
126	if vcs_table.has_key(kind):
127	kind = vcs_table[kind][0]
128	vcs_elt.setAttribute('kind', kind)
129	vcs_elt.setAttribute('url', m[tag])
130	repos_elt.appendChild(vcs_elt)
131	# Print the DOM object to a file
132	try:
133	f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
134	ext.PrettyPrint(doc, f, "UTF-8")
135	f.close()
136	except:
137	sys.stderr.write("Output problem for " + m["package"] + "\n");
138
139	def treat_sources_file(fname, dist):
140	"""Scan the given Sources file and treat each Package entry"""
141	f = open(fname, "r")
142	while 1:
143	try:
144	m = rfc822.Message(f)
145	if len(m) == 0: #eof
146	break
147	update_sources_info(m, dist)
148	except EOFError:
149	break
150	f.close()
151
152	# Load the list of sources generated the last time
153	old_done = {}
154	new_done = {}
155	new_dist_map = {}
156	if os.path.exists(odir + "/sources_done"):
157	f = open(odir + "/sources_done", "r")
158	old_done = cPickle.load(f)
159	f.close()
160
161	distros = ['oldstable', 'stable', 'testing', 'unstable', 'experimental']
162	distros.extend(['stable-proposed-updates', 'testing-proposed-updates'])
163	distros.extend(['security-oldstable', 'security-stable'])
164	distros.extend(['volatile', 'mentors'])
165
166	for distro in distros:
167	for comp in ['main', 'contrib', 'non-free']:
168	treat_sources_file(os.path.join(dir, 'Sources-%s_%s' % (distro, comp)),
169	distro)
170
171	# Store the list of sources generated
172	f = open(odir + "/sources_done", "w")
173	cPickle.dump(new_done, f, 0)
174	f.close()
175
176	# Scan the old package/distribution that existed and check if they
177	# still exist ... if they don't, remove the associated xml file.
178	for key in old_done.keys():
179	(p, v, d) = key.split("_", 2)
180	if not new_dist_map.has_key("%s_%s" % (p, d)):
181	hash = hash_name(p)
182	filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
183	filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
184	if os.path.exists(filename):
185	os.unlink(filename)
186	f = open(filenamerebuild, "w")
187	f.close()
188
189	# We're done
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision