www/bin/sources_to_xml.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

# vim: expandtab

# Copyright 2002 Raphaël Hertzog
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
from xml.dom import implementation, ext

from config import dir, odir
from common import vcs_table, hash_name, DpkgVersion

# address_from_string takes an address in RFC822 format 
# and turns it into a tuple of the form (real name, email).
# The difference with email.Utils.parseaddr  and rfc822.parseaddr 
# is that this routine allows unquoted comma's to appear in the real name
# (in violation of RFC822). This is a hack to allow a Maintainer field to
# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
def address_from_string(content):
    hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
    (name, mail) = email.Utils.parseaddr(hacked_content)
    return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))

# addresses_from_string takes a string with addresses in RFC822 format
# and changes it into a list of tuples of the form (real name, email). 
# Just as address_from_string, it tries to be forgiving about unquoted
# commas in addresses. [PvR]
def addresses_from_string(content):
    pattern = re.compile("([^>]),")
    hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
    msg = email.message_from_string("Header: " + hacked_content)
    hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
    list = map(lambda p:
               map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
               hacked_list)
    return list

def add_maintainer_info(child, name, mail, doc):
    text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
    item_elt = doc.createElement("name")
    item_elt.appendChild(text)
    child.appendChild(item_elt)
    text = doc.createTextNode(mail)
    item_elt = doc.createElement("email")
    item_elt.appendChild(text)
    child.appendChild(item_elt)

def update_sources_info(m, dist):
    """Update the XML information with the given Message (Package entry)"""
    global odir, old_done, new_done
    package = m["Package"]
    version = DpkgVersion(m["Version"])
    hash = hash_name(package)
    xml_ok = os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist))
    # Check if the work has already been done
    key = "%s_%s" % (m["Package"], dist)
    # Skip (duplicate) old entries
    if old_done.has_key(key) and version < old_done[key] and xml_ok:
        return
    if new_done.has_key(key) and version <= new_done[key] and xml_ok:
        return
    new_done[key] = version
    # Don't redo the work already done
    if old_done.has_key(key) and old_done[key] == version and xml_ok:
        return
    # Make sure the directory exists
    if not os.path.isdir(odir + "/" + hash + "/" + package):
        if not os.path.isdir(odir + "/" + hash):
            os.mkdir(odir + "/" + hash)
        os.mkdir(odir + "/" + hash + "/" + package)
    # Create the XML DOM object
    doc = implementation.createDocument(None, None, None)
    root = doc.createElement("source")
    doc.appendChild(root)
    root.setAttribute("release", dist)
    if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
        root.setAttribute("nmu", "yes")
    for tag in m.keys():
        child = doc.createElement(tag)
        root.appendChild(child)
        if tag == "binary" or tag[0:5] == "build":
            for item in re.split(",[ \n]*", m[tag]):
                # Take care of non-ascii, prevents troubles...
                text = doc.createTextNode(unicode(item,'ISO-8859-1','replace')) 
                item_elt = doc.createElement("item")
                item_elt.appendChild(text)
                child.appendChild(item_elt)
        elif tag == "maintainer":
            (name, mail) = address_from_string(m[tag])
            add_maintainer_info(child, name, mail, doc)
        elif tag == "uploaders":
            uploaders = addresses_from_string(m[tag])
            for item in uploaders:
                item_elt = doc.createElement("item")
                (name,mail) = item
                add_maintainer_info(item_elt, name, mail, doc)
                child.appendChild(item_elt)
        elif tag == "files":
            for line in string.split(m[tag], "\n"):
                item_elt = doc.createElement("item")
                child.appendChild(item_elt)
                line = line.strip()
                fields = ["md5sum", "size", "filename"]
                values = string.split(line)
                for i in range(len(fields)):
                    new_elt = doc.createElement(fields[i])
                    text = doc.createTextNode(values[i])
                    new_elt.appendChild(text)
                    item_elt.appendChild(new_elt)
        else:
            text = doc.createTextNode(m[tag])
            child.appendChild(text)
        # now compute derived information and store it in the xml
            # add 'parsed' version of VCS info
        if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs-':
            if tag[0] == 'x':
                prefix_len = 6
            else:
                prefix_len = 4
            # e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
            # would be 'browser' for 'vcs-browser' (to be handled specially)
            repos_elt = list(root.getElementsByTagName('repository'))
            if repos_elt:   # reuse existing repository element
                repos_elt = repos_elt[0] # invariant: at most 1 repository elt
            else:   # create a new repository element
                repos_elt = doc.createElement('repository')
                root.appendChild(repos_elt)
            vcs_elt = doc.createElement('vcs')
            kind = tag[prefix_len:]
            if vcs_table.has_key(kind):
                kind = vcs_table[kind][0]
            vcs_elt.setAttribute('kind', kind)
            vcs_elt.setAttribute('url', m[tag])
            repos_elt.appendChild(vcs_elt)
    # Print the DOM object to a file
    try:
        f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
        ext.PrettyPrint(doc, f, "UTF-8")
        f.close()
    except:
        sys.stderr.write("Output problem for " + m["package"] + "\n");

def treat_sources_file(fname, dist):
    """Scan the given Sources file and treat each Package entry"""
    f = open(fname, "r")
    while 1:
        try:
            m = rfc822.Message(f)
            if len(m) == 0: #eof
                break
            update_sources_info(m, dist)
        except EOFError:
            break
    f.close()

# Load the list of sources generated the last time
old_done = {}
new_done = {}
if os.path.exists(odir + "/sources_done"):
    f = open(odir + "/sources_done", "r")
    old_done = cPickle.load(f)
    f.close()

distros = ['oldstable', 'stable', 'testing', 'unstable', 'experimental']
distros.extend(['stable-proposed-updates', 'testing-proposed-updates'])
distros.extend(['security-oldstable', 'security-stable', 'security-testing'])
distros.extend(['mentors'])

for distro in distros:
    for comp in ['main', 'contrib', 'non-free']:
        treat_sources_file(os.path.join(dir, 'Sources-%s_%s' % (distro, comp)),
            distro)

# Store the list of sources generated
f = open(odir + "/sources_done", "w")
cPickle.dump(new_done, f, 0)
f.close()

# Scan the old package/distribution that existed and check if they
# still exist ... if they don't, remove the associated xml file.
for key in old_done.keys():
    (p, d) = key.split("_", 1)
    if not new_done.has_key(key):
        hash = hash_name(p)
        filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
        filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
        if os.path.exists(filename):
            os.unlink(filename)
            f = open(filenamerebuild, "w")
            f.close()

# We're done
1	jeroen	1162	#!/usr/bin/python
2	zack	2049	# -- coding: utf-8 --
3	jeroen	1162
4			# vim: expandtab
5
6			# Copyright 2002 Raphaël Hertzog
7	hertzog	351	# This file is distributed under the terms of the General Public License
8			# version 2 or (at your option) any later version.
9
10	hertzog	570	import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
11	hertzog	344	from xml.dom import implementation, ext
12
13			from config import dir, odir
14	hertzog	2324	from common import vcs_table, hash_name, DpkgVersion
15	hertzog	344
16	hertzog	570	# address_from_string takes an address in RFC822 format
17			# and turns it into a tuple of the form (real name, email).
18			# The difference with email.Utils.parseaddr and rfc822.parseaddr
19			# is that this routine allows unquoted comma's to appear in the real name
20			# (in violation of RFC822). This is a hack to allow a Maintainer field to
21			# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
22			def address_from_string(content):
23			hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
24			(name, mail) = email.Utils.parseaddr(hacked_content)
25			return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))
26
27			# addresses_from_string takes a string with addresses in RFC822 format
28	hertzog	571	# and changes it into a list of tuples of the form (real name, email).
29			# Just as address_from_string, it tries to be forgiving about unquoted
30			# commas in addresses. [PvR]
31	hertzog	570	def addresses_from_string(content):
32	hertzog	571	pattern = re.compile("([^>]),")
33			hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
34			msg = email.message_from_string("Header: " + hacked_content)
35			hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
36			list = map(lambda p:
37			map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
38			hacked_list)
39			return list
40	hertzog	570
41	zack	1773	def add_maintainer_info(child, name, mail, doc):
42	hertzog	812	text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
43	hertzog	344	item_elt = doc.createElement("name")
44			item_elt.appendChild(text)
45			child.appendChild(item_elt)
46	hertzog	570	text = doc.createTextNode(mail)
47	hertzog	344	item_elt = doc.createElement("email")
48			item_elt.appendChild(text)
49			child.appendChild(item_elt)
50
51			def update_sources_info(m, dist):
52			"""Update the XML information with the given Message (Package entry)"""
53	hertzog	2324	global odir, old_done, new_done
54	hertzog	812	package = m["Package"]
55	hertzog	2324	version = DpkgVersion(m["Version"])
56	zack	2049	hash = hash_name(package)
57	hertzog	2324	xml_ok = os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist))
58	hertzog	357	# Check if the work has already been done
59	hertzog	2324	key = "%s_%s" % (m["Package"], dist)
60			# Skip (duplicate) old entries
61			if old_done.has_key(key) and version < old_done[key] and xml_ok:
62	hertzog	357	return
63	hertzog	2324	if new_done.has_key(key) and version <= new_done[key] and xml_ok:
64			return
65			new_done[key] = version
66			# Don't redo the work already done
67			if old_done.has_key(key) and old_done[key] == version and xml_ok:
68			return
69	hertzog	344	# Make sure the directory exists
70			if not os.path.isdir(odir + "/" + hash + "/" + package):
71			if not os.path.isdir(odir + "/" + hash):
72			os.mkdir(odir + "/" + hash)
73			os.mkdir(odir + "/" + hash + "/" + package)
74			# Create the XML DOM object
75			doc = implementation.createDocument(None, None, None)
76			root = doc.createElement("source")
77			doc.appendChild(root)
78	hertzog	357	root.setAttribute("release", dist)
79	hertzog	344	if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
80			root.setAttribute("nmu", "yes")
81			for tag in m.keys():
82			child = doc.createElement(tag)
83			root.appendChild(child)
84			if tag == "binary" or tag[0:5] == "build":
85	hertzog	2273	for item in re.split(",[ \n]*", m[tag]):
86	jeroen	906	# Take care of non-ascii, prevents troubles...
87			text = doc.createTextNode(unicode(item,'ISO-8859-1','replace'))
88	hertzog	344	item_elt = doc.createElement("item")
89			item_elt.appendChild(text)
90			child.appendChild(item_elt)
91			elif tag == "maintainer":
92	hertzog	570	(name, mail) = address_from_string(m[tag])
93	zack	1773	add_maintainer_info(child, name, mail, doc)
94	hertzog	344	elif tag == "uploaders":
95	hertzog	570	uploaders = addresses_from_string(m[tag])
96			for item in uploaders:
97	hertzog	344	item_elt = doc.createElement("item")
98	hertzog	570	(name,mail) = item
99			add_maintainer_info(item_elt, name, mail, doc)
100	hertzog	344	child.appendChild(item_elt)
101			elif tag == "files":
102			for line in string.split(m[tag], "\n"):
103			item_elt = doc.createElement("item")
104			child.appendChild(item_elt)
105			line = line.strip()
106			fields = ["md5sum", "size", "filename"]
107			values = string.split(line)
108			for i in range(len(fields)):
109			new_elt = doc.createElement(fields[i])
110			text = doc.createTextNode(values[i])
111			new_elt.appendChild(text)
112			item_elt.appendChild(new_elt)
113			else:
114			text = doc.createTextNode(m[tag])
115			child.appendChild(text)
116	hertzog	1412	# now compute derived information and store it in the xml
117	hertzog	1427	# add 'parsed' version of VCS info
118	hertzog	1430	if tag[:6] == 'x-vcs-' or tag[:4] == 'vcs-':
119	hertzog	1427	if tag[0] == 'x':
120			prefix_len = 6
121			else:
122			prefix_len = 4
123			# e.g. tag[prefix_len:] would be 'svn' for 'x-vcs-svn'/'vcs-svn'
124	filippo	1500	# would be 'browser' for 'vcs-browser' (to be handled specially)
125			repos_elt = list(root.getElementsByTagName('repository'))
126			if repos_elt: # reuse existing repository element
127			repos_elt = repos_elt[0] # invariant: at most 1 repository elt
128			else: # create a new repository element
129	hertzog	1412	repos_elt = doc.createElement('repository')
130			root.appendChild(repos_elt)
131	filippo	1500	vcs_elt = doc.createElement('vcs')
132			kind = tag[prefix_len:]
133			if vcs_table.has_key(kind):
134			kind = vcs_table[kind][0]
135			vcs_elt.setAttribute('kind', kind)
136			vcs_elt.setAttribute('url', m[tag])
137			repos_elt.appendChild(vcs_elt)
138	hertzog	344	# Print the DOM object to a file
139			try:
140			f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
141	hertzog	812	ext.PrettyPrint(doc, f, "UTF-8")
142	hertzog	344	f.close()
143			except:
144	tbm	680	sys.stderr.write("Output problem for " + m["package"] + "\n");
145	hertzog	344
146	zack	1932	def treat_sources_file(fname, dist):
147	hertzog	344	"""Scan the given Sources file and treat each Package entry"""
148	zack	1932	f = open(fname, "r")
149	hertzog	344	while 1:
150			try:
151			m = rfc822.Message(f)
152			if len(m) == 0: #eof
153			break
154			update_sources_info(m, dist)
155			except EOFError:
156			break
157	zack	1932	f.close()
158	hertzog	344
159	hertzog	357	# Load the list of sources generated the last time
160			old_done = {}
161			new_done = {}
162			if os.path.exists(odir + "/sources_done"):
163			f = open(odir + "/sources_done", "r")
164			old_done = cPickle.load(f)
165			f.close()
166	hertzog	344
167	zack	1953	distros = ['oldstable', 'stable', 'testing', 'unstable', 'experimental']
168			distros.extend(['stable-proposed-updates', 'testing-proposed-updates'])
169	geissert	2389	distros.extend(['security-oldstable', 'security-stable', 'security-testing'])
170	hertzog	2478	distros.extend(['mentors'])
171	hertzog	344
172	zack	1953	for distro in distros:
173			for comp in ['main', 'contrib', 'non-free']:
174			treat_sources_file(os.path.join(dir, 'Sources-%s_%s' % (distro, comp)),
175			distro)
176
177	hertzog	357	# Store the list of sources generated
178			f = open(odir + "/sources_done", "w")
179			cPickle.dump(new_done, f, 0)
180			f.close()
181
182			# Scan the old package/distribution that existed and check if they
183			# still exist ... if they don't, remove the associated xml file.
184			for key in old_done.keys():
185	hertzog	2324	(p, d) = key.split("_", 1)
186			if not new_done.has_key(key):
187	zack	2049	hash = hash_name(p)
188	hertzog	357	filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
189	hertzog	400	filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
190	hertzog	357	if os.path.exists(filename):
191			os.unlink(filename)
192	hertzog	400	f = open(filenamerebuild, "w")
193			f.close()
194	hertzog	357
195			# We're done
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision