www/bin/sources_to_xml.py

#!/usr/bin/python
# -*- coding: utf8 -*-

# vim: expandtab

# Copyright 2002 Raphaël Hertzog
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
from xml.dom import implementation, ext

from config import dir, odir

# address_from_string takes an address in RFC822 format 
# and turns it into a tuple of the form (real name, email).
# The difference with email.Utils.parseaddr  and rfc822.parseaddr 
# is that this routine allows unquoted comma's to appear in the real name
# (in violation of RFC822). This is a hack to allow a Maintainer field to
# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
def address_from_string(content):
    hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
    (name, mail) = email.Utils.parseaddr(hacked_content)
    return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))

# addresses_from_string takes a string with addresses in RFC822 format
# and changes it into a list of tuples of the form (real name, email). 
# Just as address_from_string, it tries to be forgiving about unquoted
# commas in addresses. [PvR]
def addresses_from_string(content):
    pattern = re.compile("([^>]),")
    hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
    msg = email.message_from_string("Header: " + hacked_content)
    hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
    list = map(lambda p:
               map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
               hacked_list)
    return list

def add_maintainer_info(child, name, mail, doc):
    text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
    item_elt = doc.createElement("name")
    item_elt.appendChild(text)
    child.appendChild(item_elt)
    text = doc.createTextNode(mail)
    item_elt = doc.createElement("email")
    item_elt.appendChild(text)
    child.appendChild(item_elt)

def update_sources_info(m, dist):
    """Update the XML information with the given Message (Package entry)"""
    global odir, old_done, new_done, new_dist_map
    package = m["Package"]
    hash = package[0]
    if package[0:3] == "lib":
        hash = package[0:4]
    # Check if the work has already been done
    key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
    new_done[key] = 1
    new_dist_map["%s_%s" % (m["Package"], dist)] = 1
    if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
        return
    # Make sure the directory exists
    if not os.path.isdir(odir + "/" + hash + "/" + package):
        if not os.path.isdir(odir + "/" + hash):
            os.mkdir(odir + "/" + hash)
        os.mkdir(odir + "/" + hash + "/" + package)
    # Create the XML DOM object
    doc = implementation.createDocument(None, None, None)
    root = doc.createElement("source")
    doc.appendChild(root)
    root.setAttribute("release", dist)
    if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
        root.setAttribute("nmu", "yes")
    for tag in m.keys():
        child = doc.createElement(tag)
        root.appendChild(child)
        if tag == "binary" or tag[0:5] == "build":
            for item in re.split(", ?", m[tag]):
                # Take care of non-ascii, prevents troubles...
                text = doc.createTextNode(unicode(item,'ISO-8859-1','replace')) 
                item_elt = doc.createElement("item")
                item_elt.appendChild(text)
                child.appendChild(item_elt)
        elif tag == "maintainer":
            (name, mail) = address_from_string(m[tag])
            add_maintainer_info(child, name, mail, doc)
        elif tag == "uploaders":
            uploaders = addresses_from_string(m[tag])
            for item in uploaders:
                item_elt = doc.createElement("item")
                (name,mail) = item
                add_maintainer_info(item_elt, name, mail, doc)
                child.appendChild(item_elt)
        elif tag == "files":
            for line in string.split(m[tag], "\n"):
                item_elt = doc.createElement("item")
                child.appendChild(item_elt)
                line = line.strip()
                fields = ["md5sum", "size", "filename"]
                values = string.split(line)
                for i in range(len(fields)):
                    new_elt = doc.createElement(fields[i])
                    text = doc.createTextNode(values[i])
                    new_elt.appendChild(text)
                    item_elt.appendChild(new_elt)
        else:
            text = doc.createTextNode(m[tag])
            child.appendChild(text)
    # Print the DOM object to a file
    try:
        f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
        ext.PrettyPrint(doc, f, "UTF-8")
        f.close()
    except:
        sys.stderr.write("Output problem for " + m["package"] + "\n");

def treat_sources_file(f, dist):
    """Scan the given Sources file and treat each Package entry"""
    while 1:
        try:
            m = rfc822.Message(f)
            if len(m) == 0: #eof
                break
            update_sources_info(m, dist)
        except EOFError:
            break

# Load the list of sources generated the last time
old_done = {}
new_done = {}
new_dist_map = {}
if os.path.exists(odir + "/sources_done"):
    f = open(odir + "/sources_done", "r")
    old_done = cPickle.load(f)
    f.close()

for comp in ["main", "contrib", "non-free"]:
    for dist in ["oldstable", "stable", "testing", "unstable"]:
        f = open(dir + "/Sources_%s_%s" % (dist, comp), "r")
        treat_sources_file(f, dist) 
        f.close()
    # Experimental
    f = open(dir + "/Sources-experimental_" + comp, "r")
    treat_sources_file(f,  "experimental") 
    f.close()
    # s-p-u and t-p-u
    f = open(dir + "/Sources-spu_%s" % comp, "r")
    treat_sources_file(f, "s-p-u") 
    f.close()
    f = open(dir + "/Sources-tpu_%s" % comp, "r")
    treat_sources_file(f, "t-p-u") 
    f.close()
    # security
    f = open(dir + "/Sources-security-oldstable_%s" % comp, "r")
    treat_sources_file(f, "oldstable-security") 
    f.close()
    f = open(dir + "/Sources-security-stable_%s" % comp, "r")
    treat_sources_file(f, "stable-security") 
    f.close()
    #f = open(dir + "/Sources-security-testing_%s" % comp, "r")
    #treat_sources_file(f, "testing-security") 
    #f.close()
    f = open(dir + "/Sources-secure-testing_%s" % comp, "r")
    treat_sources_file(f, "secure-testing") 
    f.close()
    f = open(dir + "/Sources-volatile_%s" % comp, "r")
    treat_sources_file(f, "volatile") 
    f.close()

# Store the list of sources generated
f = open(odir + "/sources_done", "w")
cPickle.dump(new_done, f, 0)
f.close()

# Scan the old package/distribution that existed and check if they
# still exist ... if they don't, remove the associated xml file.
for key in old_done.keys():
    (p, v, d) = key.split("_", 2)
    if not new_dist_map.has_key("%s_%s" % (p, d)):
        hash = p[0]
        if p[0:3] == "lib":
            hash = p[0:4]
        filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
        filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
        if os.path.exists(filename):
            os.unlink(filename)
            f = open(filenamerebuild, "w")
            f.close()

# We're done
1	jeroen	1162	#!/usr/bin/python
2	jeroen	1154	# -- coding: utf8 --
3	jeroen	1162
4			# vim: expandtab
5
6			# Copyright 2002 Raphaël Hertzog
7	hertzog	351	# This file is distributed under the terms of the General Public License
8			# version 2 or (at your option) any later version.
9
10	hertzog	570	import os.path, rfc822, email, email.Utils, sys, string, re, cPickle
11	hertzog	344	from xml.dom import implementation, ext
12
13			from config import dir, odir
14
15	hertzog	570	# address_from_string takes an address in RFC822 format
16			# and turns it into a tuple of the form (real name, email).
17			# The difference with email.Utils.parseaddr and rfc822.parseaddr
18			# is that this routine allows unquoted comma's to appear in the real name
19			# (in violation of RFC822). This is a hack to allow a Maintainer field to
20			# be like 'Maintainer: John H. Robinson, IV <jaqque@debian.org>'. [PvR]
21			def address_from_string(content):
22			hacked_content = string.replace(content, ",", "WEWANTNOCOMMAS")
23			(name, mail) = email.Utils.parseaddr(hacked_content)
24			return (string.replace(name,"WEWANTNOCOMMAS",","),string.replace(mail,"WEWANTNOCOMMAS",","))
25
26			# addresses_from_string takes a string with addresses in RFC822 format
27	hertzog	571	# and changes it into a list of tuples of the form (real name, email).
28			# Just as address_from_string, it tries to be forgiving about unquoted
29			# commas in addresses. [PvR]
30	hertzog	570	def addresses_from_string(content):
31	hertzog	571	pattern = re.compile("([^>]),")
32			hacked_content = pattern.sub("\\1WEWANTNOCOMMAS", content)
33			msg = email.message_from_string("Header: " + hacked_content)
34			hacked_list = email.Utils.getaddresses(msg.get_all("Header", []))
35			list = map(lambda p:
36			map(lambda s:string.replace(s,"WEWANTNOCOMMAS",","), p),
37			hacked_list)
38			return list
39	hertzog	570
40			def add_maintainer_info(child, name, mail, doc):
41	hertzog	812	text = doc.createTextNode(unicode(name,'UTF8','replace')) # Take care of non-ascii
42	hertzog	344	item_elt = doc.createElement("name")
43			item_elt.appendChild(text)
44			child.appendChild(item_elt)
45	hertzog	570	text = doc.createTextNode(mail)
46	hertzog	344	item_elt = doc.createElement("email")
47			item_elt.appendChild(text)
48			child.appendChild(item_elt)
49
50			def update_sources_info(m, dist):
51			"""Update the XML information with the given Message (Package entry)"""
52	hertzog	357	global odir, old_done, new_done, new_dist_map
53	hertzog	812	package = m["Package"]
54			hash = package[0]
55			if package[0:3] == "lib":
56			hash = package[0:4]
57	hertzog	357	# Check if the work has already been done
58			key = "%s_%s_%s" % (m["Package"], m["Version"], dist)
59			new_done[key] = 1
60			new_dist_map["%s_%s" % (m["Package"], dist)] = 1
61	hertzog	812	if old_done.has_key(key) and os.path.isfile("%s/%s/%s/%s.xml" % (odir, hash, package, dist)):
62	hertzog	357	return
63	hertzog	344	# Make sure the directory exists
64			if not os.path.isdir(odir + "/" + hash + "/" + package):
65			if not os.path.isdir(odir + "/" + hash):
66			os.mkdir(odir + "/" + hash)
67			os.mkdir(odir + "/" + hash + "/" + package)
68			# Create the XML DOM object
69			doc = implementation.createDocument(None, None, None)
70			root = doc.createElement("source")
71			doc.appendChild(root)
72	hertzog	357	root.setAttribute("release", dist)
73	hertzog	344	if re.search("-\d+\.\d+(\.\d+)?$", m["version"]):
74			root.setAttribute("nmu", "yes")
75			for tag in m.keys():
76			child = doc.createElement(tag)
77			root.appendChild(child)
78			if tag == "binary" or tag[0:5] == "build":
79			for item in re.split(", ?", m[tag]):
80	jeroen	906	# Take care of non-ascii, prevents troubles...
81			text = doc.createTextNode(unicode(item,'ISO-8859-1','replace'))
82	hertzog	344	item_elt = doc.createElement("item")
83			item_elt.appendChild(text)
84			child.appendChild(item_elt)
85			elif tag == "maintainer":
86	hertzog	570	(name, mail) = address_from_string(m[tag])
87			add_maintainer_info(child, name, mail, doc)
88	hertzog	344	elif tag == "uploaders":
89	hertzog	570	uploaders = addresses_from_string(m[tag])
90			for item in uploaders:
91	hertzog	344	item_elt = doc.createElement("item")
92	hertzog	570	(name,mail) = item
93			add_maintainer_info(item_elt, name, mail, doc)
94	hertzog	344	child.appendChild(item_elt)
95			elif tag == "files":
96			for line in string.split(m[tag], "\n"):
97			item_elt = doc.createElement("item")
98			child.appendChild(item_elt)
99			line = line.strip()
100			fields = ["md5sum", "size", "filename"]
101			values = string.split(line)
102			for i in range(len(fields)):
103			new_elt = doc.createElement(fields[i])
104			text = doc.createTextNode(values[i])
105			new_elt.appendChild(text)
106			item_elt.appendChild(new_elt)
107			else:
108			text = doc.createTextNode(m[tag])
109			child.appendChild(text)
110			# Print the DOM object to a file
111			try:
112			f = open("%s/%s/%s/%s.xml" % (odir, hash, package, dist), "w")
113	hertzog	812	ext.PrettyPrint(doc, f, "UTF-8")
114	hertzog	344	f.close()
115			except:
116	tbm	680	sys.stderr.write("Output problem for " + m["package"] + "\n");
117	hertzog	344
118			def treat_sources_file(f, dist):
119			"""Scan the given Sources file and treat each Package entry"""
120			while 1:
121			try:
122			m = rfc822.Message(f)
123			if len(m) == 0: #eof
124			break
125			update_sources_info(m, dist)
126			except EOFError:
127			break
128
129	hertzog	357	# Load the list of sources generated the last time
130			old_done = {}
131			new_done = {}
132			new_dist_map = {}
133			if os.path.exists(odir + "/sources_done"):
134			f = open(odir + "/sources_done", "r")
135			old_done = cPickle.load(f)
136			f.close()
137	hertzog	344
138			for comp in ["main", "contrib", "non-free"]:
139	jeroen	1020	for dist in ["oldstable", "stable", "testing", "unstable"]:
140	hertzog	344	f = open(dir + "/Sources_%s_%s" % (dist, comp), "r")
141			treat_sources_file(f, dist)
142			f.close()
143			# Experimental
144			f = open(dir + "/Sources-experimental_" + comp, "r")
145			treat_sources_file(f, "experimental")
146			f.close()
147	hertzog	351	# s-p-u and t-p-u
148			f = open(dir + "/Sources-spu_%s" % comp, "r")
149			treat_sources_file(f, "s-p-u")
150			f.close()
151			f = open(dir + "/Sources-tpu_%s" % comp, "r")
152			treat_sources_file(f, "t-p-u")
153			f.close()
154			# security
155	jeroen	1020	f = open(dir + "/Sources-security-oldstable_%s" % comp, "r")
156	jeroen	1029	treat_sources_file(f, "oldstable-security")
157	jeroen	1020	f.close()
158	hertzog	351	f = open(dir + "/Sources-security-stable_%s" % comp, "r")
159			treat_sources_file(f, "stable-security")
160			f.close()
161	jeroen	1025	#f = open(dir + "/Sources-security-testing_%s" % comp, "r")
162			#treat_sources_file(f, "testing-security")
163			#f.close()
164	hertzog	1090	f = open(dir + "/Sources-secure-testing_%s" % comp, "r")
165			treat_sources_file(f, "secure-testing")
166			f.close()
167			f = open(dir + "/Sources-volatile_%s" % comp, "r")
168			treat_sources_file(f, "volatile")
169			f.close()
170	hertzog	344
171	hertzog	357	# Store the list of sources generated
172			f = open(odir + "/sources_done", "w")
173			cPickle.dump(new_done, f, 0)
174			f.close()
175
176			# Scan the old package/distribution that existed and check if they
177			# still exist ... if they don't, remove the associated xml file.
178			for key in old_done.keys():
179			(p, v, d) = key.split("_", 2)
180			if not new_dist_map.has_key("%s_%s" % (p, d)):
181			hash = p[0]
182			if p[0:3] == "lib":
183			hash = p[0:4]
184			filename = "%s/%s/%s/%s.xml" % (odir, hash, p, d)
185	hertzog	400	filenamerebuild = "%s/%s/%s/force-rebuild" % (odir, hash, p)
186	hertzog	357	if os.path.exists(filename):
187			os.unlink(filename)
188	hertzog	400	f = open(filenamerebuild, "w")
189			f.close()
190	hertzog	357
191			# We're done
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision