udd/udd/ubuntu_bugs_gatherer.py

#!/usr/bin/env python

"""
This script imports the Ubuntu bugs from Launchpad
"""

import sys
from aux import quote
from os import _exit
from gatherer import gatherer
import re
import urllib
from Queue import Queue, Empty
from threading import Thread, currentThread
import time
import httplib
import email

def get_gatherer(connection, config, source):
  return ubuntu_bugs_gatherer(connection, config, source)

class ubuntu_bugs_gatherer(gatherer):
  debug = False

  def __init__(self, connection, config, source):
    gatherer.__init__(self, connection, config, source)

  def run(self):
    my_config = self.my_config
    num_fetchers = 4
    num_writers = 1
    bugs = self.fetch_all_bugs()
    httpq = Queue()
    dbq = Queue()
    for b in bugs:
      if b == 310331:
        continue # this bug is buggy: two identical tasks
#      if self.debug:
#        if b != 310331:
#          continue
      httpq.put(b)

    # start workers
    for i in range(num_fetchers):
      t = Thread(target=self.bugfetcher, name="Fetcher-"+str(i),args=[httpq, dbq])
      t.setDaemon(True)
      t.start()

    c = self.cursor()
    c.execute("delete from ubuntu_bugs_subscribers")
    c.execute("delete from ubuntu_bugs_duplicates")
    c.execute("delete from ubuntu_bugs_tags")
    c.execute("delete from ubuntu_bugs_tasks")
    c.execute("delete from ubuntu_bugs")
    ok = True
    while ok:
      try:
        if self.debug:
          print "HTTPQ: ", httpq.qsize(), " DBQ: ", dbq.qsize()
        d = dbq.get(True, 5) # 10 secs timeout
        self.dbimport(c, d)
      except Empty:
        if httpq.qsize() == 0:
          ok = False
    c.execute("analyze ubuntu_bugs_subscribers")
    c.execute("analyze ubuntu_bugs_duplicates")
    c.execute("analyze ubuntu_bugs_tags")
    c.execute("analyze ubuntu_bugs_tasks")
    c.execute("analyze ubuntu_bugs")

  def fetch_all_bugs(self):
    fh = urllib.urlopen('https://launchpad.net/ubuntu/+bugs-text')
    text = fh.read()
    # convert to a list
    bugs = text.split('\n')
    # remove '', or map() will complain
    bugs.remove('')
    # convert each bug (string) to a int
    bugs = map(int, bugs)
    # sort, so that we can remove duplicates in O(n) later
    bugs.sort()
    # remove duplicates. apparently not in lib, see
    # http://www.python.org/dev/peps/pep-0270/
    # FIXME use set()
    nbugs = []
    on = 0
    for n in bugs:
      if n != on:
        nbugs.append(n)
#      else:
#        print "Duplicate bug: " +  str(n)
      on = n
    fh.close()
    #print nbugs
    return nbugs

  # "worker". Fetch a specific bug as text from launchpad.
  def bugfetcher(self, hq, dq):
    while True:
      conn = httplib.HTTPSConnection('bugs.launchpad.net')
      ok = True
      b = None
      while ok:
        try:
          b = hq.get(False)
        except Empty:
          return
        except:
          print "Other exception raised in bugfetcher. exiting."
          _exit(1)

        try:
          conn.request('GET', 'https://launchpad.net/bugs/' + str(b) + '/+text')
          r = conn.getresponse()
          if r.status == 200:
            data = r.read()
            if data != '':
              dq.put(data)
            else:
              print "[", currentThread().getName(), "] Bug ", b, ": Empty data."
              ok = False
              hq.put(b)
          else:
            print "[", currentThread().getName(), "] Bug ", b, ": Wrong status: ", r.status, " ", r.reason
            if r.status == 302:
              print "Exiting."
              _exit(1)
            ok = False
            hq.put(b)
        except httplib.BadStatusLine, line:
          print "[", currentThread().getName(), "] Bug ", b, ": BadStatusLine: ", line
          print str(r.getheaders())
          print r.read()
          ok = False
          hq.put(b)
        except:
          print "[", currentThread().getName(), "] Bug ", b, ": error, exiting"
          print sys.exc_info()
          _exit(1)

  parre = re.compile('^\s*(.*) \(([^(]*)\)$')
  def splitpar(self, text):
    mo = re.search(self.parre, text)
    if mo == None:
      return (text, '')
    return mo.groups()

  contenttype = re.compile('^Content-Type: ')
  def dbimport(self, c, data):
    d = data.split('\n\n')
    bug = d[0] + '\n'
    tasks = []
    for di in d[1:-1]:
      if re.match(self.contenttype, di + '\n'):
        break
      else:
        tasks.append(di)
    # OK, we have bugs and tasks.
    bm = email.message_from_string(bug)
    bugno = int(bm['bug'])
    # Check that we are not missing some fields
    # ignore attachments for now
    s = set(bm.keys()) - set(['bug', 'title', 'reporter', 'attachments',
      'subscribers', 'tags', 'duplicate-of', 'duplicates', 'date-reported',
      'date-updated', 'security'])
    if len(s) > 0:
      print s
    name, login = self.splitpar(bm['reporter'])
    if bm['duplicate-of'] != '':
      dup = int(bm['duplicate-of'])
    else:
      dup = None
    reported = time.strptime(bm['date-reported'], "%a, %d %b %Y %H:%M:%S -0000")
    updated = time.strptime(bm['date-updated'], "%a, %d %b %Y %H:%M:%S -0000")
    if bm['security'] != None:
      security = 't'
    else:
      security = 'f'
    treported = time.strftime("%a, %d %b %Y %H:%M:%S +0000", reported)
    tupdated = time.strftime("%a, %d %b %Y %H:%M:%S +0000", updated)
    c.execute('insert into ubuntu_bugs values (%s, %s, %s, %s, %s, %s, %s, %s)',
        (bugno, bm['title'], login, name, dup, treported, tupdated, security))
    # subscribers
    for sub in bm['subscribers'].split('\n'):
      name, login = self.splitpar(sub)
      c.execute('insert into ubuntu_bugs_subscribers values (%s, %s, %s)', (bugno, login, name))
    # duplicates
    for d in bm['duplicates'].split():
      c.execute('insert into ubuntu_bugs_duplicates values (%s, %s)', (bugno, int(d)))
    # tags
    for tag in bm['tags'].split():
      c.execute('insert into ubuntu_bugs_tags values (%s, %s)', (bugno, tag))
    ### Import tasks
    for t in tasks:
      tm = email.message_from_string(t)
      pkg, distro = self.splitpar(tm['task'])
      rep_name, rep_login = self.splitpar(tm['reporter'])
      if tm['assignee'] != '':
        ass_name, ass_login = self.splitpar(tm['assignee'])
      else:
        ass_name = None
        ass_login = None
      created = time.strftime("%a, %d %b %Y %H:%M:%S +0000", 
        time.strptime(tm['date-created'], "%a, %d %b %Y %H:%M:%S -0000"))
      if tm['date-assigned']:
        assigned = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-assigned'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        assigned = ''
      if tm['date-closed']:
        closed = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-closed'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        closed = ''
      if tm['date-incomplete']:
        incomplete = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-incomplete'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        incomplete = ''
      if tm['date-confirmed']:
        confirmed = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-confirmed'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        confirmed = ''
      if tm['date-inprogress']:
        inprogress = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-inprogress'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        inprogress = ''
      if tm['date-fix-committed']:
        fixcommitted = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-fix-committed'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        fixcommitted = ''
      if tm['date-fix-released']:
        fixreleased = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-fix-released'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        fixreleased = ''
      if tm['date-left-new']:
        leftnew = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-left-new'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        leftnew = ''
      if tm['date-triaged']:
        triaged = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-triaged'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        triaged = ''
      if tm['date-left-closed']:
        leftclosed = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
          time.strptime(tm['date-left-closed'], "%a, %d %b %Y %H:%M:%S -0000"))
      else:
        leftclosed = ''
      # check for missing headers
      s = set(tm.keys()) - set(['task', 'reporter', 'assignee', 'status', 'date-created', 'importance', 'component', 'milestone', 'date-assigned', 'date-closed', 'date-incomplete', 'date-confirmed', 'date-inprogress', 'date-fix-committed', 'date-fix-released', 'watch', 'date-left-new', 'date-triaged', 'date-left-closed'])
      if len(s) > 0:
        print s
        print t
      if self.debug:
        print str(bugno)+"\n"
      c.execute('insert into ubuntu_bugs_tasks values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', (bugno, pkg, distro,
        tm['status'], tm['importance'], tm['component'], tm['milestone'], created,
        assigned, closed, incomplete, confirmed, inprogress, fixcommitted, fixreleased, leftnew, triaged, leftclosed, tm['watch'],
        rep_login, rep_name, ass_login, ass_name))

1	#!/usr/bin/env python
2
3	"""
4	This script imports the Ubuntu bugs from Launchpad
5	"""
6
7	import sys
8	from aux import quote
9	from os import _exit
10	from gatherer import gatherer
11	import re
12	import urllib
13	from Queue import Queue, Empty
14	from threading import Thread, currentThread
15	import time
16	import httplib
17	import email
18
19	def get_gatherer(connection, config, source):
20	return ubuntu_bugs_gatherer(connection, config, source)
21
22	class ubuntu_bugs_gatherer(gatherer):
23	debug = False
24
25	def __init__(self, connection, config, source):
26	gatherer.__init__(self, connection, config, source)
27
28	def run(self):
29	my_config = self.my_config
30	num_fetchers = 4
31	num_writers = 1
32	bugs = self.fetch_all_bugs()
33	httpq = Queue()
34	dbq = Queue()
35	for b in bugs:
36	if b == 310331:
37	continue # this bug is buggy: two identical tasks
38	# if self.debug:
39	# if b != 310331:
40	# continue
41	httpq.put(b)
42
43	# start workers
44	for i in range(num_fetchers):
45	t = Thread(target=self.bugfetcher, name="Fetcher-"+str(i),args=[httpq, dbq])
46	t.setDaemon(True)
47	t.start()
48
49	c = self.cursor()
50	c.execute("delete from ubuntu_bugs_subscribers")
51	c.execute("delete from ubuntu_bugs_duplicates")
52	c.execute("delete from ubuntu_bugs_tags")
53	c.execute("delete from ubuntu_bugs_tasks")
54	c.execute("delete from ubuntu_bugs")
55	ok = True
56	while ok:
57	try:
58	if self.debug:
59	print "HTTPQ: ", httpq.qsize(), " DBQ: ", dbq.qsize()
60	d = dbq.get(True, 5) # 10 secs timeout
61	self.dbimport(c, d)
62	except Empty:
63	if httpq.qsize() == 0:
64	ok = False
65	c.execute("analyze ubuntu_bugs_subscribers")
66	c.execute("analyze ubuntu_bugs_duplicates")
67	c.execute("analyze ubuntu_bugs_tags")
68	c.execute("analyze ubuntu_bugs_tasks")
69	c.execute("analyze ubuntu_bugs")
70
71	def fetch_all_bugs(self):
72	fh = urllib.urlopen('https://launchpad.net/ubuntu/+bugs-text')
73	text = fh.read()
74	# convert to a list
75	bugs = text.split('\n')
76	# remove '', or map() will complain
77	bugs.remove('')
78	# convert each bug (string) to a int
79	bugs = map(int, bugs)
80	# sort, so that we can remove duplicates in O(n) later
81	bugs.sort()
82	# remove duplicates. apparently not in lib, see
83	# http://www.python.org/dev/peps/pep-0270/
84	# FIXME use set()
85	nbugs = []
86	on = 0
87	for n in bugs:
88	if n != on:
89	nbugs.append(n)
90	# else:
91	# print "Duplicate bug: " + str(n)
92	on = n
93	fh.close()
94	#print nbugs
95	return nbugs
96
97	# "worker". Fetch a specific bug as text from launchpad.
98	def bugfetcher(self, hq, dq):
99	while True:
100	conn = httplib.HTTPSConnection('bugs.launchpad.net')
101	ok = True
102	b = None
103	while ok:
104	try:
105	b = hq.get(False)
106	except Empty:
107	return
108	except:
109	print "Other exception raised in bugfetcher. exiting."
110	_exit(1)
111
112	try:
113	conn.request('GET', 'https://launchpad.net/bugs/' + str(b) + '/+text')
114	r = conn.getresponse()
115	if r.status == 200:
116	data = r.read()
117	if data != '':
118	dq.put(data)
119	else:
120	print "[", currentThread().getName(), "] Bug ", b, ": Empty data."
121	ok = False
122	hq.put(b)
123	else:
124	print "[", currentThread().getName(), "] Bug ", b, ": Wrong status: ", r.status, " ", r.reason
125	if r.status == 302:
126	print "Exiting."
127	_exit(1)
128	ok = False
129	hq.put(b)
130	except httplib.BadStatusLine, line:
131	print "[", currentThread().getName(), "] Bug ", b, ": BadStatusLine: ", line
132	print str(r.getheaders())
133	print r.read()
134	ok = False
135	hq.put(b)
136	except:
137	print "[", currentThread().getName(), "] Bug ", b, ": error, exiting"
138	print sys.exc_info()
139	_exit(1)
140
141	parre = re.compile('^\s(.) \(([^(]*)\)$')
142	def splitpar(self, text):
143	mo = re.search(self.parre, text)
144	if mo == None:
145	return (text, '')
146	return mo.groups()
147
148	contenttype = re.compile('^Content-Type: ')
149	def dbimport(self, c, data):
150	d = data.split('\n\n')
151	bug = d[0] + '\n'
152	tasks = []
153	for di in d[1:-1]:
154	if re.match(self.contenttype, di + '\n'):
155	break
156	else:
157	tasks.append(di)
158	# OK, we have bugs and tasks.
159	bm = email.message_from_string(bug)
160	bugno = int(bm['bug'])
161	# Check that we are not missing some fields
162	# ignore attachments for now
163	s = set(bm.keys()) - set(['bug', 'title', 'reporter', 'attachments',
164	'subscribers', 'tags', 'duplicate-of', 'duplicates', 'date-reported',
165	'date-updated', 'security'])
166	if len(s) > 0:
167	print s
168	name, login = self.splitpar(bm['reporter'])
169	if bm['duplicate-of'] != '':
170	dup = int(bm['duplicate-of'])
171	else:
172	dup = None
173	reported = time.strptime(bm['date-reported'], "%a, %d %b %Y %H:%M:%S -0000")
174	updated = time.strptime(bm['date-updated'], "%a, %d %b %Y %H:%M:%S -0000")
175	if bm['security'] != None:
176	security = 't'
177	else:
178	security = 'f'
179	treported = time.strftime("%a, %d %b %Y %H:%M:%S +0000", reported)
180	tupdated = time.strftime("%a, %d %b %Y %H:%M:%S +0000", updated)
181	c.execute('insert into ubuntu_bugs values (%s, %s, %s, %s, %s, %s, %s, %s)',
182	(bugno, bm['title'], login, name, dup, treported, tupdated, security))
183	# subscribers
184	for sub in bm['subscribers'].split('\n'):
185	name, login = self.splitpar(sub)
186	c.execute('insert into ubuntu_bugs_subscribers values (%s, %s, %s)', (bugno, login, name))
187	# duplicates
188	for d in bm['duplicates'].split():
189	c.execute('insert into ubuntu_bugs_duplicates values (%s, %s)', (bugno, int(d)))
190	# tags
191	for tag in bm['tags'].split():
192	c.execute('insert into ubuntu_bugs_tags values (%s, %s)', (bugno, tag))
193	### Import tasks
194	for t in tasks:
195	tm = email.message_from_string(t)
196	pkg, distro = self.splitpar(tm['task'])
197	rep_name, rep_login = self.splitpar(tm['reporter'])
198	if tm['assignee'] != '':
199	ass_name, ass_login = self.splitpar(tm['assignee'])
200	else:
201	ass_name = None
202	ass_login = None
203	created = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
204	time.strptime(tm['date-created'], "%a, %d %b %Y %H:%M:%S -0000"))
205	if tm['date-assigned']:
206	assigned = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
207	time.strptime(tm['date-assigned'], "%a, %d %b %Y %H:%M:%S -0000"))
208	else:
209	assigned = ''
210	if tm['date-closed']:
211	closed = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
212	time.strptime(tm['date-closed'], "%a, %d %b %Y %H:%M:%S -0000"))
213	else:
214	closed = ''
215	if tm['date-incomplete']:
216	incomplete = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
217	time.strptime(tm['date-incomplete'], "%a, %d %b %Y %H:%M:%S -0000"))
218	else:
219	incomplete = ''
220	if tm['date-confirmed']:
221	confirmed = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
222	time.strptime(tm['date-confirmed'], "%a, %d %b %Y %H:%M:%S -0000"))
223	else:
224	confirmed = ''
225	if tm['date-inprogress']:
226	inprogress = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
227	time.strptime(tm['date-inprogress'], "%a, %d %b %Y %H:%M:%S -0000"))
228	else:
229	inprogress = ''
230	if tm['date-fix-committed']:
231	fixcommitted = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
232	time.strptime(tm['date-fix-committed'], "%a, %d %b %Y %H:%M:%S -0000"))
233	else:
234	fixcommitted = ''
235	if tm['date-fix-released']:
236	fixreleased = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
237	time.strptime(tm['date-fix-released'], "%a, %d %b %Y %H:%M:%S -0000"))
238	else:
239	fixreleased = ''
240	if tm['date-left-new']:
241	leftnew = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
242	time.strptime(tm['date-left-new'], "%a, %d %b %Y %H:%M:%S -0000"))
243	else:
244	leftnew = ''
245	if tm['date-triaged']:
246	triaged = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
247	time.strptime(tm['date-triaged'], "%a, %d %b %Y %H:%M:%S -0000"))
248	else:
249	triaged = ''
250	if tm['date-left-closed']:
251	leftclosed = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
252	time.strptime(tm['date-left-closed'], "%a, %d %b %Y %H:%M:%S -0000"))
253	else:
254	leftclosed = ''
255	# check for missing headers
256	s = set(tm.keys()) - set(['task', 'reporter', 'assignee', 'status', 'date-created', 'importance', 'component', 'milestone', 'date-assigned', 'date-closed', 'date-incomplete', 'date-confirmed', 'date-inprogress', 'date-fix-committed', 'date-fix-released', 'watch', 'date-left-new', 'date-triaged', 'date-left-closed'])
257	if len(s) > 0:
258	print s
259	print t
260	if self.debug:
261	print str(bugno)+"\n"
262	c.execute('insert into ubuntu_bugs_tasks values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', (bugno, pkg, distro,
263	tm['status'], tm['importance'], tm['component'], tm['milestone'], created,
264	assigned, closed, incomplete, confirmed, inprogress, fixcommitted, fixreleased, leftnew, triaged, leftclosed, tm['watch'],
265	rep_login, rep_name, ass_login, ass_name))
266