udd/udd/upload_history_gatherer.py

# Last-Modified: <Sun Aug 17 12:13:02 2008>
# This file is part of the Ultimate Debian Database Project

from gatherer import gatherer
import aux
from glob import glob
import gzip
import psycopg2
import sys
import os.path

def get_gatherer(config, connection, source):
  return upload_history_gatherer(config, connection, source)

class upload_history_gatherer(gatherer):
  def __init__(self, connection, config, source):
    self.is_ubuntu = source == 'ubuntu-upload-history'
    self.is_debian = not self.is_ubuntu
    gatherer.__init__(self, connection, config, source)
    if not 'path' in self.my_config:
      raise aux.ConfigException('path not specified for source ' + source)

  def run(self):

    path = self.my_config['path']
    if 'only-recent' in self.my_config:
      onlyrecent = self.my_config['only-recent']
    else:
      onlyrecent = True

    cursor = self.cursor()

    tables = ['source', 'version', 'date', 'changed_by', 'changed_by_name', 'changed_by_email', 'maintainer', 'maintainer_name', 'maintainer_email', 'nmu', 'signed_by', 'signed_by_name', 'signed_by_email', 'key_id', 'fingerprint', 'distribution', 'file']

    if self.is_ubuntu:
      tables = tables + ['original_maintainer', 'original_maintainer_name', 'original_maintainer_email', 'component']

    indices = ', '.join(map(lambda x: '$' + str(x), range(1,len(tables)+1)))
    tables = ', '.join(tables)

    cursor.execute("PREPARE uh_insert AS INSERT INTO %s (%s) VALUES \
      (%s)" % (self.my_config['table'], tables, indices))

    if self.is_debian:
      cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s (source, \
        version, architecture, file) VALUES \
        ($1, $2, $3, $4)" % (self.my_config['table'] + '_architecture'))

    if self.is_ubuntu:
      cursor.execute("PREPARE uh_launchpad_close_insert AS INSERT INTO %s (source, version, bug, file) \
          VALUES ($1, $2, $3, $4)" % (self.my_config['table'] + '_launchpad_closes'))
    
    cursor.execute("PREPARE uh_close_insert AS INSERT INTO %s (source, version, bug, file) \
        VALUES ($1, $2, $3, $4)" % (self.my_config['table'] + '_closes'))

    query = "EXECUTE uh_insert(%(Source)s, %(Version)s, %(Message-Date)s, \
      %(Changed-By)s, %(Changed-By_name)s, %(Changed-By_email)s, \
      %(Maintainer)s, %(Maintainer_name)s, %(Maintainer_email)s, %(NMU)s, \
      %(Signed-By)s, %(Signed-By_name)s, %(Signed-By_email)s, %(Key)s, \
      %(Fingerprint)s, %(Distribution)s, %(File)s)"

    if self.is_ubuntu:
        query = query[:-1] + ", %(Original-Maintainer)s, %(Original-Maintainer_name)s, %(Original-Maintainer_email)s, %(Component)s)"
        
    if self.is_debian:
        query_archs = "EXECUTE uh_arch_insert(%(Source)s, %(Version)s, %(arch)s, %(File)s)"

    query_closes = "EXECUTE uh_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"

    if self.is_ubuntu:
        query_launchpad_closes = "EXECUTE uh_launchpad_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"

    added = {}
    files = glob(path + '/*-changes*')
    files.sort()
    if onlyrecent:
      files = files[-2:]
    else:
      print "Doing full import!"
      if self.is_debian:
        cursor.execute("delete from " + self.my_config['table'] + "_architecture")
      if self.is_ubuntu:
        cursor.execute("delete from " + self.my_config['table'] + "_launchpad_closes")
      cursor.execute("delete from " + self.my_config['table'] + "_closes")
      cursor.execute("delete from " + self.my_config['table'])
    for name in files:
      bname = os.path.basename(name).replace(".gz","").replace(".out","")
      if self.is_debian:
        cursor.execute("DELETE FROM " + self.my_config['table'] + "_architecture where file='%s'" % (bname))
      if self.is_ubuntu:
        cursor.execute("DELETE FROM " + self.my_config['table'] + "_launchpad_closes where file='%s'" % (bname))
      cursor.execute("DELETE FROM " + self.my_config['table'] + "_closes where file='%s'" % (bname))
      cursor.execute("DELETE FROM " + self.my_config['table'] +  " where file='%s'" % (bname))

      f = None
      if name.endswith(".gz"):
        f = gzip.open(name)
      else:
        f = open(name)
      current = {}
      current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
      current['NMU'] = False
      current['Key'] = ''
      current['File'] = bname
      last_field = None
      line_count = 0
      uploads = []
      uploads_archs = []
      uploads_closes = []
      uploads_launchpad_closes = []

      for line in f:
        line_count += 1
        line = line.lstrip()
        # Stupid multi-line maintainer fields *grml*
        if line == '':
          current['Changed-By_name'], current['Changed-By_email'] = aux.parse_email(current['Changed-By'])
          current['Maintainer_name'], current['Maintainer_email'] = aux.parse_email(current['Maintainer'])
          current['Signed-By_name'], current['Signed-By_email'] = aux.parse_email(current['Signed-By'])
          if current['Signed-By_name'] == '':
            current['Signed-By_name'] = current['Signed-By']
            current['Signed-By_email'] = ''

          if current.has_key('Original-Maintainer'):
            if current['Original-Maintainer'] != 'N/A':
              current['Original-Maintainer_name'], current['Original-Maintainer_email'] = aux.parse_email(current['Original-Maintainer'])
            else:
              current['Original-Maintainer_name'] = current['Original-Maintainer_email'] = 'N/A'
          if not current.has_key('Message-Date') and current.has_key('Date'):
            current['Message-Date'] = current['Date']
          current['Message-Date'] = current['Message-Date'].partition('(')[0].replace('+4200','+0000').replace('+4300','+0000').replace('+4100','+0000').replace('+4400','+0000').replace('+4000','+0000')
          if (current['Source'], current['Version']) in added or \
            (current['Source'], current['Version']) == ('libapache-authznetldap-perl', '0.07-4') or \
            (current['Source'], current['Version']) == ('knj10font', '1.01-1') or \
            (current['Source'], current['Version']) == ('xmorph', '1:20010421') or \
            current['Message-Date'] == 'None':
              print "Skipping upload: "+current['Source']+" "+current['Version']+" "+current['Date']
              current = {}
              current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
              current['NMU'] = False
              current['Key'] = ''
              current['File'] = bname
              last_field = None
              continue
          added[(current['Source'], current['Version'])] = True
          uploads.append(current)
          if self.is_debian:
            for arch in set(current['Architecture'].split()):
              current_arch = {'Source': current['Source'], 'Version': current['Version'], 'File': bname} 
              current_arch['arch'] = arch
              uploads_archs.append(current_arch)
          if current['Closes'] != 'N/A':
            for closes in set(current['Closes'].split()):
              current_closes = {'Source': current['Source'], 'Version': current['Version'], 'File': bname} 
              current_closes['closes'] = closes
              uploads_closes.append(current_closes)
          if current.has_key('Launchpad-Bugs-Fixed') and current['Launchpad-Bugs-Fixed'] != 'N/A':
            for closes in set(current['Launchpad-Bugs-Fixed'].split()):
              current_closes = {'Source': current['Source'], 'Version': current['Version'], 'File': bname} 
              current_closes['closes'] = closes
              uploads_launchpad_closes.append(current_closes)
          current = {}
          current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
          current['NMU'] = False
          current['Key'] = ''
          current['File'] = bname
          last_field = None
          continue

        if line.find(':') == -1:
          if not last_field:
            raise Exception, "Format error on line " + line_count + "of file " + name
          current[last_field] += line
          continue

        (field, data) = line.split(':', 1)
        data = data.strip()
        current[field] = data
        
        last_field = field

      #print uploads
      #for u in uploads:
      #  print u
      #  cursor.execute(query, u)
      cursor.executemany(query, uploads)
      if self.is_debian:
        cursor.executemany(query_archs, uploads_archs)
      if self.is_ubuntu:
        cursor.executemany(query_launchpad_closes, uploads_launchpad_closes)
      cursor.executemany(query_closes, uploads_closes)
      
    cursor.execute("DEALLOCATE uh_insert")
    if self.is_debian:
      cursor.execute("ANALYZE " + self.my_config['table'] + '_architecture')
    if self.is_ubuntu:
      cursor.execute("ANALYZE " + self.my_config['table'] + '_launchpad_closes')

    cursor.execute("ANALYZE " + self.my_config['table'] + '_closes')
    cursor.execute("ANALYZE " + self.my_config['table'])
1	# Last-Modified: <Sun Aug 17 12:13:02 2008>
2	# This file is part of the Ultimate Debian Database Project
3
4	from gatherer import gatherer
5	import aux
6	from glob import glob
7	import gzip
8	import psycopg2
9	import sys
10	import os.path
11
12	def get_gatherer(config, connection, source):
13	return upload_history_gatherer(config, connection, source)
14
15	class upload_history_gatherer(gatherer):
16	def __init__(self, connection, config, source):
17	self.is_ubuntu = source == 'ubuntu-upload-history'
18	self.is_debian = not self.is_ubuntu
19	gatherer.__init__(self, connection, config, source)
20	if not 'path' in self.my_config:
21	raise aux.ConfigException('path not specified for source ' + source)
22
23	def run(self):
24
25	path = self.my_config['path']
26	if 'only-recent' in self.my_config:
27	onlyrecent = self.my_config['only-recent']
28	else:
29	onlyrecent = True
30
31	cursor = self.cursor()
32
33	tables = ['source', 'version', 'date', 'changed_by', 'changed_by_name', 'changed_by_email', 'maintainer', 'maintainer_name', 'maintainer_email', 'nmu', 'signed_by', 'signed_by_name', 'signed_by_email', 'key_id', 'fingerprint', 'distribution', 'file']
34
35	if self.is_ubuntu:
36	tables = tables + ['original_maintainer', 'original_maintainer_name', 'original_maintainer_email', 'component']
37
38	indices = ', '.join(map(lambda x: '$' + str(x), range(1,len(tables)+1)))
39	tables = ', '.join(tables)
40
41	cursor.execute("PREPARE uh_insert AS INSERT INTO %s (%s) VALUES \
42	(%s)" % (self.my_config['table'], tables, indices))
43
44	if self.is_debian:
45	cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s (source, \
46	version, architecture, file) VALUES \
47	($1, $2, $3, $4)" % (self.my_config['table'] + '_architecture'))
48
49	if self.is_ubuntu:
50	cursor.execute("PREPARE uh_launchpad_close_insert AS INSERT INTO %s (source, version, bug, file) \
51	VALUES ($1, $2, $3, $4)" % (self.my_config['table'] + '_launchpad_closes'))
52
53	cursor.execute("PREPARE uh_close_insert AS INSERT INTO %s (source, version, bug, file) \
54	VALUES ($1, $2, $3, $4)" % (self.my_config['table'] + '_closes'))
55
56	query = "EXECUTE uh_insert(%(Source)s, %(Version)s, %(Message-Date)s, \
57	%(Changed-By)s, %(Changed-By_name)s, %(Changed-By_email)s, \
58	%(Maintainer)s, %(Maintainer_name)s, %(Maintainer_email)s, %(NMU)s, \
59	%(Signed-By)s, %(Signed-By_name)s, %(Signed-By_email)s, %(Key)s, \
60	%(Fingerprint)s, %(Distribution)s, %(File)s)"
61
62	if self.is_ubuntu:
63	query = query[:-1] + ", %(Original-Maintainer)s, %(Original-Maintainer_name)s, %(Original-Maintainer_email)s, %(Component)s)"
64
65	if self.is_debian:
66	query_archs = "EXECUTE uh_arch_insert(%(Source)s, %(Version)s, %(arch)s, %(File)s)"
67
68	query_closes = "EXECUTE uh_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"
69
70	if self.is_ubuntu:
71	query_launchpad_closes = "EXECUTE uh_launchpad_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"
72
73	added = {}
74	files = glob(path + '/-changes')
75	files.sort()
76	if onlyrecent:
77	files = files[-2:]
78	else:
79	print "Doing full import!"
80	if self.is_debian:
81	cursor.execute("delete from " + self.my_config['table'] + "_architecture")
82	if self.is_ubuntu:
83	cursor.execute("delete from " + self.my_config['table'] + "_launchpad_closes")
84	cursor.execute("delete from " + self.my_config['table'] + "_closes")
85	cursor.execute("delete from " + self.my_config['table'])
86	for name in files:
87	bname = os.path.basename(name).replace(".gz","").replace(".out","")
88	if self.is_debian:
89	cursor.execute("DELETE FROM " + self.my_config['table'] + "_architecture where file='%s'" % (bname))
90	if self.is_ubuntu:
91	cursor.execute("DELETE FROM " + self.my_config['table'] + "_launchpad_closes where file='%s'" % (bname))
92	cursor.execute("DELETE FROM " + self.my_config['table'] + "_closes where file='%s'" % (bname))
93	cursor.execute("DELETE FROM " + self.my_config['table'] + " where file='%s'" % (bname))
94
95	f = None
96	if name.endswith(".gz"):
97	f = gzip.open(name)
98	else:
99	f = open(name)
100	current = {}
101	current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
102	current['NMU'] = False
103	current['Key'] = ''
104	current['File'] = bname
105	last_field = None
106	line_count = 0
107	uploads = []
108	uploads_archs = []
109	uploads_closes = []
110	uploads_launchpad_closes = []
111
112	for line in f:
113	line_count += 1
114	line = line.lstrip()
115	# Stupid multi-line maintainer fields grml
116	if line == '':
117	current['Changed-By_name'], current['Changed-By_email'] = aux.parse_email(current['Changed-By'])
118	current['Maintainer_name'], current['Maintainer_email'] = aux.parse_email(current['Maintainer'])
119	current['Signed-By_name'], current['Signed-By_email'] = aux.parse_email(current['Signed-By'])
120	if current['Signed-By_name'] == '':
121	current['Signed-By_name'] = current['Signed-By']
122	current['Signed-By_email'] = ''
123
124	if current.has_key('Original-Maintainer'):
125	if current['Original-Maintainer'] != 'N/A':
126	current['Original-Maintainer_name'], current['Original-Maintainer_email'] = aux.parse_email(current['Original-Maintainer'])
127	else:
128	current['Original-Maintainer_name'] = current['Original-Maintainer_email'] = 'N/A'
129	if not current.has_key('Message-Date') and current.has_key('Date'):
130	current['Message-Date'] = current['Date']
131	current['Message-Date'] = current['Message-Date'].partition('(')[0].replace('+4200','+0000').replace('+4300','+0000').replace('+4100','+0000').replace('+4400','+0000').replace('+4000','+0000')
132	if (current['Source'], current['Version']) in added or \
133	(current['Source'], current['Version']) == ('libapache-authznetldap-perl', '0.07-4') or \
134	(current['Source'], current['Version']) == ('knj10font', '1.01-1') or \
135	(current['Source'], current['Version']) == ('xmorph', '1:20010421') or \
136	current['Message-Date'] == 'None':
137	print "Skipping upload: "+current['Source']+" "+current['Version']+" "+current['Date']
138	current = {}
139	current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
140	current['NMU'] = False
141	current['Key'] = ''
142	current['File'] = bname
143	last_field = None
144	continue
145	added[(current['Source'], current['Version'])] = True
146	uploads.append(current)
147	if self.is_debian:
148	for arch in set(current['Architecture'].split()):
149	current_arch = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
150	current_arch['arch'] = arch
151	uploads_archs.append(current_arch)
152	if current['Closes'] != 'N/A':
153	for closes in set(current['Closes'].split()):
154	current_closes = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
155	current_closes['closes'] = closes
156	uploads_closes.append(current_closes)
157	if current.has_key('Launchpad-Bugs-Fixed') and current['Launchpad-Bugs-Fixed'] != 'N/A':
158	for closes in set(current['Launchpad-Bugs-Fixed'].split()):
159	current_closes = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
160	current_closes['closes'] = closes
161	uploads_launchpad_closes.append(current_closes)
162	current = {}
163	current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
164	current['NMU'] = False
165	current['Key'] = ''
166	current['File'] = bname
167	last_field = None
168	continue
169
170	if line.find(':') == -1:
171	if not last_field:
172	raise Exception, "Format error on line " + line_count + "of file " + name
173	current[last_field] += line
174	continue
175
176	(field, data) = line.split(':', 1)
177	data = data.strip()
178	current[field] = data
179
180	last_field = field
181
182	#print uploads
183	#for u in uploads:
184	# print u
185	# cursor.execute(query, u)
186	cursor.executemany(query, uploads)
187	if self.is_debian:
188	cursor.executemany(query_archs, uploads_archs)
189	if self.is_ubuntu:
190	cursor.executemany(query_launchpad_closes, uploads_launchpad_closes)
191	cursor.executemany(query_closes, uploads_closes)
192
193	cursor.execute("DEALLOCATE uh_insert")
194	if self.is_debian:
195	cursor.execute("ANALYZE " + self.my_config['table'] + '_architecture')
196	if self.is_ubuntu:
197	cursor.execute("ANALYZE " + self.my_config['table'] + '_launchpad_closes')
198
199	cursor.execute("ANALYZE " + self.my_config['table'] + '_closes')
200	cursor.execute("ANALYZE " + self.my_config['table'])