udd/udd/upload_history_gatherer.py

# Last-Modified: <Tue Aug 12 15:09:13 2008>
# This file is part of the Ultimate Debian Database Project

from gatherer import gatherer
import aux
from glob import glob
import gzip
import psycopg2
import sys

def get_gatherer(config, connection, source):
  return upload_history_gatherer(config, connection, source)

class upload_history_gatherer(gatherer):
  def __init__(self, connection, config, source):
    gatherer.__init__(self, connection, config, source)
    if not 'path' in self.my_config:
      raise aux.ConfigException('path not specified for source ' + source)

  def drop(self):
    cur = self.cursor()
    cur.execute("DROP TABLE %s" % self.my_config['table'])
    cur.execute("DROP TABLE %s" % self.my_config['table'] + '_architecture')


  def run(self):
    path = self.my_config['path']

    cursor = self.cursor()

    cursor.execute("DELETE FROM " + self.my_config['table'])
    cursor.execute("DELETE FROM " + self.my_config['table'] + '_architecture')

    cursor.execute("PREPARE uh_insert AS INSERT INTO %s VALUES \
        ($1, $2, $3, $4, $5, $6, $7, $8, $9)" % self.my_config['table'])
    cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s VALUES \
        ($1, $2)" % (self.my_config['table'] + '_architecture'))

    id = 0
    for name in glob(path + '/debian-devel-*'):
      print name
      f = None
      if name.endswith(".gz"):
        f = gzip.open(name)
      else:
        f = open(name)
      
      current = {'id': id}
      last_field = None
      line_count = 0
      for line in f:
        line_count += 1
        line = line.strip()
        # Stupid multi-line maintainer fields *grml*
        if line == '':
          for arch in set(current['Architecture'].split()):
            current['arch'] = arch
            query = "EXECUTE uh_arch_insert(%(id)s, %(arch)s)"
            cursor.execute(query, current)
          query = "EXECUTE uh_insert(%(id)s, %(Source)s, %(Version)s, %(Date)s, %(Changed-By)s, \
              %(Maintainer)s, %(NMU)s, %(Key)s, %(Signed-By)s)"
          try:
            cursor.execute(query, current)
          except psycopg2.ProgrammingError, s:
            print "Error at line %d of file %s" % (line_count, name)
            raise
          id += 1
          current = {'id': id}
          last_field = None
          continue

        if line.find(':') == -1:
          if not last_field:
            raise Exception, "Format error on line " + line_count + "of file " + name
          current[last_field] += line
          continue


        (field, data) = line.split(':', 1)
        data = data.strip()

        if field != 'NMU':
          current[field] = aux.quote(data)
        else:
          current[field] = data
        
        last_field = field
    
    cursor.execute("DEALLOCATE uh_insert")
1	# Last-Modified: <Tue Aug 12 15:09:13 2008>
2	# This file is part of the Ultimate Debian Database Project
3
4	from gatherer import gatherer
5	import aux
6	from glob import glob
7	import gzip
8	import psycopg2
9	import sys
10
11	def get_gatherer(config, connection, source):
12	return upload_history_gatherer(config, connection, source)
13
14	class upload_history_gatherer(gatherer):
15	def __init__(self, connection, config, source):
16	gatherer.__init__(self, connection, config, source)
17	if not 'path' in self.my_config:
18	raise aux.ConfigException('path not specified for source ' + source)
19
20	def drop(self):
21	cur = self.cursor()
22	cur.execute("DROP TABLE %s" % self.my_config['table'])
23	cur.execute("DROP TABLE %s" % self.my_config['table'] + '_architecture')
24
25
26	def run(self):
27	path = self.my_config['path']
28
29	cursor = self.cursor()
30
31	cursor.execute("DELETE FROM " + self.my_config['table'])
32	cursor.execute("DELETE FROM " + self.my_config['table'] + '_architecture')
33
34	cursor.execute("PREPARE uh_insert AS INSERT INTO %s VALUES \
35	($1, $2, $3, $4, $5, $6, $7, $8, $9)" % self.my_config['table'])
36	cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s VALUES \
37	($1, $2)" % (self.my_config['table'] + '_architecture'))
38
39	id = 0
40	for name in glob(path + '/debian-devel-*'):
41	print name
42	f = None
43	if name.endswith(".gz"):
44	f = gzip.open(name)
45	else:
46	f = open(name)
47
48	current = {'id': id}
49	last_field = None
50	line_count = 0
51	for line in f:
52	line_count += 1
53	line = line.strip()
54	# Stupid multi-line maintainer fields grml
55	if line == '':
56	for arch in set(current['Architecture'].split()):
57	current['arch'] = arch
58	query = "EXECUTE uh_arch_insert(%(id)s, %(arch)s)"
59	cursor.execute(query, current)
60	query = "EXECUTE uh_insert(%(id)s, %(Source)s, %(Version)s, %(Date)s, %(Changed-By)s, \
61	%(Maintainer)s, %(NMU)s, %(Key)s, %(Signed-By)s)"
62	try:
63	cursor.execute(query, current)
64	except psycopg2.ProgrammingError, s:
65	print "Error at line %d of file %s" % (line_count, name)
66	raise
67	id += 1
68	current = {'id': id}
69	last_field = None
70	continue
71
72	if line.find(':') == -1:
73	if not last_field:
74	raise Exception, "Format error on line " + line_count + "of file " + name
75	current[last_field] += line
76	continue
77
78
79	(field, data) = line.split(':', 1)
80	data = data.strip()
81
82	if field != 'NMU':
83	current[field] = aux.quote(data)
84	else:
85	current[field] = data
86
87	last_field = field
88
89	cursor.execute("DEALLOCATE uh_insert")