udd/udd/hints_gatherer.py

# Last-Modified: <Sun Aug 17 12:13:02 2008>
# This file is part of the Ultimate Debian Database Project

from gatherer import gatherer
import aux
from glob import glob
import gzip
import psycopg2
import sys
import email.Utils
import os.path
import re

def get_gatherer(config, connection, source):
  return hints_gatherer(config, connection, source)

class Hint(object):
    """Base clase for representing a hint.

    It also acts as a factory: Hint(hintame, *args) will return an object of
    the appropriate class (eg. UnstableToTestingHint).
    """

    def __init__(self, type, *args):
        starts_with_letter = re.compile(r'(?i)^[a-z]')

        self.type = type
        self.pkgs = []
        self.versionmap = {} # original versions on the hint
        self.archmap = {}
        self.arg = None
        if self.type == 'age-days':
            self.arg = args[0]
            args = args[1:]

        for pkg in args:
            if '/' in pkg:
                pkg, ver = pkg.split('/', 1)
                if starts_with_letter.search(ver):
                    # Package names start with digits, so assume
                    # this is an architecture.  This will break if
                    # an architecture name ever starts with a digit
                    # but that seems unlikely.
                    if '/' in ver:
                        arch, ver = ver.split('/', 1)
                    else:
                        arch = ver
                        ver = ''
                else:
                    arch = ''
            else:
                ver = ''
                arch = ''

            src = pkg
            if src.startswith('-'):
                src = src[1:]
            self.pkgs.append(src)

            self.versionmap[src] = ver
            if len(arch) > 0:
                if src not in self.archmap:
                    self.archmap[src] = set()
                self.archmap[src].add(arch)

class hints_gatherer(gatherer):
  def __init__(self, connection, config, source):
    gatherer.__init__(self, connection, config, source)
    if not 'path' in self.my_config:
      raise aux.ConfigException('path not specified for source ' + source)

  def tables(self):
    return [ 'hints' ]

  RE_COMMENT = re.compile(r'^\s*#')
  RE_BLANKLINE = re.compile(r'^\s*$')
  def parse_hints(self, path):
    files = glob(path + '/*')
    files.sort()
    hints = []
    for file in files:
        f = open(file)
        comments = []
        lc = 0
        for line in f:
            lc += 1
            line = line.rstrip('\r\n')
            if self.RE_COMMENT.search(line):
                comments.append(line)
                continue
            elif self.RE_BLANKLINE.search(line):
                comments = []
                continue
            elif line.startswith('finished'):
                break
            hint = Hint(*line.split())
            hints.append([hint, os.path.basename(file), '\n'.join(comments)])
    return hints

  def run(self):
    path = self.my_config['path']

    hints = self.parse_hints(path)

    cursor = self.cursor()
    cursor.execute("PREPARE h_insert AS INSERT INTO hints (source, version, architecture, type, argument, file, comment) VALUES ($1, $2 , $3, $4, $5, $6, $7)")
    cursor.execute("DELETE FROM hints")
    query = "EXECUTE h_insert(%(source)s, %(version)s, %(arch)s, %(type)s, %(argument)s, %(file)s, %(comment)s)"
    hs = []
    for hint in hints:
        h = {}
        h['file'] = hint[1]
        h['comment'] = hint[2]
        h['argument'] = hint[0].arg
        h['type'] = hint[0].type
        for src in hint[0].pkgs:
            h['source'] = src
            if hint[0].versionmap.has_key(src):
                h['version'] = hint[0].versionmap[src]
                if h['version'] == '':
                    h['version'] = None
            else:
                h['version'] = None
            if hint[0].archmap.has_key(src):
                h['arch'] = ' '.join(hint[0].archmap[src])
            else:
                h['arch'] = None
            hs.append(h.copy())
    cursor.executemany(query, hs)
    cursor.execute("DEALLOCATE h_insert")
    cursor.execute("ANALYZE hints")
1	lucas	1808	# Last-Modified: <Sun Aug 17 12:13:02 2008>
2			# This file is part of the Ultimate Debian Database Project
3
4			from gatherer import gatherer
5			import aux
6			from glob import glob
7			import gzip
8			import psycopg2
9			import sys
10			import email.Utils
11			import os.path
12			import re
13
14			def get_gatherer(config, connection, source):
15			return hints_gatherer(config, connection, source)
16
17			class Hint(object):
18			"""Base clase for representing a hint.
19
20			It also acts as a factory: Hint(hintame, *args) will return an object of
21			the appropriate class (eg. UnstableToTestingHint).
22			"""
23
24			def __init__(self, type, *args):
25			starts_with_letter = re.compile(r'(?i)^[a-z]')
26
27			self.type = type
28			self.pkgs = []
29			self.versionmap = {} # original versions on the hint
30			self.archmap = {}
31	lucas	1813	self.arg = None
32			if self.type == 'age-days':
33			self.arg = args[0]
34			args = args[1:]
35	lucas	1808
36			for pkg in args:
37			if '/' in pkg:
38			pkg, ver = pkg.split('/', 1)
39			if starts_with_letter.search(ver):
40			# Package names start with digits, so assume
41			# this is an architecture. This will break if
42			# an architecture name ever starts with a digit
43			# but that seems unlikely.
44			if '/' in ver:
45			arch, ver = ver.split('/', 1)
46			else:
47			arch = ver
48			ver = ''
49			else:
50			arch = ''
51			else:
52			ver = ''
53			arch = ''
54
55			src = pkg
56			if src.startswith('-'):
57			src = src[1:]
58			self.pkgs.append(src)
59
60			self.versionmap[src] = ver
61			if len(arch) > 0:
62			if src not in self.archmap:
63			self.archmap[src] = set()
64			self.archmap[src].add(arch)
65
66			class hints_gatherer(gatherer):
67			def __init__(self, connection, config, source):
68			gatherer.__init__(self, connection, config, source)
69			if not 'path' in self.my_config:
70			raise aux.ConfigException('path not specified for source ' + source)
71
72			def tables(self):
73			return [ 'hints' ]
74
75			RE_COMMENT = re.compile(r'^\s*#')
76			RE_BLANKLINE = re.compile(r'^\s*$')
77			def parse_hints(self, path):
78			files = glob(path + '/*')
79			files.sort()
80			hints = []
81			for file in files:
82			f = open(file)
83			comments = []
84			lc = 0
85			for line in f:
86			lc += 1
87			line = line.rstrip('\r\n')
88			if self.RE_COMMENT.search(line):
89			comments.append(line)
90			continue
91			elif self.RE_BLANKLINE.search(line):
92			comments = []
93			continue
94			elif line.startswith('finished'):
95			break
96			hint = Hint(*line.split())
97			hints.append([hint, os.path.basename(file), '\n'.join(comments)])
98			return hints
99
100			def run(self):
101			path = self.my_config['path']
102
103			hints = self.parse_hints(path)
104
105			cursor = self.cursor()
106	lucas	1813	cursor.execute("PREPARE h_insert AS INSERT INTO hints (source, version, architecture, type, argument, file, comment) VALUES ($1, $2 , $3, $4, $5, $6, $7)")
107	lucas	1808	cursor.execute("DELETE FROM hints")
108	lucas	1813	query = "EXECUTE h_insert(%(source)s, %(version)s, %(arch)s, %(type)s, %(argument)s, %(file)s, %(comment)s)"
109	lucas	1808	hs = []
110			for hint in hints:
111			h = {}
112			h['file'] = hint[1]
113			h['comment'] = hint[2]
114	lucas	1813	h['argument'] = hint[0].arg
115	lucas	1808	h['type'] = hint[0].type
116			for src in hint[0].pkgs:
117			h['source'] = src
118			if hint[0].versionmap.has_key(src):
119			h['version'] = hint[0].versionmap[src]
120	lucas	1809	if h['version'] == '':
121			h['version'] = None
122	lucas	1808	else:
123			h['version'] = None
124			if hint[0].archmap.has_key(src):
125			h['arch'] = ' '.join(hint[0].archmap[src])
126			else:
127			h['arch'] = None
128	lucas	1809	hs.append(h.copy())
129	lucas	1808	cursor.executemany(query, hs)
130			cursor.execute("DEALLOCATE h_insert")
131	lucas	1976	cursor.execute("ANALYZE hints")