| 1 |
#!/usr/bin/env python
|
| 2 |
|
| 3 |
"""
|
| 4 |
This script imports the popcon data into the database
|
| 5 |
See http://popcon.debian.org/
|
| 6 |
"""
|
| 7 |
|
| 8 |
import aux
|
| 9 |
import sys
|
| 10 |
import gzip
|
| 11 |
from gatherer import gatherer
|
| 12 |
import re
|
| 13 |
|
| 14 |
def get_gatherer(connection, config, source):
|
| 15 |
return popcon_gatherer(connection, config, source)
|
| 16 |
|
| 17 |
class popcon_gatherer(gatherer):
|
| 18 |
def __init__(self, connection, config, source):
|
| 19 |
gatherer.__init__(self, connection, config, source)
|
| 20 |
|
| 21 |
self.assert_my_config('path', 'table', 'packages-table', 'schema')
|
| 22 |
|
| 23 |
def tables(self):
|
| 24 |
ret = []
|
| 25 |
for sub in ('', '_src', '_src_average'):
|
| 26 |
ret.append(self.my_config['table'] + sub)
|
| 27 |
return ret
|
| 28 |
|
| 29 |
def run(self):
|
| 30 |
my_config = self.my_config
|
| 31 |
|
| 32 |
table = my_config['table']
|
| 33 |
table_src = table + "_src"
|
| 34 |
table_src_average = table + "_src_average"
|
| 35 |
|
| 36 |
cur = self.cursor()
|
| 37 |
|
| 38 |
cur.execute("PREPARE pop_insert AS INSERT INTO %s (package, insts, vote, olde, recent, nofiles) VALUES ($1, $2, $3, $4, $5, $6)" % (table))
|
| 39 |
|
| 40 |
popcon = gzip.open(my_config['path'])
|
| 41 |
|
| 42 |
cur.execute("DELETE FROM " + table)
|
| 43 |
cur.execute("DELETE FROM " + table_src)
|
| 44 |
cur.execute("DELETE FROM " + table_src_average)
|
| 45 |
|
| 46 |
# used for ignoring ubuntu's broken popcon lines
|
| 47 |
ascii_match = re.compile("^[A-Za-z0-9-.+_]+$")
|
| 48 |
|
| 49 |
linenr = 0
|
| 50 |
d = {}
|
| 51 |
for line in popcon:
|
| 52 |
linenr += 1
|
| 53 |
name, data = line.split(None, 1)
|
| 54 |
if name == "Submissions:":
|
| 55 |
d['data'] = int(data)
|
| 56 |
cur.execute("INSERT INTO " + table + " (package, vote) VALUES ('_submissions', %(data)s)", d)
|
| 57 |
continue
|
| 58 |
try:
|
| 59 |
(name, vote, old, recent, nofiles) = data.split()
|
| 60 |
d['name'] = name
|
| 61 |
for k in ['vote', 'old', 'recent', 'nofiles']:
|
| 62 |
exec '%s = int(%s)' % (k,k)
|
| 63 |
exec 'd["%s"] = %s' % (k,k)
|
| 64 |
d['insts'] = vote + old + recent + nofiles
|
| 65 |
if ascii_match.match(name) == None:
|
| 66 |
print "%s:%d - illegal package name %s" % (my_config['path'], linenr, line.rstrip("\n"))
|
| 67 |
continue
|
| 68 |
query = "EXECUTE pop_insert(%(name)s, %(insts)s, %(vote)s, %(old)s, %(recent)s, %(nofiles)s)"
|
| 69 |
cur.execute(query, d)
|
| 70 |
except ValueError:
|
| 71 |
continue
|
| 72 |
|
| 73 |
cur.execute("DEALLOCATE pop_insert")
|
| 74 |
|
| 75 |
#calculate _src and _src_avg
|
| 76 |
cur.execute("""
|
| 77 |
INSERT INTO %(table)s_src (source, insts, vote, olde, recent, nofiles)
|
| 78 |
SELECT DISTINCT pkgs.source, max(insts) AS insts, max(vote) AS vote,
|
| 79 |
max(olde) AS old, max(recent) AS recent, max(nofiles) as nofiles
|
| 80 |
FROM %(table)s, %(packages-table)s_summary AS pkgs
|
| 81 |
WHERE %(table)s.package = pkgs.package
|
| 82 |
GROUP BY pkgs.source;
|
| 83 |
""" % my_config)
|
| 84 |
cur.execute("""
|
| 85 |
INSERT INTO %(table)s_src_average (source, insts, vote, olde, recent,
|
| 86 |
nofiles)
|
| 87 |
SELECT pkgs.source, avg(insts) AS insts, avg(vote) AS vote,
|
| 88 |
avg(olde) AS old, avg(recent) AS recent, avg(nofiles) as nofiles
|
| 89 |
FROM %(table)s, %(packages-table)s_summary AS pkgs
|
| 90 |
WHERE %(table)s.package = pkgs.package
|
| 91 |
GROUP BY pkgs.source;
|
| 92 |
""" % my_config)
|
| 93 |
cur.execute("ANALYZE " + table)
|
| 94 |
cur.execute("ANALYZE " + table_src)
|
| 95 |
cur.execute("ANALYZE " + table_src_average)
|
| 96 |
|
| 97 |
if __name__ == '__main__':
|
| 98 |
main()
|
Parent Directory
| 
Revision Log