src/udd/sources_gatherer.py

#/usr/bin/env python
# Last-Modified: <Sun Aug 17 12:07:25 2008>
# This file is a part of the Ultimate Debian Database project

import debian_bundle.deb822
import gzip
import os
import sys
import aux
import tempfile
from aux import ConfigException
from aux import null_or_quote, quote
from gatherer import gatherer

def get_gatherer(connection, config, source):
  return sources_gatherer(connection, config, source)

class sources_gatherer(gatherer):
  "This class imports the data from Sources.gz files into the database"
  mandatory = {'Format': 0, 'Maintainer': 0, 'Package': 0, 'Version': 0, 'Files': 0}
  non_mandatory = {'Uploaders': 0, 'Binary': 0, 'Architecture': 0,
      'Standards-Version': 0, 'Homepage': 0, 'Build-Depends': 0,
      'Build-Depends-Indep': 0, 'Build-Conflicts': 0, 'Build-Conflicts-Indep': 0,
      'Priority': 0, 'Section': 0, 'Python-Version': 0, 'Checksums-Sha1':0,
      'Checksums-Sha256':0, 'Original-Maintainer':0, 'Dm-Upload-Allowed':0} 
  ignorable = {'Vcs-Arch': 0, 'Vcs-Bzr': 0,
      'Vcs-Cvs': 0, 'Vcs-Darcs': 0, 'Vcs-Git': 0, 'Vcs-Hg': 0, 'Vcs-Svn': 0,
      'X-Vcs-Browser': 0, 'Vcs-Browser': 0, 'X-Vcs-Bzr': 0, 'X-Vcs-Darcs': 0, 'X-Vcs-Svn': 0,
      'Directory':0}
  vcs = { 'Arch':0, 'Bzr':0, 'Cvs':0, 'Darcs':0, 'Git':0, 'Hg':0, 'Svn':0}

  warned_about = {}

  def __init__(self, connection, config, source):
    gatherer.__init__(self, connection, config, source)
    self._distr = None
    self.assert_my_config('directory', 'components', 'distribution', 'release', 'sources-table', 'sources-schema')

  def build_dict(self, control):
    """Build a dictionary from the control dictionary.

    Influenced by global variables mandatory, non_mandatory and ignorable"""
    d = {}
    for k in sources_gatherer.mandatory:
      if k not in control:
        raise "Mandatory field %s not specified" % k
      d[k] = control[k]
    for k in sources_gatherer.non_mandatory:
      if k in control:
        d[k] = control[k]
      else:
        d[k] = None
    
    d['Vcs-Type'] = None
    d['Vcs-Url'] = None
    for vcs in sources_gatherer.vcs:
      if control.has_key("Vcs-"+vcs):  
        d['Vcs-Type'] = vcs
        d['Vcs-Url'] = control["Vcs-"+vcs]
        break
      elif control.has_key("X-Vcs-"+vcs):  
        d['Vcs-Type'] = vcs
        d['Vcs-Url'] = control["X-Vcs-"+vcs]
        break
    if control.has_key("Vcs-Browser"):  
        d['Vcs-Browser'] = control["Vcs-Browser"]
    elif control.has_key("X-Vcs-Browser"):  
        d['Vcs-Browser'] = control["X-Vcs-Browser"]
    else:
        d['Vcs-Browser'] = None
    
    for k in control.keys():
      if k not in sources_gatherer.mandatory and k not in sources_gatherer.non_mandatory and k not in sources_gatherer.ignorable:
        if k not in sources_gatherer.warned_about:
          sources_gatherer.warned_about[k] = 1
        else:
          sources_gatherer.warned_about[k] += 1
    return d

  def import_sources(self, file):
    """Import the sources from the file into the database-connection conn.

    Sequence has to have an iterator interface, that yields a line every time it
    is called.The Format of the file is expected to be that of a debian
    source file."""
    cur = self.cursor()
    for control in debian_bundle.deb822.Packages.iter_paragraphs(file):
      d = self.build_dict(control)
      query = """EXECUTE source_insert
          (%(Package)s, %(Version)s, %(Maintainer)s, %(Format)s, %(Files)s,
          %(Uploaders)s, %(Binary)s, %(Architecture)s, %(Standards-Version)s,
          %(Homepage)s, %(Build-Depends)s, %(Build-Depends-Indep)s,
          %(Build-Conflicts)s, %(Build-Conflicts-Indep)s, %(Priority)s,
          %(Section)s, %(Vcs-Type)s, %(Vcs-Url)s, %(Vcs-Browser)s,
          %(Python-Version)s, %(Checksums-Sha1)s, %(Checksums-Sha256)s,
          %(Original-Maintainer)s, %(Dm-Upload-Allowed)s)
          """ 
      cur.execute(query, d)

  def tables(self):
    return [self.my_config['sources-table']]

  def run(self):
    src_cfg = self.my_config

    table = src_cfg['sources-table']

    aux.debug = self.config['general']['debug']

    cur = self.cursor()

    for comp in src_cfg['components']:
      path = os.path.join(src_cfg['directory'], comp, 'source', 'Sources.gz')
      cur.execute("DELETE from %s WHERE Distribution = '%s' AND\
        release = '%s' AND component = '%s'"\
        % (table, src_cfg['distribution'], src_cfg['release'], comp))
      try:
        query = """PREPARE source_insert as INSERT INTO %s
          (Source, Version, Maintainer, Format, Files, Uploaders, Bin,
          Architecture, Standards_Version, Homepage, Build_Depends,
          Build_Depends_Indep, Build_Conflicts, Build_Conflicts_Indep, Priority,
          Section, Vcs_Type, Vcs_Url, Vcs_Browser, python_version, checksums_sha1,
          checksums_sha256, original_maintainer, dm_upload_allowed,
          Distribution, Release, Component)
        VALUES
          ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16,
          $17, $18, $19, $20, $21, $22, $23, $24, '%s', '%s', '%s')"""\
          % (table, src_cfg['distribution'], src_cfg['release'], comp)
        cur.execute(query)

        aux.print_debug("Reading file " + path)
        # Copy content from gzipped file to temporary file, so that apt_pkg is
        # used by debian_bundle
        tmp = tempfile.NamedTemporaryFile()
        file = gzip.open(path)
        tmp.write(file.read())
        file.close()
        tmp.seek(0)
        aux.print_debug("Importing from " + path)
        self.import_sources(open(tmp.name))
        tmp.close()
      except IOError, (e, message):
        print "Could not read packages from %s: %s" % (path, message)
      cur.execute("DEALLOCATE source_insert")

    self.print_warnings()

  def setup(self):
    if 'schema-dir' in self.config['general']:
      schema_dir = self.config['general']['schema-dir']
      if 'sources-schema' in self.my_config:
        schema = schema_dir + '/' + self.my_config['sources-schema']
        self.eval_sql_file(schema, self.my_config)
      else:
        raise Exception("'packages-schema' not specified for source " + self.source)
    else:
      raise Exception("'schema-dir' not specified")

  def print_warnings(self):
    for key in sources_gatherer.warned_about:
      print "Unknown key %s appeared %d times" % (key, sources_gatherer.warned_about[key])
1	#/usr/bin/env python
2	# Last-Modified: <Sun Aug 17 12:07:25 2008>
3	# This file is a part of the Ultimate Debian Database project
4
5	import debian_bundle.deb822
6	import gzip
7	import os
8	import sys
9	import aux
10	import tempfile
11	from aux import ConfigException
12	from aux import null_or_quote, quote
13	from gatherer import gatherer
14
15	def get_gatherer(connection, config, source):
16	return sources_gatherer(connection, config, source)
17
18	class sources_gatherer(gatherer):
19	"This class imports the data from Sources.gz files into the database"
20	mandatory = {'Format': 0, 'Maintainer': 0, 'Package': 0, 'Version': 0, 'Files': 0}
21	non_mandatory = {'Uploaders': 0, 'Binary': 0, 'Architecture': 0,
22	'Standards-Version': 0, 'Homepage': 0, 'Build-Depends': 0,
23	'Build-Depends-Indep': 0, 'Build-Conflicts': 0, 'Build-Conflicts-Indep': 0,
24	'Priority': 0, 'Section': 0, 'Python-Version': 0, 'Checksums-Sha1':0,
25	'Checksums-Sha256':0, 'Original-Maintainer':0, 'Dm-Upload-Allowed':0}
26	ignorable = {'Vcs-Arch': 0, 'Vcs-Bzr': 0,
27	'Vcs-Cvs': 0, 'Vcs-Darcs': 0, 'Vcs-Git': 0, 'Vcs-Hg': 0, 'Vcs-Svn': 0,
28	'X-Vcs-Browser': 0, 'Vcs-Browser': 0, 'X-Vcs-Bzr': 0, 'X-Vcs-Darcs': 0, 'X-Vcs-Svn': 0,
29	'Directory':0}
30	vcs = { 'Arch':0, 'Bzr':0, 'Cvs':0, 'Darcs':0, 'Git':0, 'Hg':0, 'Svn':0}
31
32	warned_about = {}
33
34	def __init__(self, connection, config, source):
35	gatherer.__init__(self, connection, config, source)
36	self._distr = None
37	self.assert_my_config('directory', 'components', 'distribution', 'release', 'sources-table', 'sources-schema')
38
39	def build_dict(self, control):
40	"""Build a dictionary from the control dictionary.
41
42	Influenced by global variables mandatory, non_mandatory and ignorable"""
43	d = {}
44	for k in sources_gatherer.mandatory:
45	if k not in control:
46	raise "Mandatory field %s not specified" % k
47	d[k] = control[k]
48	for k in sources_gatherer.non_mandatory:
49	if k in control:
50	d[k] = control[k]
51	else:
52	d[k] = None
53
54	d['Vcs-Type'] = None
55	d['Vcs-Url'] = None
56	for vcs in sources_gatherer.vcs:
57	if control.has_key("Vcs-"+vcs):
58	d['Vcs-Type'] = vcs
59	d['Vcs-Url'] = control["Vcs-"+vcs]
60	break
61	elif control.has_key("X-Vcs-"+vcs):
62	d['Vcs-Type'] = vcs
63	d['Vcs-Url'] = control["X-Vcs-"+vcs]
64	break
65	if control.has_key("Vcs-Browser"):
66	d['Vcs-Browser'] = control["Vcs-Browser"]
67	elif control.has_key("X-Vcs-Browser"):
68	d['Vcs-Browser'] = control["X-Vcs-Browser"]
69	else:
70	d['Vcs-Browser'] = None
71
72	for k in control.keys():
73	if k not in sources_gatherer.mandatory and k not in sources_gatherer.non_mandatory and k not in sources_gatherer.ignorable:
74	if k not in sources_gatherer.warned_about:
75	sources_gatherer.warned_about[k] = 1
76	else:
77	sources_gatherer.warned_about[k] += 1
78	return d
79
80	def import_sources(self, file):
81	"""Import the sources from the file into the database-connection conn.
82
83	Sequence has to have an iterator interface, that yields a line every time it
84	is called.The Format of the file is expected to be that of a debian
85	source file."""
86	cur = self.cursor()
87	for control in debian_bundle.deb822.Packages.iter_paragraphs(file):
88	d = self.build_dict(control)
89	query = """EXECUTE source_insert
90	(%(Package)s, %(Version)s, %(Maintainer)s, %(Format)s, %(Files)s,
91	%(Uploaders)s, %(Binary)s, %(Architecture)s, %(Standards-Version)s,
92	%(Homepage)s, %(Build-Depends)s, %(Build-Depends-Indep)s,
93	%(Build-Conflicts)s, %(Build-Conflicts-Indep)s, %(Priority)s,
94	%(Section)s, %(Vcs-Type)s, %(Vcs-Url)s, %(Vcs-Browser)s,
95	%(Python-Version)s, %(Checksums-Sha1)s, %(Checksums-Sha256)s,
96	%(Original-Maintainer)s, %(Dm-Upload-Allowed)s)
97	"""
98	cur.execute(query, d)
99
100	def tables(self):
101	return [self.my_config['sources-table']]
102
103	def run(self):
104	src_cfg = self.my_config
105
106	table = src_cfg['sources-table']
107
108	aux.debug = self.config['general']['debug']
109
110	cur = self.cursor()
111
112	for comp in src_cfg['components']:
113	path = os.path.join(src_cfg['directory'], comp, 'source', 'Sources.gz')
114	cur.execute("DELETE from %s WHERE Distribution = '%s' AND\
115	release = '%s' AND component = '%s'"\
116	% (table, src_cfg['distribution'], src_cfg['release'], comp))
117	try:
118	query = """PREPARE source_insert as INSERT INTO %s
119	(Source, Version, Maintainer, Format, Files, Uploaders, Bin,
120	Architecture, Standards_Version, Homepage, Build_Depends,
121	Build_Depends_Indep, Build_Conflicts, Build_Conflicts_Indep, Priority,
122	Section, Vcs_Type, Vcs_Url, Vcs_Browser, python_version, checksums_sha1,
123	checksums_sha256, original_maintainer, dm_upload_allowed,
124	Distribution, Release, Component)
125	VALUES
126	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16,
127	$17, $18, $19, $20, $21, $22, $23, $24, '%s', '%s', '%s')"""\
128	% (table, src_cfg['distribution'], src_cfg['release'], comp)
129	cur.execute(query)
130
131	aux.print_debug("Reading file " + path)
132	# Copy content from gzipped file to temporary file, so that apt_pkg is
133	# used by debian_bundle
134	tmp = tempfile.NamedTemporaryFile()
135	file = gzip.open(path)
136	tmp.write(file.read())
137	file.close()
138	tmp.seek(0)
139	aux.print_debug("Importing from " + path)
140	self.import_sources(open(tmp.name))
141	tmp.close()
142	except IOError, (e, message):
143	print "Could not read packages from %s: %s" % (path, message)
144	cur.execute("DEALLOCATE source_insert")
145
146	self.print_warnings()
147
148	def setup(self):
149	if 'schema-dir' in self.config['general']:
150	schema_dir = self.config['general']['schema-dir']
151	if 'sources-schema' in self.my_config:
152	schema = schema_dir + '/' + self.my_config['sources-schema']
153	self.eval_sql_file(schema, self.my_config)
154	else:
155	raise Exception("'packages-schema' not specified for source " + self.source)
156	else:
157	raise Exception("'schema-dir' not specified")
158
159	def print_warnings(self):
160	for key in sources_gatherer.warned_about:
161	print "Unknown key %s appeared %d times" % (key, sources_gatherer.warned_about[key])