Contents of /bin/list-queue

#!/usr/bin/python
# list-queue -- list security-master queue contents
# Copyright (C) 2011 Florian Weimer <fw@deneb.enyo.de>
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

# This script is intended to be run on security-master to get an
# unprocessed dump of the contents of the embargoed and unembargoed
# queues.
#
# The script reads .deb and .changes files.  A caching database is
# written to ~/.cache.


######################################################################
# Configuration

DIRECTORIES = ('/org/security-master.debian.org/queue/embargoed',
               '/org/security-master.debian.org/queue/unembargoed')

# End Configuration
######################################################################

import json
import os.path
import re
import sqlite3
import sys

from debian.deb822 import Changes
from debian.debfile import DebFile

def createdb():
    cache = os.path.expanduser("~/.cache")
    if not os.path.isdir(cache):
        os.mkdir(cache)
    dbfile = os.path.join(cache, "secure-testing_list-queue.sqlite")
    db = sqlite3.connect(dbfile, isolation_level="IMMEDIATE")
    db.execute("PRAGMA page_size = 4096")
    db.execute("PRAGMA journal_mode = WAL")
    db.execute("""CREATE TABLE IF NOT EXISTS package (
  path TEXT NOT NULL PRIMARY KEY,
  size INTEGER NOT NULL CHECK (size >= 0),
  mtime INTEGER NOT NULL CHECK (size >= 0),
  name TEXT NOT NULL,
  version TEXT NOT NULL,
  arch TEXT NOT NULL,
  source TEXT NOT NULL,
  source_version TEXT NOT NULL
)""")
    db.execute("""CREATE TABLE IF NOT EXISTS changes (
  path TEXT NOT NULL PRIMARY KEY,
  size INTEGER NOT NULL CHECK (size >= 0),
  mtime INTEGER NOT NULL CHECK (size >= 0),
  dist TEXT NOT NULL,
  debs TEXT NOT NULL
)""")
    return db

def readdirs():
    """Returns two dicts, mapping paths to pairs (SIZE, MTIME).

    First dict is for .deb files, second is for .changes files."""
    debs = {}
    changes = {}
    for path in DIRECTORIES:
        for entry in os.listdir(path):
            if entry.startswith("."):
                continue
            name = os.path.join(path, entry)
            stat = os.stat(name)
            where = None
            if entry.endswith(".deb"):
                where = debs
            elif entry.endswith(".changes"):
                where = changes
            if where is not None:
                where[name] = (stat.st_size, stat.st_mtime)                
    return (debs, changes)

def readpackages(db):
    result = {}
    for row in db.execute("SELECT * FROM package"):
        name, size, mtime = row[0:3]
        pkg = tuple(row[3:])
        result[name] = (size, mtime, pkg)
    return result

def readchanges(db):
    result = {}
    for name, size, mtime, dist, debs in db.execute("SELECT * FROM changes"):
        result[name] = (size, mtime, dist, set(debs.split()))
    return result

def deletepaths(db, table, paths):
    db.executemany("DELETE FROM " + table + " WHERE path = ?", paths)

def prepareupdate(db, ondisk, indb, table):
    need_update = [(path, stat) for (path, stat) in ondisk.items()
                   if path not in indb or stat <> tuple(indb[path][0:2])]
    db.executemany("DELETE FROM " + table + " WHERE path = ?",
                   ((path,) for path, _ in need_update))
    return need_update

def expire(db, ondisk, indb, table):
    need_delete = [(path,) for path in indb if path not in ondisk]
    db.executemany("DELETE FROM " + table + " WHERE path = ?", need_delete)
    for (path,) in need_delete:
        del indb[path]

def stripstat(data):
    "Removes the stat pair from the values in data."
    for (key, value) in data.items():
        data[key] = value[2:]

# See debian_support.BinaryPackage.loadtuple().
def deb822totuple(data, re_source=re.compile(
        r'^([a-zA-Z0-9.+-]+)(?:\s+\(([a-zA-Z0-9.+:~-]+)\))?$')):
    """Turns an Deb822-like object into a 5-tuple.

    Returns (PACKAGE-NAME, VERSION, ARCHITECTURE, SOURCE,
    SOURCE-VERSION)."""

    pkg = data["Package"]
    version = data["Version"]
    if "Source" in data:
        source = data.get("Source", None)
        match = re_source.match(source)
        if match is None:
            raise ValueError("invalid Source field: " + repr(source))
        src, src_version = match.groups()
        if src_version is None:
            src_version = version
    else:
        src = pkg
        src_version = version
    return (pkg, version, data["Architecture"], src, src_version)

def updatepackages(db, ondisk):
    "Updates the package table from the file system."
    indb = readpackages(db)
    expire(db, ondisk, indb, "package")
    
    # Update the cache in indb and the database
    need_update = prepareupdate(db, ondisk, indb, "package")
    def do_update():
        for (path, stat) in need_update:
            deb = DebFile(path)
            pkg = deb822totuple(deb.debcontrol())
            indb[path] = stat + (pkg,)
            yield (path,) + stat + pkg
    db.executemany("INSERT INTO package VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
                   do_update())

    stripstat(indb)
    return indb

def updatechanges(db, ondisk):
    "Updates the changes table from the file system."
    indb = readchanges(db)
    expire(db, ondisk, indb, "changes")

    # Update the cache in indb and the database
    need_update = prepareupdate(db, ondisk, indb, "changes")
    def do_update():
        for (path, stat) in need_update:
            changes = Changes(file(path))
            try:
                dist = changes["Distribution"]
                debs = set(pkg["name"] for pkg in changes["Checksums-Sha1"])
            except KeyError, e:
                raise IOError("missing key in " + repr(path) + ": "
                              + repr(e.args[0]))
            indb[path] = stat + (dist, debs)
            yield (path,) + stat + (dist, " ".join(sorted(debs)),)
    db.executemany("INSERT INTO changes VALUES (?, ?, ?, ?, ?)", do_update())

    stripstat(indb)
    return indb

def distdict(changes):
    "Computes a dict from .deb files to sets of distributions"
    result = {}
    for path, (dist, debs) in changes.items():
        base = os.path.dirname(path)
        distset = set((dist,))
        for deb in debs:
            name = os.path.join(base, deb)
            if name in result:
                result[name].add(dist)
            else:
                result[name] = set(distset)
    return result

def pkgwithdist(debs, dists):
    """Merge packages and distribution information.

    Returns a list of tuples (PACKAGE-NAME, VERSION, ARCHITECTURE,
    SOURCE-NAME, SOURCE-VERSION, TUPLE-OF-DISTRIBUTIONS).
    """
    return [pkg + (sorted(dists.get(path, ())),)
            for (path, (pkg,)) in debs.items()]

def main():
    db = createdb()
    debs, changes = readdirs()
    debs = updatepackages(db, debs)
    changes = updatechanges(db, changes)
    dists = distdict(changes)
    db.commit()
    result = {
        "version" : 1,
        "binary" : pkgwithdist(debs, dists),
        }
    print json.dumps(result)
main()
1	#!/usr/bin/python
2	# list-queue -- list security-master queue contents
3	# Copyright (C) 2011 Florian Weimer <fw@deneb.enyo.de>
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License as published by
7	# the Free Software Foundation; either version 2 of the License, or
8	# (at your option) any later version.
9	#
10	# This program is distributed in the hope that it will be useful,
11	# but WITHOUT ANY WARRANTY; without even the implied warranty of
12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	# GNU General Public License for more details.
14	#
15	# You should have received a copy of the GNU General Public License
16	# along with this program; if not, write to the Free Software
17	# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
19	# This script is intended to be run on security-master to get an
20	# unprocessed dump of the contents of the embargoed and unembargoed
21	# queues.
22	#
23	# The script reads .deb and .changes files. A caching database is
24	# written to ~/.cache.
25
26
27	######################################################################
28	# Configuration
29
30	DIRECTORIES = ('/org/security-master.debian.org/queue/embargoed',
31	'/org/security-master.debian.org/queue/unembargoed')
32
33	# End Configuration
34	######################################################################
35
36	import json
37	import os.path
38	import re
39	import sqlite3
40	import sys
41
42	from debian.deb822 import Changes
43	from debian.debfile import DebFile
44
45	def createdb():
46	cache = os.path.expanduser("~/.cache")
47	if not os.path.isdir(cache):
48	os.mkdir(cache)
49	dbfile = os.path.join(cache, "secure-testing_list-queue.sqlite")
50	db = sqlite3.connect(dbfile, isolation_level="IMMEDIATE")
51	db.execute("PRAGMA page_size = 4096")
52	db.execute("PRAGMA journal_mode = WAL")
53	db.execute("""CREATE TABLE IF NOT EXISTS package (
54	path TEXT NOT NULL PRIMARY KEY,
55	size INTEGER NOT NULL CHECK (size >= 0),
56	mtime INTEGER NOT NULL CHECK (size >= 0),
57	name TEXT NOT NULL,
58	version TEXT NOT NULL,
59	arch TEXT NOT NULL,
60	source TEXT NOT NULL,
61	source_version TEXT NOT NULL
62	)""")
63	db.execute("""CREATE TABLE IF NOT EXISTS changes (
64	path TEXT NOT NULL PRIMARY KEY,
65	size INTEGER NOT NULL CHECK (size >= 0),
66	mtime INTEGER NOT NULL CHECK (size >= 0),
67	dist TEXT NOT NULL,
68	debs TEXT NOT NULL
69	)""")
70	return db
71
72	def readdirs():
73	"""Returns two dicts, mapping paths to pairs (SIZE, MTIME).
74
75	First dict is for .deb files, second is for .changes files."""
76	debs = {}
77	changes = {}
78	for path in DIRECTORIES:
79	for entry in os.listdir(path):
80	if entry.startswith("."):
81	continue
82	name = os.path.join(path, entry)
83	stat = os.stat(name)
84	where = None
85	if entry.endswith(".deb"):
86	where = debs
87	elif entry.endswith(".changes"):
88	where = changes
89	if where is not None:
90	where[name] = (stat.st_size, stat.st_mtime)
91	return (debs, changes)
92
93	def readpackages(db):
94	result = {}
95	for row in db.execute("SELECT * FROM package"):
96	name, size, mtime = row[0:3]
97	pkg = tuple(row[3:])
98	result[name] = (size, mtime, pkg)
99	return result
100
101	def readchanges(db):
102	result = {}
103	for name, size, mtime, dist, debs in db.execute("SELECT * FROM changes"):
104	result[name] = (size, mtime, dist, set(debs.split()))
105	return result
106
107	def deletepaths(db, table, paths):
108	db.executemany("DELETE FROM " + table + " WHERE path = ?", paths)
109
110	def prepareupdate(db, ondisk, indb, table):
111	need_update = [(path, stat) for (path, stat) in ondisk.items()
112	if path not in indb or stat <> tuple(indb[path][0:2])]
113	db.executemany("DELETE FROM " + table + " WHERE path = ?",
114	((path,) for path, _ in need_update))
115	return need_update
116
117	def expire(db, ondisk, indb, table):
118	need_delete = [(path,) for path in indb if path not in ondisk]
119	db.executemany("DELETE FROM " + table + " WHERE path = ?", need_delete)
120	for (path,) in need_delete:
121	del indb[path]
122
123	def stripstat(data):
124	"Removes the stat pair from the values in data."
125	for (key, value) in data.items():
126	data[key] = value[2:]
127
128	# See debian_support.BinaryPackage.loadtuple().
129	def deb822totuple(data, re_source=re.compile(
130	r'^([a-zA-Z0-9.+-]+)(?:\s+\(([a-zA-Z0-9.+:~-]+)\))?$')):
131	"""Turns an Deb822-like object into a 5-tuple.
132
133	Returns (PACKAGE-NAME, VERSION, ARCHITECTURE, SOURCE,
134	SOURCE-VERSION)."""
135
136	pkg = data["Package"]
137	version = data["Version"]
138	if "Source" in data:
139	source = data.get("Source", None)
140	match = re_source.match(source)
141	if match is None:
142	raise ValueError("invalid Source field: " + repr(source))
143	src, src_version = match.groups()
144	if src_version is None:
145	src_version = version
146	else:
147	src = pkg
148	src_version = version
149	return (pkg, version, data["Architecture"], src, src_version)
150
151	def updatepackages(db, ondisk):
152	"Updates the package table from the file system."
153	indb = readpackages(db)
154	expire(db, ondisk, indb, "package")
155
156	# Update the cache in indb and the database
157	need_update = prepareupdate(db, ondisk, indb, "package")
158	def do_update():
159	for (path, stat) in need_update:
160	deb = DebFile(path)
161	pkg = deb822totuple(deb.debcontrol())
162	indb[path] = stat + (pkg,)
163	yield (path,) + stat + pkg
164	db.executemany("INSERT INTO package VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
165	do_update())
166
167	stripstat(indb)
168	return indb
169
170	def updatechanges(db, ondisk):
171	"Updates the changes table from the file system."
172	indb = readchanges(db)
173	expire(db, ondisk, indb, "changes")
174
175	# Update the cache in indb and the database
176	need_update = prepareupdate(db, ondisk, indb, "changes")
177	def do_update():
178	for (path, stat) in need_update:
179	changes = Changes(file(path))
180	try:
181	dist = changes["Distribution"]
182	debs = set(pkg["name"] for pkg in changes["Checksums-Sha1"])
183	except KeyError, e:
184	raise IOError("missing key in " + repr(path) + ": "
185	+ repr(e.args[0]))
186	indb[path] = stat + (dist, debs)
187	yield (path,) + stat + (dist, " ".join(sorted(debs)),)
188	db.executemany("INSERT INTO changes VALUES (?, ?, ?, ?, ?)", do_update())
189
190	stripstat(indb)
191	return indb
192
193	def distdict(changes):
194	"Computes a dict from .deb files to sets of distributions"
195	result = {}
196	for path, (dist, debs) in changes.items():
197	base = os.path.dirname(path)
198	distset = set((dist,))
199	for deb in debs:
200	name = os.path.join(base, deb)
201	if name in result:
202	result[name].add(dist)
203	else:
204	result[name] = set(distset)
205	return result
206
207	def pkgwithdist(debs, dists):
208	"""Merge packages and distribution information.
209
210	Returns a list of tuples (PACKAGE-NAME, VERSION, ARCHITECTURE,
211	SOURCE-NAME, SOURCE-VERSION, TUPLE-OF-DISTRIBUTIONS).
212	"""
213	return [pkg + (sorted(dists.get(path, ())),)
214	for (path, (pkg,)) in debs.items()]
215
216	def main():
217	db = createdb()
218	debs, changes = readdirs()
219	debs = updatepackages(db, debs)
220	changes = updatechanges(db, changes)
221	dists = distdict(changes)
222	db.commit()
223	result = {
224	"version" : 1,
225	"binary" : pkgwithdist(debs, dists),
226	}
227	print json.dumps(result)
228	main()