# parsers -- various text file parsers # Copyright (C) 2010 Florian Weimer # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import operator import re import debian_support import regexpcase import xcollections import xpickle FORMAT = "1" @xpickle.loader("BINARY" + FORMAT) def binarypackages(name, f): """Returns a sequence of binary package names""" obj = set(v for p in debian_support.PackageFile(name, f) for k, v in p if k == "Package") obj = list(obj) obj.sort() return tuple(obj) @xpickle.loader("SOURCE" + FORMAT) def sourcepackages(name, f): """Returns a dictionary of source package objects. The keys are strings, containing the source package name, the values are corresponding source package versions.""" data = {} for p in debian_support.PackageFile(name, f): pkg_name, pkg_version = (None, None) for name, contents in p: if name == "Package": pkg_name = intern(contents) elif name == "Version": pkg_version = contents if pkg_name is None: raise SyntaxError("package record does not contain package name") if pkg_version is None: raise SyntaxError("package record for %s does not contain version" % pkg_name) if pkg_name in data: oversion = debian_support.Version(data[pkg_name]) if oversion >= debian_support.Version(pkg_version): continue data[pkg_name] = pkg_version return data def _sortedtuple(seq): l = list(seq) l.sort() return tuple(l) Message = xcollections.namedtuple("Message", "file line level message") def addmessage(messages, file, line, level, msg): if level not in ("error", "warning"): raise ValueError("invalid message level: " + repr(level)) messages.append(Message(file, line, level, msg)) FlagAnnotation = xcollections.namedtuple("FlagAnnotation", "line type") StringAnnotation = xcollections.namedtuple("StringAnnotation", "line type description") XrefAnnotation = xcollections.namedtuple("XrefAnnotation", "line type bugs") PackageAnnotation = xcollections.namedtuple( "PackageAnnotation", "line type release package kind version description " + "urgency debian_bugs bug_filed") def _annotationdispatcher(): # Parser for inner annotations, like (bug #1345; low) urgencies=set("unimportant low medium high".split()) @regexpcase.rule('(bug filed|%s)' % '|'.join(urgencies)) def innerflag(groups, file, line, messages, flags, bugs): f = groups[0] if f in flags: addmessage(messages, file, line, "error", "duplicate flag: " + repr(f)) else: flags.add(f) @regexpcase.rule(r'bug #(\d+)') def innerbug(groups, file, line, messages, flags, bugs): no = int(groups[0]) if no in bugs: messages.add(file, line, "error", "duplicate bug number: " + groups[0]) else: bugs.add(no) def innerdefault(text, file, line, messages, flags, bugs): addmessage(messages, file, line, "error", "invalid inner annotation: " + repr(text)) innerdispatch = regexpcase.RegexpCase((innerflag, innerbug), default=innerdefault) def parseinner(file, line, messages, inner): if not inner: return (None, (), False) flags = set() bugs = set() for innerann in inner.split(";"): innerdispatch(innerann.strip(), file, line, messages, flags, bugs) urgency = urgencies.intersection(flags) if urgency: if len(urgency) > 1: addmessage(messages, file, line, "error", "multiple urgencies: " + ", ".join(urgency)) else: urgency = urgency.pop() else: urgency = None bug_filed = "bug filed" in flags if bugs and bug_filed: addmessage(messages, file, line, "error", "'bug filed' and bug numbers listed") bug_filed = False return (urgency, _sortedtuple(bugs), bug_filed) # Parsers for indented annotations (NOT-FOR-US:, " - foo " etc.) @regexpcase.rule(r'(?:\[([a-z]+)\]\s)?-\s([A-Za-z0-9:.+-]+)\s*' + r'(?:\s([A-Za-z0-9:.+~-]+)\s*)?(?:\s\((.*)\))?') def package_version(groups, file, line, messages, anns): release, package, version, inner = groups inner = parseinner(file, line, messages, inner) if version is None: kind = "unfixed" else: kind = "fixed" anns.append(PackageAnnotation( *((line, "package", release, package, kind, version, None) + inner))) pseudo_freetext = "no-dsa not-affected".split() pseudo_struct = set("unfixed removed end-of-life itp undetermined".split()) @regexpcase.rule(r'(?:\[([a-z]+)\]\s)?-\s([A-Za-z0-9:.+-]+)' + r'\s+<([a-z-]+)>\s*(?:\s\((.*)\))?') def package_pseudo(groups, file, line, messages, anns): release, package, version, inner = groups if version in pseudo_freetext: anns.append(PackageAnnotation( line, "package", release, package, version, None, inner, None, (), False)) elif version in pseudo_struct: inner = parseinner(file, line, messages, inner) if version == "itp" and not inner[1]: addmessage(messages, file, line, "error", " needs Debian bug reference") anns.append(PackageAnnotation( *((line, "package", release, package, version, None, None) + inner))) else: addmessage(messages, file, line, "error", "invalid pseudo-version: " + repr(version)) @regexpcase.rule(r'\{(.*)\}') def xref(groups, file, line, messages, anns): x = _sortedtuple(groups[0].strip().split()) if x: anns.append(XrefAnnotation(line, "xref", x)) else: addmessage(messages, file, line, "error", "empty cross-reference") return regexpcase.RegexpCase( ((r'(RESERVED|REJECTED)', lambda groups, file, line, messages, anns: anns.append(FlagAnnotation(line, groups[0]))), (r'(NOT-FOR-US|NOTE|TODO):\s+(\S.*)', lambda groups, file, line, messages, anns: anns.append(StringAnnotation(line, *groups))), package_version, package_pseudo, xref), prefix=r"\s+", suffix=r"\s*", default=lambda text, file, line, messages, anns: addmessage(messages, file, line, "error", "invalid annotation")) _annotationdispatcher = _annotationdispatcher() def _test(): o = binarypackages("../../data/packages/sid__main_i386_Packages") assert type(o) == type(()) assert "bash" in o o = sourcepackages("../../data/packages/sid__main_Sources") assert type(o) == type({}) assert "bash" in o for (line, res, xmsgs) in [ (' - foo ', PackageAnnotation(17, "package", None, "foo", "unfixed", None, None, None, (), False), ()), (' - foo', PackageAnnotation(17, "package", None, "foo", "unfixed", None, None, None, (), False), ()), (' [lenny] - foo ', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, None, None, (), False), ()), (' [lenny] - foo (bug #1234)', PackageAnnotation(17, "package", "lenny", "foo", "undetermined", None, None, None, (1234,), False), ()), (' [lenny] - foo (bug #1234)', PackageAnnotation(17, "package", "lenny", "foo", "itp", None, None, None, (1234,), False), ()), (' [lenny] - foo ', PackageAnnotation(17, "package", "lenny", "foo", "itp", None, None, None, (), False), (Message("CVE", 17, "error", " needs Debian bug reference"),)), (' [lenny] - foo 1.0', PackageAnnotation(17, "package", "lenny", "foo", "fixed", "1.0" , None, None, (), False), ()), (' [lenny] - foo (bug filed)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, None, None, (), True), ()), (' [lenny] - foo (bug filed; bug #1234)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, None, None, (1234,), False), (Message("CVE", 17, "error", "'bug filed' and bug numbers listed"),)), (' [lenny] - foo (low)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, None, "low", (), False), ()), (' [lenny] - foo (low; low)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, None, "low", (), False), (Message("CVE", 17, "error", "duplicate flag: 'low'"),)), (' [lenny] - foo (bug #1234; garbled)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, None, None, (1234,), False), (Message("CVE", 17, "error", "invalid inner annotation: 'garbled'"),)), (' [lenny] - foo (explanation goes here)', PackageAnnotation(17, "package", "lenny", "foo", "no-dsa", None, "explanation goes here", None, (), False), ()), (' [lenny] - foo (explanation goes here)', PackageAnnotation(17, "package", "lenny", "foo", "not-affected", None, "explanation goes here", None, (), False), ()), ('\t{CVE-2009-1234 CVE-2009-1235}', XrefAnnotation(17, "xref", tuple("CVE-2009-1234 CVE-2009-1235".split())), ()), ('\t{}', None, (Message("CVE", 17, "error", "empty cross-reference"),)), (' NOT-FOR-US: Plan 9', StringAnnotation(17, "NOT-FOR-US", "Plan 9"), ()), (' TODO: to-do', StringAnnotation(17, "TODO", "to-do"), ()), (' NOTE: note', StringAnnotation(17, "NOTE", "note"), ()), (' RESERVED', FlagAnnotation(17, 'RESERVED'), ()), (' REJECTED', FlagAnnotation(17, 'REJECTED'), ()), (' garbled', None, (Message("CVE", 17, "error", "invalid annotation"),)), (' [lenny] - foo (bug #1234)', None, (Message("CVE", 17, "error", "invalid pseudo-version: 'garbled'"),)), ]: anns = [] msgs = [] _annotationdispatcher(line, "CVE", 17, msgs, anns) assert tuple(msgs) == xmsgs, repr(msgs) if anns: r = anns[0] else: r = None assert r == res, repr(anns) if __name__ == "__main__": _test()