path: root/scripts/hint

#! /usr/bin/python3
## encoding: utf-8
## vim:set et ts=4 sw=4:
#
# Copyright (c) 2007, 2008 Adeodato Simó (dato@net.com.org.es)
# Licensed under the terms of the MIT license.

# TODO: teach the script about the _tpu suffix for packages?

"""Tool to work with hints and hint files for the Debian Release team.

Accepts a subcommand as the first argument that specifies what the script
should do. Available subcommands are:

  - grep
  - clean
  - check
  - report
  - conflicts
  - cat
  - edit
  - excuses

Run `%prog help <command>` for help on each command.

Composing hints can be done from the command line, by using the name of a hint
as the subcommand (see `%prog help compose`), or by passing hints to compose
through stdin, eg. from your editor (see `%prog help filter`).
"""

_help_topics = {
        'compose': """\
Creating hints from the command line
------------------------------------

  It will output a line suitable for inclusion in a hint file. Version numbers
  will be grabbed from the appropriate distribution depending on the type of
  the hint.
""",
        'filter': """\
Creating hints from your editor (using "hint" as a filter)
----------------------------------------------------------

  TODO: make more robust.

  The script can be run with no arguments, and then it will act as a filter,
  reading hints from standard input, and printing them in standard output with
  added or updated version numbers.

  To make use of this in eg. vim, write a line like "remove foo", go to normal
  mode and, while over the line, type "!!hint<Enter>".
""",
        'check': """\
Checking your hint file for done hints, or hints with out of date versions
--------------------------------------------------------------------------

  TODO

  It will print in standard output a report of the hints that are already done
  or out of date. "who" defaults to $LOGNAME.
""",
        'clean': """\
Automatically cleaning of your hint file
----------------------------------------

  To move done hints after the "finished" line. All preceding comments in the
  same paragraph as the hint will be moved with it. If one of the comments is
  in the format "yyyy-mm-dd" or "yyyymmdd", a "; done yyyymmdd" stamp will be
  appended to it.

  By default, only full paragraphs are moved; that is, in order for a paragraph
  to be moved after the "finished" line, all its hints must be done. If you wish
  to take hints out individually, you can enable it for a given paragraph by
  including the string ":<split>:" somewhere in one of its comments.

  A backup of the hint file is made in /tmp/hint.$who.bak.

  If you cron this, a good idea is to put it right before dinstall, to avoid
  possible undoing of your hints if a non-incremental rerun of britney is
  performed.
""",
        'report': """\
Obtaining a report of the execution of one's hints
--------------------------------------------------

  This will print to stdout a summary of the execution of somebody's easy,
  hint, and force-hint hints, by looking at update_output.txt and printing the
  first and last few lines regarding each hint.
""",
        'conflicts': """\
Warnings about overriden hints
------------------------------

  TODO

  Checks whether two or more hint files contain hints for the same package. By
  default, it will warn to standard output. If the "--mail" option is given,
  mail will be sent to the involved parties instead.
""",
        'grep': """\
Grepping through all hint files
-------------------------------

  Greps through the contents of all hint files, but only up to their "finished"
  line (unless --include-old is specified). pkgname can be a plain string or a
  Python regular expression (if delimited by slashes). If you specify --type,
  pkgname becomes optional.
""",
        'cat': """\
Printing the content of one's hint file
--------------------------------------

  Prints the content of a user's hint file, but only up to their "finished" line.
""",
        'edit': """\
Editing one's hint file
--------------------------------------

  Launches an editor to edit your hint file.
""",
        'excuses': """\
Obtaining a report based on excuses file
----------------------------------------

  Reads excuses file and outputs excuses for packages mentioned in pending
  hints.

  It reads data from /srv/release.debian.org/www/britney/update_excuses.html
""",
}

import os
import re
import sys
import time
import shutil
import optparse
import string
import subprocess

from configparser import ConfigParser

config_file = os.path.dirname(os.path.realpath(__file__)) + '/../config.ini'

config = ConfigParser()
if not config.read([ config_file ]):
    config = None

# add the base path to fetch our central libraries
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + '/../lib/')
from debrelease.projectb import ProjectB
projectb = ProjectB()

if config is None:
    release_architectures = None
else:
    release_architectures = \
        set(config.get('testing', 'release_architectures').split(' '))

import apt_pkg
apt_pkg.init()

##

HINTDIR = '/srv/release.debian.org/britney/hints'
UPDATE_OUTPUT = '/srv/release.debian.org/www/britney/update_output.txt'
UPDATE_EXCUSES = '/srv/release.debian.org/www/britney/update_excuses.html'

HINTS = [ 'age-days', 'approve', 'block', 'block-all', 'easy',
          'force', 'force-hint', 'hint', 'remove', 'unblock', 'urgent',
          'block-udeb', 'unblock-udeb', 'ignore-rc-bugs', 'ignore-piuparts' ]
ALIASES = {'rm': 'remove'}

RE_COMMENT = re.compile(r'^\s*#')
RE_BLANKLINE = re.compile(r'^\s*$')
RE_RE = re.compile(r'^/(.*)/$')

##

def expand_aliases(hint):
    if hint[0] in ALIASES:
        hint[0] = ALIASES[hint[0]]

    return hint


def main():
    options, args = parse_options()

    if not args:
        filter_stdin()
        return 0

    args = expand_aliases(args)

    if args[0] in HINTS:
        hint = Hint(*args)
        hint.update_versions()
        print(hint)

    else:
        actions = {
            'check': check_hintfile,
            'clean': clean_hintfile,
            'cat': cat_hintfile,
            'edit': edit_hintfile,
            'excuses': grep_excuses,
            'conflicts': check_conflicts,
            'grep': grep_hintfiles,
            'report': hint_report,
        }
        try:
            function = actions[args[0]]
        except KeyError:
            print('E: unknown action %r' % (args[0],), file=sys.stderr)
            return 1
        else:
            return function(options, args[1:])

##

def filter_stdin(outf=sys.stdout):
    """Reads hints from stdin, and prints them with added or updated version numbers."""
    for line in sys.stdin:
        hint = Hint(*expand_aliases(line.strip().split()))
        hint.update_versions()
        print(hint, file=outf)


def check_hintfile(options, args):
    print('Not implemented yet')


def cat_hintfile(options, args):
    """Print hints above the finished line."""
    who = get_user(args)
    path = os.path.join(HINTDIR, who)

    if not os.path.exists(path):
        print("E: %s doesn't exist!" % path)
    else:
        with open(path) as f:
            for line in f:
                if line.startswith('finished'):
                    break
                print(line, end='')


def edit_hintfile(options, args):
    """Spaws an editor to edit hint file."""
    who = os.environ['LOGNAME'] # Yes. I don't use get_user here.
    path = os.path.join(HINTDIR, who)

    if not os.path.exists(path):
        print("E: %s doesn't exist!" % path, file=sys.stderr)
        return 1
    return subprocess.call(['sensible-editor', path])


def grep_excuses(options, args):
    """Shows excuses for packages mentioned in pending hints."""
    who = get_user(args)
    path = os.path.join(HINTDIR, who)

    RE_EXCUSES = re.compile(r'\<li\>\<a id="(?P<pkg>[^"]+)".*\>.*\</a\> (.* to .*)')
    RE_LINK = re.compile(r'<a [^>]*>(?P<text>[^<]+)</a>')

    if not os.path.exists(path):
        print("E: %s doesn't exist!" % path)
    else:
        pkgs = []
        with open(path) as hints:
            for line in hints:
                if line.startswith('finished'):
                    break
                if RE_COMMENT.search(line) or RE_BLANKLINE.search(line):
                    continue
                hint, list = line.split(None, 1)
                list = list.split()
                if hint == 'age-days' or hint == 'ignore-rc-bugs':
                    # Skip days/bugs-list in hint
                    list.pop(0)
                if hint == 'block':
                    # Skip blocked packages
                    continue
                for pkg in list:
                    if pkg not in pkgs:
                        name = pkg.rsplit('/', 1)[0]
                        if hint == 'remove':
                            pkgs.append("-%s" % name)
                        elif hint == 'approve':
                            pkgs.append("%s_tpu" % name)
                        else:
                            pkgs.append(name)

        if pkgs:
            should_write = False
            should_indent = False
            with open(UPDATE_EXCUSES) as excuses:
                for line in excuses:
                    line = line.strip()
                    matches = RE_EXCUSES.match(line)
                    if matches and matches.group('pkg') in pkgs:
                        should_write = True
                    if line.startswith('</ul>'):
                        should_write = False
                        should_indent = False
                    if line.startswith('<ul>'):
                        should_indent = True
                        continue
                    if should_write:
                        if should_indent:
                            print("   ",)
                        print(RE_LINK.sub("\g<text>", line[4:]))


def clean_hintfile(options, args):
    """Move done hints below the finished line."""
    # TODO The loop of this function needs to be somehow factored out so that
    # check_hintfile() above can use it as well.
    who = get_user(args)
    path = os.path.join(HINTDIR, who)
    with open(path) as hints:
        lines = list(hints)

    new_lines = []
    done_lines = []
    current_paragraph_lines = []
    seen_finished = False

    for i, line in enumerate(lines): # NOTE: "i" used after the loop
        if RE_BLANKLINE.search(line):
            if seen_finished:
                new_lines.extend(current_paragraph_lines)
                new_lines.append(line)
                break
            elif current_paragraph_lines:
                p = HintfileParagraph(current_paragraph_lines)
                done, pending = p.lines_by_status()
                if done:
                    done_lines.extend(done)
                    done_lines.append('\n')
                if pending:
                    new_lines.extend(pending)
                    new_lines.append(line)
                current_paragraph_lines[:] = []
            else:
                new_lines.append(line)
        else:
            current_paragraph_lines.append(line)
            if line.startswith('finished'):
                seen_finished = True

    if len(done_lines) > 0:
        new_lines.extend(done_lines)
        new_lines.extend(lines[i+1:])

        shutil.copyfile(path, '/tmp/hint.%s.bak' % (who,))
        with open(path, 'w') as dst:
            dst.write(''.join(new_lines))

def check_conflicts(options, args):
    print('Not implemented yet')

def grep_hintfiles(options, args):
    """Greps through active hints for a package name or pattern."""
    retval = 1
    is_re = 0
    try:
        opattern = RE_RE.search(args[0])
        if opattern:
            pattern = re.compile(opattern.group(1))
            is_re = 1
        else:
            pattern = args[0]
    except IndexError:
        pattern = re.compile('.') # TODO only allow this with --type?

    if options.type:
        types = set(options.type)
    else:
        types = None

    for name, fileobj in yield_hintfiles():
        comments = []
        lineno = 0
        errors = 0
        for line in fileobj:
            lineno += 1
            line = line.rstrip('\r\n')
            if RE_COMMENT.search(line):
                comments.append(line)
                continue
            elif RE_BLANKLINE.search(line):
                comments = []
                continue
            elif line.startswith('finished'):
                if options.include_old:
                    continue
                else:
                    break

            try:
                hint = Hint(*line.split(), is_mutable=False)
            except UnknownHintError:
                print("W: Unknown or invalid hint in %s's hint file at line %d: %s" % (name, lineno, line),
                      file=sys.stderr)
                errors += 1
                if errors > 2:
                    print("E: Too many errors in %s's hint file - skipping the rest of this file" % (name,),
                          file=sys.stderr)
                    break
                continue

            if types and hint.name not in types:
                continue

            for pkg in hint.pkgs:
                print_this = 0
                if is_re and pattern.search(pkg):
                    print_this = 1
                elif not is_re and pkg.find(pattern) != -1:
                    print_this = 1
                if print_this:
                    if name is not None:
                        print('==> %s' % (name,))
                        name = None
                    for c in comments:
                        print('  ' + c)
                    print('  ' + line)
                    comments = []
                    retval = 0
                    break

    return retval

def hint_report(options, args):
    if get_user(args) == "all":
        for user, _ in yield_hintfiles():
            hint_report_user(options, [user])
    else:
        hint_report_user(options, args)


def hint_report_user(options, args):
    """Grep through update_output.txt, printing a summary of one's hints.

    This is implemented as a simple Mealy machine, where transitions are
    triggered by matching specific regular expressions, and actions consist in
    a simple print_line/no_print_line boolean.
    """
    # TODO: print a === separator between hints (requires non-boolean "action")
    who = get_user(args)
    re_start_own_hint = re.compile(r'^Trying (\S+) from %s:' % who)
    re_start_other_hint = re.compile(r'^Trying (\S+) (from (?!%s:)|at random)' % who)
    re_main_run = re.compile(r'^info: main run')
    re_finish_hint = re.compile(r'^ finish|^failed:|^ Version mismatch')
    re_default = re.compile('')

    mealy = {
            # state: regex, print line?, next state
            1: [
                (re_start_own_hint, True, 2),
                (re_main_run, False, 3),
                (re_default, False, 1),
            ], 2: [
                (re_start_other_hint, False, 1),
                (re_main_run, False, 3),
                (re_default, True, 2),
            ], 3: [
                (re_start_own_hint, True, 4),
                (re_default, False, 3),
            ], 4: [
                (re_finish_hint, True, 5),
                (re_default, False, 4),
            ], 5: [
                (re_start_own_hint, True, 4),
                (re_start_other_hint, False, 3),
                (re_default, True, 5),
            ],
    }

    state = 1

    with open(UPDATE_OUTPUT) as output:
        for line in open(UPDATE_OUTPUT):
            for regex, print_line, next_state in mealy[state]:
                if regex.search(line):
                    state = next_state
                    if print_line:
                        print(line.strip())
                    break

##

class Hint(object):
    """Base class for representing a hint.

    It also acts as a factory: Hint(hintame, *args) will return an object of
    the appropriate class (eg. UnstableToTestingHint).
    """

    def is_done(self):
        """Checks whether a hint is already done.

        To be considered as done, version numbers from the hint must exactly
        match those in the target distribution.
        """
        raise NotImplementedError(
                'Hint.is_done() must be provided by subclasses')

    def update_versions(self):
        """Updates versions to the latest available in the source distribution."""
        raise NotImplementedError(
                'Hint.update_versions() must be provided by subclasses')

    ##

    def update_versions_report(self):
        """Returns a string representing the changes made by update_versions().

        This is intended to be a human-readable summary of version numbers
        mismatches between the versions in the hint, and those in the source
        distribution.

        If version numbers in the hint are up to date, None is returned.
        """
        self.update_versions()

        if self.versionmap0 == self.versionmap1:
            return None
        else:
            raise NotImplementedError('Hint.update_versions_report: TODO')

    ##

    def __init__(self, hintname, *args, is_mutable=True):
        starts_with_letter = re.compile(r'(?i)^[a-z]')

        self.name = hintname
        self.is_mutable = is_mutable
        self.pkgs = []

        self.versionmap0 = {} # original versions on the hint
        self.versionmap1 = self.versionmap0 # cow-updated by update_versions()

        self.archmap = {}

        # set of packages prefixed with a dash (i.e., to be removed)
        self.remove_pkgs = set()

        self.input_pkgs = []
        self.source_map = {}

        pkglist =  [pkg.strip() for sublist in
                     [arg.split(',') for arg in args]
                    for pkg in sublist if len(pkg)]

        for pkg in pkglist:
            if '/' in pkg:
                pkg, ver = pkg.split('/', 1)
                if starts_with_letter.search(ver):
                    # Package versions start with digits, so assume
                    # this is an architecture.  This will break if
                    # an architecture name ever starts with a digit
                    # but that seems unlikely.
                    if '/' in ver:
                        arch, ver = ver.split('/', 1)
                    else:
                        arch = ver
                        ver = ''
                else:
                    arch = ''
            else:
                ver = ''
                arch = ''

            # Find source package. If we can't find a source package then
            # we pretend that we did as the non-existence is handled later
            src = pkg
            if src.startswith('-'):
                src = src[1:]
            try:
                suite = self.SOURCE_DISTRIBUTION
            except AttributeError:
                suite = 'unstable'
            if not projectb.is_source_pkg(suite, src) and self.is_mutable:
                tmpsrc = src
                src = projectb.get_source_pkg(suite, src)
                if src is None:
                    src = tmpsrc

            if pkg.startswith('-'):
                pkg = pkg[1:]
                self.remove_pkgs.add(src)

            if pkg not in self.input_pkgs:
                self.input_pkgs.append(pkg)
            if src not in self.pkgs:
                self.pkgs.append(src)
            self.source_map[pkg] = src
            self.versionmap0[src] = ver
            if len(arch) > 0:
                if src not in self.archmap:
                    self.archmap[src] = set()
                self.archmap[src].add(arch)

    def __str__(self):
        if not self.pkgs:
            return '# No packages left in hint.'

        pkgs_output = []
        words = [ self.name ]

        for input_pkg in self.input_pkgs:
            pkg = self.source_map[input_pkg]
            if pkg in pkgs_output:
                continue
            if pkg not in self.pkgs:
                continue
            pkgs_output.append(pkg)
            if pkg in self.remove_pkgs:
                pkg2 = '-' + pkg
            else:
                pkg2 = pkg
            if self.name in [ 'easy', 'hint', 'force-hint' ] and \
                    pkg in self.archmap:
                if 'any' in self.archmap[pkg]:
                    if release_architectures is None:
                        print(
                            'W: architecture list not available, '
                            'unable to expand "any" for package %s.'
                            % (pkg), file=sys.stderr)
                    else:
                        # We intentionally don't limit the search to the
                        # binaries available for the version we want to
                        # migrate as the hint will only be viable once
                        # all the architectures affected have the same
                        # version
                        arches = projectb.get_architectures('unstable', pkg)
                        arches = release_architectures.intersection(arches)
                        self.archmap[pkg] = self.archmap[pkg].union(arches)
                    self.archmap[pkg].remove('any')

                for arch in sorted(self.archmap[pkg]):
                    words.append(pkg2 + '/' + arch + '/' +
                            self.versionmap1[pkg])
            else:
                words.append(pkg2 + '/' + self.versionmap1[pkg])

        # If all of the packages were specified using the "any"
        # pseudo-architecture and we don't have a list of release
        # architectures, the result will simply be "<hintname> ";
        # make it blank instead, as that's less confusing.
        if len(words) == 1:
            words = [ ]
        return ' '.join(words)

    def __new__(cls, hintname, *args, is_mutable=True):
        if hintname in \
                [ 'easy', 'hint', 'force', 'force-hint', 'unblock', 'urgent', 'unblock-udeb', 'ignore-piuparts', 'remark' ]:
            subclass = UnstableToTestingHint

        elif hintname == 'remove':
            subclass = RemovalHint

        elif hintname == 'approve':
            subclass = TpuToTestingHint

        elif hintname == 'age-days':
            subclass = AgeDaysHint

        elif hintname == 'ignore-rc-bugs':
            subclass = IgnoreRCBugsHint

        elif hintname == 'block':
            subclass = BlockHint

        elif hintname == 'block-udeb':
            subclass = BlockHint

        elif hintname == 'block-all':
            subclass = BlockHint

        else:
            raise UnknownHintError(hintname)

        return object.__new__(subclass)


class UnversionedHint(Hint):
    """A hint that doesn't know about versions."""

    def update_versions(self):
        pass

    def __str__(self):
        if self.pkgs:
            pkgs = []
            for pkg in self.pkgs:
                if pkg in self.remove_pkgs:
                    pkgs.append('-%s' % (pkg))
                else:
                    pkgs.append(pkg)
            return '%s %s' % (self.name, ' '.join(pkgs))
        else:
            return self.name


class MigrationHint(Hint):
    """A hint whose packages are meant to move from one distribution to another."""

    def __init__(self, *args, is_mutable=True):
        super().__init__(*args, is_mutable=is_mutable)
        self._versions = None

    @property
    def versions(self):
        if self._versions is not None:
            return self._versions

        self._versions = {
            'source': get_version_numbers(self.SOURCE_DISTRIBUTION, self.pkgs),
            'target': get_version_numbers(self.TARGET_DISTRIBUTION, self.pkgs),
        }

        for pkg, ver in self._versions['source'].items():
            if ver is None and pkg not in self.remove_pkgs and self.is_mutable:
                print(
                    'W: package %s not in %s, skipping.'
                    % (pkg, self.SOURCE_DISTRIBUTION), file=sys.stderr)
                self.pkgs.remove(pkg)

        return self._versions

    def is_done(self):
        # We require that packages in target have a greater or equal version
        # than the version in the hint, except for -removeme packages, which we
        # require not to be present.
        for pkg in self.pkgs:
            if pkg in self.remove_pkgs:
                if self.versions['target'][pkg] is not None:
                    return False
            elif self.versions['target'][pkg] is None:
                return False
            # Check the source package versions. If source > target then the
            # migration is incomplete.
            elif apt_pkg.version_compare(
                    self.versions['target'][pkg], self.versionmap0[pkg]) < 0:
                return False
            # Source package versions are the same; check if this is a binary
            # migration.
            elif pkg in self.archmap:
                for arch in self.archmap[pkg]:
                    # Check the status of each binary package produced by the
                    # version of the source package in the "source" suite.
                    # We explicitly do not iterate the list of binary packages
                    # from the "target" suite, as there may be packages in
                    # that suite which are cruft.
                    for binpkg in projectb.get_binaries(self.SOURCE_DISTRIBUTION, pkg):
                        binver_s = projectb.get_binary_version(
                                      self.SOURCE_DISTRIBUTION, binpkg, arch, True)
                        binver_t = projectb.get_binary_version(
                                      self.TARGET_DISTRIBUTION, binpkg, arch, True)
                        # The binary package doesn't exist on this architecture,
                        # so skip it.
                        if binver_s is None and binver_t is None:
                            continue
                        if apt_pkg.version_compare(binver_t, binver_s) < 0:
                             return False

        return True

    def update_versions(self):
        # I'm sorry for this gross hack, but versions() may modify self.pkgs,
        # and we want to iterate over that modified copy...
        self.versions

        for pkg in sorted(set(self.pkgs) - self.remove_pkgs):
            if self.versions['target'][pkg] == self.versions['source'][pkg] and \
                    pkg not in self.archmap:
                print('W: package %s has the same version '
                    'in %s and %s, skipping.' % (pkg,
                        self.TARGET_DISTRIBUTION, self.SOURCE_DISTRIBUTION),
                        file=sys.stderr)
                self.pkgs.remove(pkg)

        for pkg in self.remove_pkgs:
            targetver = self.versions['target'][pkg]
            if targetver is None:
                print('W: package %s is marked for removal '
                    'but is not present in %s, skipping.' % (pkg,
                        self.TARGET_DISTRIBUTION),
                        file=sys.stderr)
                self.pkgs.remove(pkg)
            else:
                self.versions['source'][pkg] = targetver

        self.versionmap1 = dict((p, self.versions['source'][p]) for p in self.pkgs)


class UnstableToTestingHint(MigrationHint):
    SOURCE_DISTRIBUTION = 'unstable'
    TARGET_DISTRIBUTION = 'testing'


class TpuToTestingHint(MigrationHint):
    SOURCE_DISTRIBUTION = 'testing-proposed-updates'
    TARGET_DISTRIBUTION = 'testing'


class AgeDaysHint(UnstableToTestingHint):
    def __init__(self, hintname, days, *args, is_mutable=True):
        super().__init__('%s %s' % (hintname, days), *args,
          is_mutable=is_mutable)

class IgnoreRCBugsHint(UnstableToTestingHint):
    def __init__(self, hintname, bugs, *args, is_mutable=True):
        super().__init__('%s %s' % (hintname, bugs), *args,
          is_mutable=is_mutable)

class RemovalHint(Hint):
    SOURCE_DISTRIBUTION = 'testing'

    def is_done(self):
        testing_versions = get_version_numbers('testing', self.pkgs)

        for pkg, ver in testing_versions.items():
            if ver is not None:
                return False

        return True

    def update_versions(self):
        testing_versions = get_version_numbers('testing', self.pkgs)

        for pkg in self.pkgs[:]:
            if testing_versions[pkg] is None:
                print(
                    'W: package %s not in testing, skipping.' % (pkg,),
                    file=sys.stderr)
                self.pkgs.remove(pkg)

        self.versionmap1 = dict((p, testing_versions[p]) for p in self.pkgs)


class BlockHint(UnversionedHint):
    def is_done(self):
        return False


class UnknownHintError(Exception):
    pass

##

class HintfileParagraph(object):
    """A representation of a paragraph in a hint file.

    The basic operation on a paragraph used to be querying if it's done.
    However, by popular demand, paragraphs must be able to be split between
    "done" and "pending" parts. For this, we divide paragraphs in "groups",
    which are parts that can be considered done or not as a whole.

    For :<split>: paragraphs, each hint has its own group. In paragraphs
    without :<split>:, there is only one group. Instead of providing a method
    to query if a paragraph is done, we just provide a method to split between
    done and pending parts.
    """

    # A regex to match a date; '?' inside the group intentionally
    RE_DATE = re.compile(r'\b(?P<date>\d{4}(?P<sep>\S?)\d{2}\2\d{2})\b')

    # Enables moving single hints around
    RE_SPLIT_OUT = re.compile(r':<split>:')

    def __init__(self, lines):
        self._groups = [[]]
        seen_hint = False
        split_out = False

        for line in lines:
            line = line.rstrip('\r\n')
            if RE_COMMENT.search(line):
                found_split_line_now = False
                if self.RE_SPLIT_OUT.search(line):
                    split_out = True
                    found_split_line_now = True
                if seen_hint:
                    seen_hint = False
                    if split_out:
                        self._groups.append([])
                self._groups[-1].append(line)
                if found_split_line_now:
                    # We force a new group after :<split>:, so that the
                    # :<split>: line is never moved below the "finished" line.
                    self._groups.append([])
            else:
                if seen_hint and split_out:
                    self._groups.append([])
                hint = Hint(*line.strip().split(), is_mutable=False)
                self._groups[-1].append(hint)
                seen_hint = True

    def lines_by_status(self):
        """Return a pair (done_lines, pending_lines).

        The done_lines list contains groups that are completely done, and the
        pending_lines list contains groups where at least one hint is not done.
        """
        done_lines = []
        pending_lines = []

        for group in self._groups:
            hints = [ x for x in group if isinstance(x, Hint) ]
            if not hints:
                # comment-only groups are never done
                pending_lines.extend(group)
            else:
                for hint in hints:
                    if not hint.is_done():
                        pending_lines.extend(group)
                        break
                else:
                    done_lines.extend(group)

        # append the "done" timestamp
        for i, line in enumerate(done_lines):
            if not isinstance(line, Hint):
                m = self.RE_DATE.search(line)
                if m:
                    groups = m.groupdict()
                    donestamp = time.strftime(
                            '; done %%Y%(sep)s%%m%(sep)s%%d' % groups)
                    new_line = self.RE_DATE.sub(groups['date'] + donestamp, line)
                    done_lines[i] = new_line
                    break
        else:
            if done_lines:
                # always insert a timestamp
                done_lines.insert(0, '# done ' + time.strftime('%Y-%m-%d'))

        return ([ str(x) + '\n' for x in done_lines ],
                [ str(x) + '\n' for x in pending_lines ])

##

def get_version_numbers(suite, srcpkgs):
    """Obtain version numbers for source packages in a given suite.

    :param suite: The name of the suite to look in.
    :param srcpkgs: A list of source package names.
    :return: A mapping { srcpkg1: version1, ... }.

    If a package is not present in the suite, None is returned as its version
    number.
    """
    return dict((pkg, projectb.get_version(suite, pkg, False, True)) for pkg in srcpkgs)

def yield_hintfiles():
    """Yields pairs (basename, fileobj) for all hint files.

    >>> for name, f in yield_hintfiles():
    >>>     print '==> %s' % (name,)
    >>>     print f.readlines()
    """
    hintfiles = set(os.listdir(HINTDIR))
    valid_name = re.compile(r'^[a-z0-9-]+$')

    for fname in hintfiles:
        path = os.path.join(HINTDIR, fname)
        if valid_name.search(fname) and os.path.isfile(path):
            with open(path) as f:
                yield fname, f

def get_user(args):
    return args and args[0] or os.environ['LOGNAME']

##

class MyFormatter(optparse.IndentedHelpFormatter):
    """Formatter that does not touch the description."""
    def format_description(self, description):
        return description

def parse_options():
    """Parse options for the program, and provide --help.

    :return: A pair (options, args).
    """
    p = optparse.OptionParser(usage='%prog [ <hintname> | <command> | help ]',
            description=__doc__, formatter=MyFormatter())

    p.disable_interspersed_args()
    options, args = p.parse_args()

    try:
        action = args[0]
    except IndexError:
        c = []
        action = 'filter'
    else:
        c = [ args[0] ]
        if action in HINTS:
            action = 'compose'

    if action == 'help':
        args.pop(0)
        try:
            if args[0] in _help_topics:
                action = args[0]
                args.append('--help')
            else:
                p.error('unknown help topic %s' % args[0])
        except IndexError:
            p.print_help()
            p.exit()

    p = get_subcommand_parser(action)
    options, args = p.parse_args(args[1:])

    # XXX We are not returning global options, should there be any
    return options, c + args

def get_subcommand_parser(command):
    kwargs = { 'description': _help_topics.get(command, ''),
               'formatter': MyFormatter() }
    add_options = None

    if command == 'compose':
        kwargs['usage'] = '%prog <hintname> srcpkg1 [ srcpkg2 -srcpkg3 ... ]'

        def add_options(p):
            p.disable_interspersed_args()

    elif command == 'filter':
        kwargs['usage'] = '%prog'

    elif command in ['check', 'clean', 'report']:
        kwargs['usage'] = '%%prog %s [ who ]' % command

    elif command == 'conflicts':
        kwargs['usage'] = '%prog conflicts'

    elif command == 'grep':
        kwargs['usage'] = '%prog grep [options] pkgname'

        def add_options(p):
            p.add_option('--include-old', action='store_true',
                    help='include hints after the "finished" line')
            p.add_option('--type', action='append',
                    help='only include hints of the specified types')

    p = optparse.OptionParser(**kwargs)

    if add_options is not None:
        add_options(p)

    return p

##

if __name__ == '__main__':
    sys.exit(main())