Contents of /trunk/data/watch/update-sf-list

#!/bin/sh

####################
#    Copyright (C) 2009 by Raphael Geissert <atomo64@gmail.com>
#
#    This file is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 2 of the License, or
#    (at your option) any later version.
#
#    This file is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this file.  If not, see <http://www.gnu.org/licenses/>.
####################

set -eu

scratch=/srv/scratch/qa.debian.org/sf-list
export scratch

mkdir -p "$scratch"

ret=0
rsync -Pvan --log-file=/dev/null --list-only ftp.heanet.ie::sourceforge > "$scratch/rsync-stdout.txt" || ret=$?
if [ $ret -ne 0 -a $ret -ne 23 -a $ret -ne 24 ] ; then
        exit $ret
fi

sed -rn 's/^-.+[0-9]{4}\/[0-9]{2}\/[0-9]{2}[[:space:]]+[0-9]{2}:[0-9]{2}:[0-9]{2}[[:space:]]+(.+)$/\1/;T;p' < "$scratch/rsync-stdout.txt" > "$scratch/sf-list.new"

unlink "$scratch/rsync-stdout.txt"

perl -w <<'PCODE'

use strict;
use BerkeleyDB;

my %db;

tie %db, 'BerkeleyDB::Btree',
            -Filename   => "sf-list.db-new",
            -Flags      => DB_CREATE | DB_TRUNCATE,
            -Mode       => 0664
        or die("Failed to open sf-list.db-new: $! $BerkeleyDB::Error\n");

sub gen_html;

open(LIST, '<', "$ENV{scratch}/sf-list.new")
        or die("Failed to open sf-list.new: $!\n");

my $cur_project = undef;
my @files;

while(<LIST>) {
    next unless(m,^\w/.{2}/([^/]+)/(?:([^/]+)|([^/]+)/(?:[^/]+/)*(.+))?$,);
    my ($project, $file1, $mdir, $file2) = ($1, $2, $3, $4);
    my $file = $file1 || $file2;

    $mdir = $mdir || '';

    next if ($mdir eq 'OldFiles');

    $cur_project = $project unless (defined($cur_project));

    if ($project ne $cur_project) {
        $db{$cur_project} = gen_html;
        $cur_project = $project;
    }

    push @files, $file;
}

if (@files) {
    $db{$cur_project} = gen_html;
}

close(LIST);
untie %db;

sub gen_html {
    my $html = '';
    while (my $f = pop @files) {
        $f =~ s/&/&amp;/g;
        $f =~ s/</&lt;/g;
        $f =~ s/>/&gt;/g;
        $html .= "<a href='$f'>$f</a><br/>\n";
    }
    return $html;
}

PCODE

gzip -n "$scratch/sf-list.new"
mv sf-list.db-new sf-list.db
mv "$scratch/sf-list.new.gz" "$scratch/sf-list.gz"
1	geissert	2266	#!/bin/sh
2
3			####################
4			# Copyright (C) 2009 by Raphael Geissert <atomo64@gmail.com>
5			#
6			# This file is free software: you can redistribute it and/or modify
7			# it under the terms of the GNU General Public License as published by
8			# the Free Software Foundation, either version 2 of the License, or
9			# (at your option) any later version.
10			#
11			# This file is distributed in the hope that it will be useful,
12			# but WITHOUT ANY WARRANTY; without even the implied warranty of
13			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14			# GNU General Public License for more details.
15			#
16			# You should have received a copy of the GNU General Public License
17			# along with this file. If not, see <http://www.gnu.org/licenses/>.
18			####################
19
20	geissert	2741	set -eu
21	geissert	2266
22	geissert	2740	scratch=/srv/scratch/qa.debian.org/sf-list
23			export scratch
24
25			mkdir -p "$scratch"
26
27	pabs	2726	ret=0
28	geissert	2740	rsync -Pvan --log-file=/dev/null --list-only ftp.heanet.ie::sourceforge > "$scratch/rsync-stdout.txt" \|\| ret=$?
29	pabs	2726	if [ $ret -ne 0 -a $ret -ne 23 -a $ret -ne 24 ] ; then
30			exit $ret
31			fi
32
33	geissert	2740	sed -rn 's/^-.+[0-9]{4}\/[0-9]{2}\/[0-9]{2}[[:space:]]+[0-9]{2}:[0-9]{2}:[0-9]{2}[[:space:]]+(.+)$/\1/;T;p' < "$scratch/rsync-stdout.txt" > "$scratch/sf-list.new"
34	geissert	2266
35	geissert	2740	unlink "$scratch/rsync-stdout.txt"
36
37	geissert	2266	perl -w <<'PCODE'
38
39			use strict;
40			use BerkeleyDB;
41
42			my %db;
43
44			tie %db, 'BerkeleyDB::Btree',
45			-Filename => "sf-list.db-new",
46			-Flags => DB_CREATE \| DB_TRUNCATE,
47	geissert	2268	-Mode => 0664
48	geissert	2266	or die("Failed to open sf-list.db-new: $! $BerkeleyDB::Error\n");
49
50			sub gen_html;
51
52	geissert	2740	open(LIST, '<', "$ENV{scratch}/sf-list.new")
53	geissert	2266	or die("Failed to open sf-list.new: $!\n");
54
55			my $cur_project = undef;
56			my @files;
57
58			while(<LIST>) {
59	pabs	2728	next unless(m,^\w/.{2}/([^/]+)/(?:([^/]+)\|([^/]+)/(?:[^/]+/)*(.+))?$,);
60	geissert	2352	my ($project, $file1, $mdir, $file2) = ($1, $2, $3, $4);
61			my $file = $file1 \|\| $file2;
62	geissert	2266
63	geissert	2352	$mdir = $mdir \|\| '';
64
65	geissert	2287	next if ($mdir eq 'OldFiles');
66
67	geissert	2266	$cur_project = $project unless (defined($cur_project));
68
69			if ($project ne $cur_project) {
70			$db{$cur_project} = gen_html;
71			$cur_project = $project;
72			}
73
74			push @files, $file;
75			}
76
77			if (@files) {
78			$db{$cur_project} = gen_html;
79			}
80
81			close(LIST);
82			untie %db;
83
84			sub gen_html {
85			my $html = '';
86			while (my $f = pop @files) {
87	geissert	2727	$f =~ s/&/&/g;
88			$f =~ s/</</g;
89			$f =~ s/>/>/g;
90	geissert	2266	$html .= "<a href='$f'>$f</a><br/>\n";
91			}
92			return $html;
93			}
94
95			PCODE
96
97	geissert	2740	gzip -n "$scratch/sf-list.new"
98	geissert	2266	mv sf-list.db-new sf-list.db
99	geissert	2740	mv "$scratch/sf-list.new.gz" "$scratch/sf-list.gz"