Contents of /trunk/data/watch/update-sf-list

#!/bin/sh

####################
#    Copyright (C) 2009 by Raphael Geissert <atomo64@gmail.com>
#
#    This file is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 2 of the License, or
#    (at your option) any later version.
#
#    This file is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this file.  If not, see <http://www.gnu.org/licenses/>.
####################

set -e

ret=0
rsync -Pvan --log-file=/dev/null --list-only ftp.heanet.ie::sourceforge > rsync-stdout.txt || ret=$?
if [ $ret -ne 0 -a $ret -ne 23 -a $ret -ne 24 ] ; then
        exit $ret
fi

sed -rn 's/^-.+[0-9]{4}\/[0-9]{2}\/[0-9]{2}[[:space:]]+[0-9]{2}:[0-9]{2}:[0-9]{2}[[:space:]]+(.+)$/\1/;T;p' < rsync-stdout.txt > sf-list.new

perl -w <<'PCODE'

use strict;
use BerkeleyDB;

my %db;

tie %db, 'BerkeleyDB::Btree',
            -Filename   => "sf-list.db-new",
            -Flags      => DB_CREATE | DB_TRUNCATE,
            -Mode       => 0664
        or die("Failed to open sf-list.db-new: $! $BerkeleyDB::Error\n");

sub gen_html;

open(LIST, '< sf-list.new')
        or die("Failed to open sf-list.new: $!\n");

my $cur_project = undef;
my @files;

while(<LIST>) {
    next unless(m,^\w/project/.{2}/([^/]+)/(?:([^/]+)|([^/]+)/(?:[^/]+/)*(.+))?$,);
    my ($project, $file1, $mdir, $file2) = ($1, $2, $3, $4);
    my $file = $file1 || $file2;

    $mdir = $mdir || '';

    next if ($mdir eq 'OldFiles');

    $cur_project = $project unless (defined($cur_project));

    if ($project ne $cur_project) {
        $db{$cur_project} = gen_html;
        $cur_project = $project;
    }

    push @files, $file;
}

if (@files) {
    $db{$cur_project} = gen_html;
}

close(LIST);
untie %db;

sub gen_html {
    my $html = '';
    while (my $f = pop @files) {
        $f =~ s/&/&amp;/g;
        $f =~ s/</&lt;/g;
        $f =~ s/>/&gt;/g;
        $html .= "<a href='$f'>$f</a><br/>\n";
    }
    return $html;
}

PCODE

gzip -n sf-list.new
mv sf-list.db-new sf-list.db
mv sf-list.new.gz sf-list.gz
1	geissert	2266	#!/bin/sh
2
3			####################
4			# Copyright (C) 2009 by Raphael Geissert <atomo64@gmail.com>
5			#
6			# This file is free software: you can redistribute it and/or modify
7			# it under the terms of the GNU General Public License as published by
8			# the Free Software Foundation, either version 2 of the License, or
9			# (at your option) any later version.
10			#
11			# This file is distributed in the hope that it will be useful,
12			# but WITHOUT ANY WARRANTY; without even the implied warranty of
13			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14			# GNU General Public License for more details.
15			#
16			# You should have received a copy of the GNU General Public License
17			# along with this file. If not, see <http://www.gnu.org/licenses/>.
18			####################
19
20			set -e
21
22	pabs	2726	ret=0
23			rsync -Pvan --log-file=/dev/null --list-only ftp.heanet.ie::sourceforge > rsync-stdout.txt \|\| ret=$?
24			if [ $ret -ne 0 -a $ret -ne 23 -a $ret -ne 24 ] ; then
25			exit $ret
26			fi
27
28	pabs	2616	sed -rn 's/^-.+[0-9]{4}\/[0-9]{2}\/[0-9]{2}[[:space:]]+[0-9]{2}:[0-9]{2}:[0-9]{2}[[:space:]]+(.+)$/\1/;T;p' < rsync-stdout.txt > sf-list.new
29	geissert	2266
30			perl -w <<'PCODE'
31
32			use strict;
33			use BerkeleyDB;
34
35			my %db;
36
37			tie %db, 'BerkeleyDB::Btree',
38			-Filename => "sf-list.db-new",
39			-Flags => DB_CREATE \| DB_TRUNCATE,
40	geissert	2268	-Mode => 0664
41	geissert	2266	or die("Failed to open sf-list.db-new: $! $BerkeleyDB::Error\n");
42
43			sub gen_html;
44
45			open(LIST, '< sf-list.new')
46			or die("Failed to open sf-list.new: $!\n");
47
48			my $cur_project = undef;
49			my @files;
50
51			while(<LIST>) {
52	geissert	2352	next unless(m,^\w/project/.{2}/([^/]+)/(?:([^/]+)\|([^/]+)/(?:[^/]+/)*(.+))?$,);
53			my ($project, $file1, $mdir, $file2) = ($1, $2, $3, $4);
54			my $file = $file1 \|\| $file2;
55	geissert	2266
56	geissert	2352	$mdir = $mdir \|\| '';
57
58	geissert	2287	next if ($mdir eq 'OldFiles');
59
60	geissert	2266	$cur_project = $project unless (defined($cur_project));
61
62			if ($project ne $cur_project) {
63			$db{$cur_project} = gen_html;
64			$cur_project = $project;
65			}
66
67			push @files, $file;
68			}
69
70			if (@files) {
71			$db{$cur_project} = gen_html;
72			}
73
74			close(LIST);
75			untie %db;
76
77			sub gen_html {
78			my $html = '';
79			while (my $f = pop @files) {
80	geissert	2727	$f =~ s/&/&/g;
81			$f =~ s/</</g;
82			$f =~ s/>/>/g;
83	geissert	2266	$html .= "<a href='$f'>$f</a><br/>\n";
84			}
85			return $html;
86			}
87
88			PCODE
89
90			gzip -n sf-list.new
91			mv sf-list.db-new sf-list.db
92			mv sf-list.new.gz sf-list.gz