udd/udd/bugs_gatherer.pl

#!/usr/bin/perl -w
# Last-Modified: <Mon Aug 18 14:29:47 2008>

use strict;
use warnings;

use FindBin '$Bin';

# We need our own copy of Debbugs::Status for now
use lib $Bin, qw{/org/udd.debian.net/mirrors/bugs.debian.org/perl}, qw{/org/udd.debian.org/bugs/bugs.debian.org/perl};

use DBI;
use DBI qw{:sql_types};
use YAML::Syck;
use Time::Local;

use Debbugs::Bugs qw{get_bugs};
use Debbugs::Status qw{read_bug get_bug_status bug_presence};
use Debbugs::Packages qw{getpkgsrc};
use Debbugs::Config qw{:globals %config};
use Debbugs::User;
use Mail::Address;
#use Debbugs::User qw{read_usertags};

$YAML::Syck::ImplicitTyping = 1;

#Used for measuring time
our $t;
our $timing = 0;
our @archs = grep {  !/(^m68k$|^kfreebsd|^hurd)/ } @{$config{default_architectures}};

# Return the list of usernames
sub get_bugs_users {
        my $topdir = "$gSpoolDir/user";
        my @ret = ();
        # see Debbugs::User::filefromemail for why 0...6
        for(my $i = 0; $i < 7; $i++) {
                my $dir = "$topdir/$i";
                opendir DIR, $dir or die "Can't open dir $dir: $!";
                # Replace all occurences of %dd with the corresponding
                # character represented by dd, where dd is a hexadecimal
                # number
                push @ret, map { s/%(..)/chr(hex($1))/ge; $_ } readdir DIR;
        }
        return @ret;
}

sub parse_time {
        if(shift =~ /(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)/) {
                return ($1, $2, $3, $4, $5, $6);
        }
        return undef;
}


sub get_db_max_last_modified {
        my $dbh = shift or die "Argument required";
        my $sth = $dbh->prepare("SELECT MAX (last_modified) FROM bugs");
        $sth->execute() or die $!;
        my $date = $sth->fetchrow_array();
        if(defined $date) {
                my ($year, $month, $day, $hour, $minute, $second) = parse_time($date);
                return timelocal($second, $minute, $hour, $day, $month-1, $year);
        } else {
                return 0;
        }
}

sub get_mtime {
        return ((stat(shift))[9]);
}

sub get_modified_bugs {
        my $prune_stamp = shift;
        die "Argument required" unless defined $prune_stamp;
        my $top_dir = $gSpoolDir;
        my @result = ();
        foreach my $sub (qw(archive db-h)) {
                my $spool = "$top_dir/$sub";
                foreach my $subsub (glob "$spool/*") {
                        if( -d $subsub and get_mtime($subsub) > $prune_stamp ) {
                                push @result, 
                                        map { s{.*/(.*)\.log}{$1}; $_ } 
                                                grep { get_mtime("$_") > $prune_stamp }
                                                        glob "$subsub/*.log";
                        }
                }
        }
        return \@result;
}

sub without_duplicates {
        my %h = ();
        return (grep { ($h{$_}++ == 0) || 0 } @_);
}

sub run_usertags {
        my ($config, $source, $dbh) = @_;
        my %src_config = %{$config->{$source}};
        my $table = $src_config{'usertags-table'} or die "usertags-table not specified for source $source";
        our $timing;
        our $t;


        $t = time();
        # Free usertags table
        $dbh->do("DELETE FROM $table") or die
                "Couldn't empty $table: $!";
        print "Deleting usertags: ",(time() - $t),"s\n" if $timing;
        $t = time();
        # read and insert user tags
        my @users = get_bugs_users();
        foreach my $user (@users) {
                #read_usertags(\%tags, $user);
                my $u = Debbugs::User->new($user);
                my %tags = %{$u->{tags}};
                $user = $dbh->quote($user);
                foreach my $tag (keys %tags) {
                        my $qtag = $dbh->quote($tag);
                        map { $dbh->do("INSERT INTO $table (email, tag, id) VALUES ($user, $qtag, $_)") or die $! } @{$tags{$tag}};
                }
        }
}

sub run {
        my ($config, $source, $dbh) = @_;

        our $t;
        our $timing;
        my %src_config = %{$config->{$source}};
        my $table = $src_config{table};
        my $archived_table = $src_config{'archived-table'};

        if (!$src_config{debug}) {
                run_usertags($config, $source, $dbh);
                print "Inserting usertags: ",(time() - $t),"s\n" if $timing;
                $t = time();
        }


        my %pkgsrc = %{getpkgsrc()};

        my @modified_bugs;

        if($src_config{archived}) {
                # some bugs (the unarchived ones) are in both list. exclude them.
                my %unarchived;
                foreach my $b (get_bugs()) {
                        $unarchived{$b} = 1;
                }
                foreach my $b (get_bugs(archive => 1)) {
                        push(@modified_bugs, $b) if not $unarchived{$b};
                }
        } else {
                @modified_bugs = get_bugs();
        }
        my @modified_bugs2;
        if ($src_config{debug}) {
                print "Running in debug mode with restricted bug list!!\n";
                foreach my $b (@modified_bugs) {
                        push(@modified_bugs2, $b) if ($b =~ /58$/);
                }
                @modified_bugs = @modified_bugs2;
        }

        print "Fetching list of ",scalar(@modified_bugs), " bugs to insert: ",(time() - $t),"s\n" if $timing;
        $t = time();

        foreach my $prefix ($table, $archived_table) {
                foreach my $postfix (qw{_packages _merged_with _found_in _fixed_in _tags _blocks _blockedby}, '') {
                        my $sth = $dbh->prepare("DELETE FROM $prefix$postfix WHERE id = \$1");
                        map {
                                $sth->execute($_) or die $!;
                        } @modified_bugs;
                }
        }
        print "Deleting bugs: ",(time() - $t),"s\n" if $timing;
        $t = time();

        my $location = $src_config{archived} ? 'archive' : 'db_h';
        $table = $src_config{archived} ? $archived_table : $table;
        # Read all bugs
        my $insert_bugs_handle = $dbh->prepare("INSERT INTO $table (id, package, source, arrival, status, severity, submitter, submitter_name, submitter_email, owner, owner_name, owner_email, done, done_name, done_email, title, forwarded, last_modified, affects_oldstable, affects_stable, affects_testing, affects_unstable, affects_experimental) VALUES (\$1, \$2, \$3, \$4::abstime, \$5, \$6, \$7, \$8, \$9, \$10, \$11, \$12, \$13, \$14, \$15, \$16, \$17, \$18::abstime, \$19, \$20, \$21, \$22, \$23)");
        my $insert_bugs_packages_handle = $dbh->prepare("INSERT INTO ${table}_packages (id, package, source) VALUES (\$1, \$2, \$3)");
        my $insert_bugs_found_handle = $dbh->prepare("INSERT INTO ${table}_found_in (id, version) VALUES (\$1, \$2)");
        my $insert_bugs_fixed_handle = $dbh->prepare("INSERT INTO ${table}_fixed_in (id, version) VALUES (\$1, \$2)");
        my $insert_bugs_merged_handle = $dbh->prepare("INSERT INTO ${table}_merged_with (id, merged_with) VALUES (\$1, \$2)");
        my $insert_bugs_tags_handle = $dbh->prepare("INSERT INTO ${table}_tags (id, tag) VALUES (\$1, \$2)");
        my $insert_bugs_blocks_handle = $dbh->prepare("INSERT INTO ${table}_blocks (id, blocked) VALUES (\$1, \$2)");
        my $insert_bugs_blockedby_handle = $dbh->prepare("INSERT INTO ${table}_blockedby (id, blocker) VALUES (\$1, \$2)");
        $insert_bugs_handle->bind_param(4, undef, SQL_INTEGER);
        $insert_bugs_handle->bind_param(18, undef, SQL_INTEGER);

        $t = time();
        foreach my $bug_nr (@modified_bugs) {
                # Fetch bug using Debbugs
                # Bugs which were once archived and have been unarchived again will appear in get_bugs(archive => 1).
                # However, those bugs are not to be found in location 'archive', so we detect them, and skip them
                my $bug_ref = read_bug(bug => $bug_nr, location => $location) or (print STDERR "Could not read file for bug $bug_nr; skipping\n" and next);
                # Yeah, great, why does get_bug_status not accept a location?
                my %bug = %{get_bug_status(bug => $bug_nr, status => $bug_ref)};
                
                # Convert data where necessary
                my @found_versions = @{$bug{found_versions}};
                my @fixed_versions = @{$bug{fixed_versions}};
                my @tags = split / /, $bug{keywords};

                # log_modified and date are not necessarily set. If they are not available, they
                # are assumed to be epoch (i.e. bug #4170)
                map {
                        if($bug{$_}) {
                                $bug{$_} = int($bug{$_});
                        } else {
                                $bug{$_} = 0;
                        }
                } qw{date log_modified};

                my $srcpkg;
                if ($bug{package} =~ /^src:(.*)/)
                {
                        $srcpkg = $1;
                } else {
                        $srcpkg = exists($pkgsrc{$bug{package}}) ? $pkgsrc{$bug{package}} : $bug{package};
                }

                # split emails
                my (@addr, $submitter_name, $submitter_email, $owner_name, $owner_email, $done_name, $done_email);
                if ($bug{originator}) {
                        @addr = Mail::Address->parse($bug{originator});
                        $submitter_name = $addr[0]->phrase;
                        $submitter_email = $addr[0]->address;
                } else {
                        $submitter_name = '';
                        $submitter_email = '';
                }

                if ($bug{owner}) {
                        @addr = Mail::Address->parse($bug{owner});
                        $owner_name = $addr[0]->phrase;
                        $owner_email = $addr[0]->address;
                } else {
                        $owner_name = '';
                        $owner_email = '';
                }

                if ($bug{done}) {
                        @addr = Mail::Address->parse($bug{done});
                        $done_name = $addr[0]->phrase;
                        $done_email = $addr[0]->address;
                } else {
                        $done_name = '';
                        $done_email = '';
                }

                #Calculate bug presence in distributions
                my (present_in_oldstable, $present_in_stable, $present_in_testing, $present_in_unstable, $present_in_experimental);
                if($src_config{archived}) {
                        $present_in_oldstable = $present_in_stable = $present_in_testing = $present_in_unstable = $present_in_experimental = 'FALSE';
                } else {
                        $present_in_oldstable =
                                bug_presence(bug => $bug_nr, status => \%bug,
                                                         dist => 'oldstable',
                                                         arch => \@archs);
                        $present_in_stable =
                                bug_presence(bug => $bug_nr, status => \%bug,
                                                         dist => 'stable',
                                                         arch => \@archs);
                        $present_in_testing =
                                bug_presence(bug => $bug_nr, status => \%bug,
                                                         dist => 'testing',
                                                         arch => \@archs);
                        $present_in_unstable =
                                bug_presence(bug => $bug_nr, status => \%bug,
                                                         dist => 'unstable',
                                                         arch => \@archs);
                        $present_in_experimental =
                                bug_presence(bug => $bug_nr, status => \%bug,
                                                         dist => 'experimental',
                                                         arch => \@archs);

                        if(!defined($present_in_oldstable) or !defined($present_in_stable) or !defined($present_in_unstable) or !defined($present_in_testing) or !defined($present_in_experimental)) {
                                print "NUMBER: $bug_nr\n";
                        }
                
                        if(defined($present_in_oldstable) and ($present_in_oldstable eq 'absent' or $present_in_oldstable eq 'fixed')) {
                                $present_in_oldstable = 'FALSE';
                        } else {
                                $present_in_oldstable = 'TRUE';
                        }
                        if(defined($present_in_stable) and ($present_in_stable eq 'absent' or $present_in_stable eq 'fixed')) {
                                $present_in_stable = 'FALSE';
                        } else {
                                $present_in_stable = 'TRUE';
                        }
                        if(defined($present_in_testing) and ($present_in_testing eq 'absent' or $present_in_testing eq 'fixed')) {
                                $present_in_testing = 'FALSE';
                        } else {
                                $present_in_testing = 'TRUE';
                        }
                        if(defined($present_in_unstable) and ($present_in_unstable eq 'absent' or $present_in_unstable eq 'fixed')) {
                                $present_in_unstable = 'FALSE';
                        } else {
                                $present_in_unstable = 'TRUE';
                        }
                        if(defined($present_in_experimental) and ($present_in_experimental eq 'absent' or $present_in_experimental eq 'fixed')) {
                                $present_in_experimental = 'FALSE';
                        } else {
                                $present_in_experimental = 'TRUE';
                        }
                }

                # Insert data into bugs table
                $insert_bugs_handle->execute($bug_nr, $bug{package}, $srcpkg, $bug{date}, $bug{pending},
                        $bug{severity}, $bug{originator}, $submitter_name, $submitter_email, $bug{owner},
                        $owner_name, $owner_email, $bug{done}, $done_name, $done_email, $bug{subject},
                        $bug{forwarded}, $bug{log_modified},
                        $present_in_oldstable, $present_in_stable, $present_in_testing, $present_in_unstable, $present_in_experimental) or die $!;

                my $src;
                foreach my $pkg (keys %{{ map { $_ => 1 } split(/\s*[, ]\s*/, $bug{package})}}) {
                        if ($pkg =~ /^src:(.*)/)
                        {
                                $src = $1;
                        } else {
                                $src = exists($pkgsrc{$pkg}) ? $pkgsrc{$pkg} : $pkg;
                        }
                        $insert_bugs_packages_handle->execute($bug_nr, $pkg, $src) or die $!;
                }

                # insert data into bug_fixed_in and bug_found_in tables
                foreach my $version (without_duplicates(@found_versions)) {
                        $insert_bugs_found_handle->execute($bug_nr, $version) or die $!;
                }
                foreach my $version (without_duplicates(@fixed_versions)) {
                        $insert_bugs_fixed_handle->execute($bug_nr, $version) or die $!;
                }
                foreach my $mergee (without_duplicates(split / /, $bug{mergedwith})) {
                        $insert_bugs_merged_handle->execute($bug_nr, $mergee) or die $!;
                }
                foreach my $blocked (without_duplicates(split / /, $bug{blocks})) {
                        $insert_bugs_blocks_handle->execute($bug_nr, $blocked) or die $!;
                }
                foreach my $blocker (without_duplicates(split / /, $bug{blockedby})) {
                        $insert_bugs_blockedby_handle->execute($bug_nr, $blocker) or die $!;
                }
                foreach my $tag (without_duplicates(@tags)) {
                        $insert_bugs_tags_handle->execute($bug_nr, $tag) or die $!;
                }
        }
        print "Inserting bugs: ",(time() - $t),"s\n" if $timing;

        foreach my $postfix (qw{_packages _merged_with _found_in _fixed_in _tags}, '') {
                my $sth = $dbh->prepare("ANALYZE $table$postfix");
                $sth->execute() or die $!;
        }

        my $sth = $dbh->prepare("ANALYZE ".$src_config{'usertags-table'});
        $sth->execute() or die $!;

        print "Analyzing bugs: ",(time() - $t),"s\n" if $timing;

        $dbh->commit();
        print "Committing bugs: ",(time() - $t),"s\n" if $timing;
}

sub main {
        if(@ARGV != 3) {
                print STDERR "Usage: $0 <config> <command> <source>\n";
                exit 1;
        }

        our $t = time();
        our $timing;

        my $config = LoadFile($ARGV[0]) or die "Could not load configuration: $!";
        my $command = $ARGV[1];
        my $source = $ARGV[2];

        my $dbname = $config->{general}->{dbname};
        my $dbport;
        if ($config->{general}->{dbport} ne '') {
          $dbport = ";port=".$config->{general}->{dbport};
        } else {
          $dbport = "";
        }
        # Connection to DB
        my $dbh = DBI->connect("dbi:Pg:dbname=$dbname".$dbport);
        # We want to commit the transaction as a hole at the end
        $dbh->{AutoCommit} = 0;
        $dbh->do('SET CONSTRAINTS ALL DEFERRED');

        if($command eq 'run') {
                run($config, $source, $dbh);
        } else {
                print STDERR "<command> has to be one of run, drop and setup\n";
                exit(1)
        }

}

main();
1	neronus-guest	1070	#!/usr/bin/perl -w
2	neronus-guest	1118	# Last-Modified: <Mon Aug 18 14:29:47 2008>
3	neronus-guest	919
4			use strict;
5			use warnings;
6
7	neronus-guest	946	use FindBin '$Bin';
8	neronus-guest	919
9	neronus-guest	946	# We need our own copy of Debbugs::Status for now
10	lucas	1466	use lib $Bin, qw{/org/udd.debian.net/mirrors/bugs.debian.org/perl}, qw{/org/udd.debian.org/bugs/bugs.debian.org/perl};
11	neronus-guest	946
12	neronus-guest	919	use DBI;
13	neronus-guest	1068	use DBI qw{:sql_types};
14	neronus-guest	919	use YAML::Syck;
15	neronus-guest	973	use Time::Local;
16	neronus-guest	919
17			use Debbugs::Bugs qw{get_bugs};
18	neronus-guest	942	use Debbugs::Status qw{read_bug get_bug_status bug_presence};
19	lucas	1588	use Debbugs::Packages qw{getpkgsrc};
20	lucas	1331	use Debbugs::Config qw{:globals %config};
21	neronus-guest	1053	use Debbugs::User;
22	lucas	1539	use Mail::Address;
23	neronus-guest	1053	#use Debbugs::User qw{read_usertags};
24	neronus-guest	919
25			$YAML::Syck::ImplicitTyping = 1;
26
27	neronus-guest	1068	#Used for measuring time
28	neronus-guest	1071	our $t;
29	neronus-guest	1118	our $timing = 0;
30	lucas	1331	our @archs = grep { !/(^m68k$\|^kfreebsd\|^hurd)/ } @{$config{default_architectures}};
31	neronus-guest	1068
32	neronus-guest	968	# Return the list of usernames
33			sub get_bugs_users {
34			my $topdir = "$gSpoolDir/user";
35			my @ret = ();
36			# see Debbugs::User::filefromemail for why 0...6
37			for(my $i = 0; $i < 7; $i++) {
38			my $dir = "$topdir/$i";
39			opendir DIR, $dir or die "Can't open dir $dir: $!";
40			# Replace all occurences of %dd with the corresponding
41			# character represented by dd, where dd is a hexadecimal
42			# number
43			push @ret, map { s/%(..)/chr(hex($1))/ge; $_ } readdir DIR;
44			}
45			return @ret;
46			}
47
48	neronus-guest	973	sub parse_time {
49			if(shift =~ /(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)/) {
50			return ($1, $2, $3, $4, $5, $6);
51			}
52			return undef;
53			}
54
55
56			sub get_db_max_last_modified {
57			my $dbh = shift or die "Argument required";
58			my $sth = $dbh->prepare("SELECT MAX (last_modified) FROM bugs");
59			$sth->execute() or die $!;
60			my $date = $sth->fetchrow_array();
61			if(defined $date) {
62			my ($year, $month, $day, $hour, $minute, $second) = parse_time($date);
63			return timelocal($second, $minute, $hour, $day, $month-1, $year);
64			} else {
65			return 0;
66			}
67			}
68
69			sub get_mtime {
70			return ((stat(shift))[9]);
71			}
72
73			sub get_modified_bugs {
74			my $prune_stamp = shift;
75			die "Argument required" unless defined $prune_stamp;
76			my $top_dir = $gSpoolDir;
77			my @result = ();
78			foreach my $sub (qw(archive db-h)) {
79			my $spool = "$top_dir/$sub";
80			foreach my $subsub (glob "$spool/*") {
81			if( -d $subsub and get_mtime($subsub) > $prune_stamp ) {
82			push @result,
83			map { s{./(.)\.log}{$1}; $_ }
84			grep { get_mtime("$_") > $prune_stamp }
85			glob "$subsub/*.log";
86			}
87			}
88			}
89			return \@result;
90			}
91
92			sub without_duplicates {
93			my %h = ();
94			return (grep { ($h{$_}++ == 0) \|\| 0 } @_);
95			}
96
97	neronus-guest	1068	sub run_usertags {
98			my ($config, $source, $dbh) = @_;
99	neronus-guest	945	my %src_config = %{$config->{$source}};
100	neronus-guest	1068	my $table = $src_config{'usertags-table'} or die "usertags-table not specified for source $source";
101			our $timing;
102			our $t;
103	neronus-guest	919
104
105	lucas	1055	$t = time();
106	neronus-guest	968	# Free usertags table
107	neronus-guest	1068	$dbh->do("DELETE FROM $table") or die
108			"Couldn't empty $table: $!";
109	lucas	1055	print "Deleting usertags: ",(time() - $t),"s\n" if $timing;
110			$t = time();
111	neronus-guest	971	# read and insert user tags
112	neronus-guest	968	my @users = get_bugs_users();
113			foreach my $user (@users) {
114	neronus-guest	1053	#read_usertags(\%tags, $user);
115			my $u = Debbugs::User->new($user);
116			my %tags = %{$u->{tags}};
117	neronus-guest	968	$user = $dbh->quote($user);
118			foreach my $tag (keys %tags) {
119			my $qtag = $dbh->quote($tag);
120	kroeckx	1264	map { $dbh->do("INSERT INTO $table (email, tag, id) VALUES ($user, $qtag, $_)") or die $! } @{$tags{$tag}};
121	neronus-guest	968	}
122			}
123	neronus-guest	1068	}
124
125			sub run {
126			my ($config, $source, $dbh) = @_;
127
128			our $t;
129			our $timing;
130			my %src_config = %{$config->{$source}};
131			my $table = $src_config{table};
132			my $archived_table = $src_config{'archived-table'};
133
134	lucas	1539	if (!$src_config{debug}) {
135			run_usertags($config, $source, $dbh);
136			print "Inserting usertags: ",(time() - $t),"s\n" if $timing;
137			$t = time();
138			}
139
140
141	lucas	1299	my %pkgsrc = %{getpkgsrc()};
142
143	neronus-guest	973	my @modified_bugs;
144	lucas	1243
145	neronus-guest	973	if($src_config{archived}) {
146	lucas	1249	# some bugs (the unarchived ones) are in both list. exclude them.
147			my %unarchived;
148			foreach my $b (get_bugs()) {
149			$unarchived{$b} = 1;
150			}
151			foreach my $b (get_bugs(archive => 1)) {
152			push(@modified_bugs, $b) if not $unarchived{$b};
153			}
154	neronus-guest	973	} else {
155			@modified_bugs = get_bugs();
156			}
157	lucas	1059	my @modified_bugs2;
158			if ($src_config{debug}) {
159	lucas	1249	print "Running in debug mode with restricted bug list!!\n";
160			foreach my $b (@modified_bugs) {
161			push(@modified_bugs2, $b) if ($b =~ /58$/);
162	lucas	1059	}
163			@modified_bugs = @modified_bugs2;
164			}
165
166	lucas	1055	print "Fetching list of ",scalar(@modified_bugs), " bugs to insert: ",(time() - $t),"s\n" if $timing;
167			$t = time();
168	neronus-guest	973
169	neronus-guest	1068	foreach my $prefix ($table, $archived_table) {
170	lucas	1694	foreach my $postfix (qw{_packages _merged_with _found_in _fixed_in _tags _blocks _blockedby}, '') {
171	neronus-guest	1068	my $sth = $dbh->prepare("DELETE FROM $prefix$postfix WHERE id = \$1");
172			map {
173			$sth->execute($_) or die $!;
174			} @modified_bugs;
175			}
176	neronus-guest	973	}
177	lucas	1055	print "Deleting bugs: ",(time() - $t),"s\n" if $timing;
178			$t = time();
179	neronus-guest	919
180	neronus-guest	945	my $location = $src_config{archived} ? 'archive' : 'db_h';
181	neronus-guest	1068	$table = $src_config{archived} ? $archived_table : $table;
182	neronus-guest	919	# Read all bugs
183	lucas	1907	my $insert_bugs_handle = $dbh->prepare("INSERT INTO $table (id, package, source, arrival, status, severity, submitter, submitter_name, submitter_email, owner, owner_name, owner_email, done, done_name, done_email, title, forwarded, last_modified, affects_oldstable, affects_stable, affects_testing, affects_unstable, affects_experimental) VALUES (\$1, \$2, \$3, \$4::abstime, \$5, \$6, \$7, \$8, \$9, \$10, \$11, \$12, \$13, \$14, \$15, \$16, \$17, \$18::abstime, \$19, \$20, \$21, \$22, \$23)");
184	lucas	1304	my $insert_bugs_packages_handle = $dbh->prepare("INSERT INTO ${table}_packages (id, package, source) VALUES (\$1, \$2, \$3)");
185	kroeckx	1264	my $insert_bugs_found_handle = $dbh->prepare("INSERT INTO ${table}_found_in (id, version) VALUES (\$1, \$2)");
186			my $insert_bugs_fixed_handle = $dbh->prepare("INSERT INTO ${table}_fixed_in (id, version) VALUES (\$1, \$2)");
187			my $insert_bugs_merged_handle = $dbh->prepare("INSERT INTO ${table}_merged_with (id, merged_with) VALUES (\$1, \$2)");
188			my $insert_bugs_tags_handle = $dbh->prepare("INSERT INTO ${table}_tags (id, tag) VALUES (\$1, \$2)");
189	lucas	1694	my $insert_bugs_blocks_handle = $dbh->prepare("INSERT INTO ${table}_blocks (id, blocked) VALUES (\$1, \$2)");
190			my $insert_bugs_blockedby_handle = $dbh->prepare("INSERT INTO ${table}_blockedby (id, blocker) VALUES (\$1, \$2)");
191	neronus-guest	1068	$insert_bugs_handle->bind_param(4, undef, SQL_INTEGER);
192	lucas	1539	$insert_bugs_handle->bind_param(18, undef, SQL_INTEGER);
193	neronus-guest	1068
194			$t = time();
195	neronus-guest	973	foreach my $bug_nr (@modified_bugs) {
196	neronus-guest	930	# Fetch bug using Debbugs
197	neronus-guest	946	# Bugs which were once archived and have been unarchived again will appear in get_bugs(archive => 1).
198			# However, those bugs are not to be found in location 'archive', so we detect them, and skip them
199	neronus-guest	973	my $bug_ref = read_bug(bug => $bug_nr, location => $location) or (print STDERR "Could not read file for bug $bug_nr; skipping\n" and next);
200	neronus-guest	942	# Yeah, great, why does get_bug_status not accept a location?
201	neronus-guest	945	my %bug = %{get_bug_status(bug => $bug_nr, status => $bug_ref)};
202	neronus-guest	936
203	neronus-guest	930	# Convert data where necessary
204	neronus-guest	1077	my @found_versions = @{$bug{found_versions}};
205			my @fixed_versions = @{$bug{fixed_versions}};
206			my @tags = split / /, $bug{keywords};
207	neronus-guest	939
208	neronus-guest	945	# log_modified and date are not necessarily set. If they are not available, they
209			# are assumed to be epoch (i.e. bug #4170)
210			map {
211			if($bug{$_}) {
212	neronus-guest	1068	$bug{$_} = int($bug{$_});
213	neronus-guest	945	} else {
214	neronus-guest	1068	$bug{$_} = 0;
215	neronus-guest	945	}
216			} qw{date log_modified};
217	neronus-guest	939
218	lucas	1596	my $srcpkg;
219	lucas	1588	if ($bug{package} =~ /^src:(.*)/)
220			{
221	lucas	1596	$srcpkg = $1;
222	lucas	1588	} else {
223	lucas	1596	$srcpkg = exists($pkgsrc{$bug{package}}) ? $pkgsrc{$bug{package}} : $bug{package};
224	lucas	1588	}
225	neronus-guest	945
226	lucas	1539	# split emails
227			my (@addr, $submitter_name, $submitter_email, $owner_name, $owner_email, $done_name, $done_email);
228	lucas	1540	if ($bug{originator}) {
229			@addr = Mail::Address->parse($bug{originator});
230			$submitter_name = $addr[0]->phrase;
231			$submitter_email = $addr[0]->address;
232			} else {
233			$submitter_name = '';
234			$submitter_email = '';
235			}
236	lucas	1539
237			if ($bug{owner}) {
238			@addr = Mail::Address->parse($bug{owner});
239			$owner_name = $addr[0]->phrase;
240			$owner_email = $addr[0]->address;
241			} else {
242			$owner_name = '';
243			$owner_email = '';
244			}
245
246			if ($bug{done}) {
247			@addr = Mail::Address->parse($bug{done});
248			$done_name = $addr[0]->phrase;
249			$done_email = $addr[0]->address;
250			} else {
251			$done_name = '';
252			$done_email = '';
253			}
254
255	neronus-guest	922	#Calculate bug presence in distributions
256	lucas	1907	my (present_in_oldstable, $present_in_stable, $present_in_testing, $present_in_unstable, $present_in_experimental);
257	neronus-guest	1068	if($src_config{archived}) {
258	lucas	1907	$present_in_oldstable = $present_in_stable = $present_in_testing = $present_in_unstable = $present_in_experimental = 'FALSE';
259	neronus-guest	922	} else {
260	lucas	1907	$present_in_oldstable =
261			bug_presence(bug => $bug_nr, status => \%bug,
262			dist => 'oldstable',
263			arch => \@archs);
264	neronus-guest	1068	$present_in_stable =
265			bug_presence(bug => $bug_nr, status => \%bug,
266	lucas	1331	dist => 'stable',
267	lucas	1335	arch => \@archs);
268	neronus-guest	1068	$present_in_testing =
269			bug_presence(bug => $bug_nr, status => \%bug,
270	lucas	1331	dist => 'testing',
271	lucas	1335	arch => \@archs);
272	neronus-guest	1068	$present_in_unstable =
273			bug_presence(bug => $bug_nr, status => \%bug,
274	lucas	1331	dist => 'unstable',
275	lucas	1335	arch => \@archs);
276	kroeckx	1287	$present_in_experimental =
277			bug_presence(bug => $bug_nr, status => \%bug,
278	lucas	1331	dist => 'experimental',
279	lucas	1335	arch => \@archs);
280	lucas	1331
281	lucas	1907	if(!defined($present_in_oldstable) or !defined($present_in_stable) or !defined($present_in_unstable) or !defined($present_in_testing) or !defined($present_in_experimental)) {
282	neronus-guest	1068	print "NUMBER: $bug_nr\n";
283			}
284
285	lucas	1907	if(defined($present_in_oldstable) and ($present_in_oldstable eq 'absent' or $present_in_oldstable eq 'fixed')) {
286			$present_in_oldstable = 'FALSE';
287			} else {
288			$present_in_oldstable = 'TRUE';
289			}
290	neronus-guest	1068	if(defined($present_in_stable) and ($present_in_stable eq 'absent' or $present_in_stable eq 'fixed')) {
291			$present_in_stable = 'FALSE';
292			} else {
293			$present_in_stable = 'TRUE';
294			}
295			if(defined($present_in_testing) and ($present_in_testing eq 'absent' or $present_in_testing eq 'fixed')) {
296			$present_in_testing = 'FALSE';
297			} else {
298			$present_in_testing = 'TRUE';
299			}
300			if(defined($present_in_unstable) and ($present_in_unstable eq 'absent' or $present_in_unstable eq 'fixed')) {
301			$present_in_unstable = 'FALSE';
302			} else {
303			$present_in_unstable = 'TRUE';
304			}
305	kroeckx	1287	if(defined($present_in_experimental) and ($present_in_experimental eq 'absent' or $present_in_experimental eq 'fixed')) {
306			$present_in_experimental = 'FALSE';
307			} else {
308			$present_in_experimental = 'TRUE';
309			}
310	neronus-guest	922	}
311
312	neronus-guest	920	# Insert data into bugs table
313	lucas	1596	$insert_bugs_handle->execute($bug_nr, $bug{package}, $srcpkg, $bug{date}, $bug{pending},
314	lucas	1539	$bug{severity}, $bug{originator}, $submitter_name, $submitter_email, $bug{owner},
315			$owner_name, $owner_email, $bug{done}, $done_name, $done_email, $bug{subject},
316			$bug{forwarded}, $bug{log_modified},
317	lucas	1907	$present_in_oldstable, $present_in_stable, $present_in_testing, $present_in_unstable, $present_in_experimental) or die $!;
318	neronus-guest	920
319	lucas	1304	my $src;
320	lucas	1596	foreach my $pkg (keys %{{ map { $_ => 1 } split(/\s[, ]\s/, $bug{package})}}) {
321	lucas	1588	if ($pkg =~ /^src:(.*)/)
322			{
323			$src = $1;
324			} else {
325			$src = exists($pkgsrc{$pkg}) ? $pkgsrc{$pkg} : $pkg;
326			}
327	lucas	1304	$insert_bugs_packages_handle->execute($bug_nr, $pkg, $src) or die $!;
328			}
329
330	neronus-guest	920	# insert data into bug_fixed_in and bug_found_in tables
331	neronus-guest	973	foreach my $version (without_duplicates(@found_versions)) {
332	neronus-guest	1068	$insert_bugs_found_handle->execute($bug_nr, $version) or die $!;
333	neronus-guest	920	}
334	neronus-guest	973	foreach my $version (without_duplicates(@fixed_versions)) {
335	neronus-guest	1068	$insert_bugs_fixed_handle->execute($bug_nr, $version) or die $!;
336	neronus-guest	920	}
337	neronus-guest	973	foreach my $mergee (without_duplicates(split / /, $bug{mergedwith})) {
338	neronus-guest	1068	$insert_bugs_merged_handle->execute($bug_nr, $mergee) or die $!;
339	neronus-guest	921	}
340	lucas	1694	foreach my $blocked (without_duplicates(split / /, $bug{blocks})) {
341			$insert_bugs_blocks_handle->execute($bug_nr, $blocked) or die $!;
342			}
343			foreach my $blocker (without_duplicates(split / /, $bug{blockedby})) {
344			$insert_bugs_blockedby_handle->execute($bug_nr, $blocker) or die $!;
345			}
346	neronus-guest	983	foreach my $tag (without_duplicates(@tags)) {
347	neronus-guest	1068	$insert_bugs_tags_handle->execute($bug_nr, $tag) or die $!;
348	neronus-guest	983	}
349	neronus-guest	919	}
350	lucas	1249	print "Inserting bugs: ",(time() - $t),"s\n" if $timing;
351	lucas	1531
352			foreach my $postfix (qw{_packages _merged_with _found_in _fixed_in _tags}, '') {
353			my $sth = $dbh->prepare("ANALYZE $table$postfix");
354			$sth->execute() or die $!;
355			}
356
357			my $sth = $dbh->prepare("ANALYZE ".$src_config{'usertags-table'});
358			$sth->execute() or die $!;
359
360			print "Analyzing bugs: ",(time() - $t),"s\n" if $timing;
361	lucas	1532
362			$dbh->commit();
363			print "Committing bugs: ",(time() - $t),"s\n" if $timing;
364	neronus-guest	1068	}
365	neronus-guest	919
366	neronus-guest	1068	sub main {
367			if(@ARGV != 3) {
368			print STDERR "Usage: $0 <config> <command> <source>\n";
369			exit 1;
370			}
371
372	neronus-guest	1077	our $t = time();
373	neronus-guest	1070	our $timing;
374
375	neronus-guest	1068	my $config = LoadFile($ARGV[0]) or die "Could not load configuration: $!";
376			my $command = $ARGV[1];
377			my $source = $ARGV[2];
378
379			my $dbname = $config->{general}->{dbname};
380	lucas	1351	my $dbport;
381			if ($config->{general}->{dbport} ne '') {
382			$dbport = ";port=".$config->{general}->{dbport};
383			} else {
384			$dbport = "";
385			}
386	neronus-guest	1068	# Connection to DB
387	lucas	1351	my $dbh = DBI->connect("dbi:Pg:dbname=$dbname".$dbport);
388	neronus-guest	1068	# We want to commit the transaction as a hole at the end
389			$dbh->{AutoCommit} = 0;
390	lucas	1243	$dbh->do('SET CONSTRAINTS ALL DEFERRED');
391	neronus-guest	1068
392			if($command eq 'run') {
393			run($config, $source, $dbh);
394			} else {
395			print STDERR "<command> has to be one of run, drop and setup\n";
396			exit(1)
397			}
398
399	neronus-guest	919	}
400
401			main();