dehs/trunk/dehs_pg.php

#!/usr/bin/php4 -q
<?php
/*

Originally written by Stefano Fabri <bluefuture@nospam@email.it>
Copyright 2004, Stefano Fabri
Copyright 2007, 2008 by Raphael Geissert <atomo64@gmail.com>

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

// stabilire una connessione
#ini_set ( "error_reporting", "E_COMPILE_ERROR|E_ERROR|E_CORE_ERROR");
ini_set ("html_errors", "off");
set_time_limit(0);
ini_set ( "memory_limit", "40M");
#print_R($_SERVER);
require "dehs.config.inc";
require "events.class.php";
dir_ctrl($dirs);
if (!function_exists('file_get_contents')) {
    function file_get_contents($filename) {
        if (!($file = fopen($filename,"rb"))) return false;
        else {
            $contents = "";
            do {
                $data = fread($file, 4096);
                if (strlen($data) == 0) {
                    break;
                }
                $contents .= $data;
            } while (true);
            fclose($file);
            return $contents;
        }
    }
}

if (!function_exists('file_put_contents')) {
    function file_put_contents($filename, $content, $flags = 0) {
        if (!($file = fopen($filename,$flags))) return false;
        else {
            $n = fwrite($file, $content);
            fclose($file);
            return true;
        }
    }
}

if (in_array("-db_add",$argv))  db_add((!empty($argv[2])?$argv[2]:''), false);
    elseif (in_array("-db_query",$argv))  db_query($argv[2],(!empty($argv[3])?$argv[3]:'%') );
    elseif (in_array("-dl_diffs",$argv) || in_array("-keep_diff",$argv))  dl_diffs($argv[2]);
    elseif (in_array("-keep_sources",$argv) || in_array("-download_sources",$argv))  download_sources();
    elseif (in_array("-db_popcon",$argv))  db_popcon();
    elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv))  dl_popcon();
    elseif (in_array("-db_upstream",$argv))  db_upstream((!empty($argv[2])?$argv[2]:'%'));
    elseif (in_array("-dl_from_vcs",$argv))  dl_from_vcs((!empty($argv[2])?$argv[2]:'%'),(!empty($argv[3])?$argv[3]:'%'));
    elseif (in_array("-update_all",$argv))  update_all();
    elseif (in_array("-update_new",$argv))  update_new();
    elseif (in_array("-update_bogus",$argv))  update_bogus();
    elseif (in_array("-check_db",$argv))  check_db();
    elseif (in_array("-dehsqa_db",$argv))  dehsqa_db();
    elseif (in_array("-clear_db",$argv))  clear_db();
    elseif (in_array("-db_up_error",$argv))  db_up_error();
    elseif (in_array("-up_changes",$argv))  up_changes((!empty($argv[2])?$argv[2]:'%'), true);
    elseif (in_array("-watchf",$argv) || in_array("-wwiz",$argv))  watch_wizard((!empty($argv[2])?$argv[2]:'%'));
    elseif (in_array("-kb_total",$argv))  print "Diff
#stat_up_error();
#stat_up_error();to download Kb => " . kb_total($argv[2]) . "\n" ;
    elseif (in_array("-watch_stat",$argv))  watch_stat();
    else print "Command $argv[1] not found\n";
#stat_up_error();
function dir_ctrl($dirs) {
    if (!is_array($dirs)) {
        $dirs = array($dirs);
    }
    foreach ($dirs as $dir) {
        if (!is_dir($dir)) exec("mkdir -m 0740 -p " .  escapeshellarg($dir), $output, $res);
        if ($res != 0) die_status("Error creating basic dir: $dir\n");
    }
}
function ext_watch ($file,$pkg,$version) {
    if (!file_exists($file)) return "";
    $zp=gzopen($file,"r") or die_status("Fail to open $file");
    while (!gzeof($zp)) {
        $line = gzgets ($zp,4096);
        if (preg_match('/\+\+\+\s.+\/debian\/watch$/',$line)) {
            $line = gzgets ($zp,4096);
            preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght);
            for ($i = 1; $i <= $watch_lenght[1]; $i++)      {
                $line=gzgets ($zp,4096);
                $watch .=substr($line,1);
            }
            break;
        }
    }
    if ($watch!=null) return $watch;
    else return NULL;
}
function ftp_cn($login,$password,$ftp_server) {
    $conn_id = @ftp_connect($ftp_server) or die_status("Error: Cannot contact ftp server $ftp_server\n");
    // login con user name e password
    ftp_pasv($conn_id,1);
    $login_result = ftp_login($conn_id, "anonymous","bo@bo.org");
    // controllo della connessione
    if ((!$conn_id) || (!$login_result)) {
        echo "Ftp connection failed!\n";
        echo "Failed connection to $ftp_server\n";
        die_status();
    } else {
        return $conn_id;
    }
}
function dl_diffs($initial = '') {
    global $dirs,$dbconn,$mirrors;
    check_db();
    $conn_normal=ftp_cn("anonymous","bo@bo.org",$mirrors['unstable']);
    #$conn_non_us=ftp_cn("anonymous","bo@bo.org",$mirrors["non-US"]);
    $db = pg_pconnect($dbconn) ;
    $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR die_status("Error in diff query\n");
    $total=pg_numrows($rsql);
    $btotal=byte_total($db,$initial);
    print "Total Diff to downlaod Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n";
    while ($res_array=pg_fetch_array($rsql)) {
        #if ($res_array['dist']=='non-US') $conn_id = $conn_non_us;
        #else
        $conn_id = $conn_normal;
        if   ($localfile=download_diff($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) {
            $watch=ext_watch($localfile,$res_array[name],$res_array[version]);
            unlink($localfile);
            if ($watch!=NULL) {
                $watch=iconv("ISO-8859-1","UTF-8",$watch);
                $watch= pg_escape_string($watch);
                pg_exec($db, "UPDATE pkgs SET watch='" . $watch . "', md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or die_status("Query error");
            }
            else pg_exec($db, "UPDATE pkgs SET watch=NULL, md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or die_status("Errore nella query");
        }
        $btotal=$btotal-$res_array['bytes'];
        number_format ($btotal/1024, 2, ",",".");
        print "\rDiffs left =>  " . $total-- . " Bytes left => " . number_format ($btotal/1024, 2, ",",".") . " - " . $res_array[name];
    }
    ftp_quit($conn_normal);
    pg_close($db);
    return "Fatto\n";
}
function download_diff(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) {
    global $dirs,$mirrors;
    $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz";
    #if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz";
    #else
    $remotefile="/debian/$directory/$pkg" . "_" . "$version.diff.gz";
    #print "$localfile - $remotefile\n";
    if      (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
        #    print "$pkg Diff.gz => Downloaded\n";
        if(md5_of_file($localfile)==$md5_atsource) {
            #print "MD5 => OK\n";
            #unlink($localfile);
            return $localfile;
        }
    }
    print "Download Diff.gz of package $pkg => Fallito\n";
    return false;
}
function db_add($initial='') {
    global $dirs,$dists,$dbconn;
    check_db();
    download_sources();

    $db = pg_pconnect($dbconn)  or  die_status(pg_last_error($db));
    pg_exec($db, "CREATE TEMP TABLE pkgs_atsrc (name text,dist text)") or die_status('Error creating temp table pkgs_atsrc');
    pg_exec($db, "CREATE UNIQUE INDEX idxdis on pkgs_atsrc (name,dist)") or die_status('Error creating index on temp table');
    pg_exec($db, "CREATE TEMP TABLE bin_atsrc (name text,bin_name text, dist text)") or die_status('Error creating temp table pkgs_atsrc');
    pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die_status('Error creating index on temp table bin_atsrc');

    foreach ($dists as $dist=>$sections) {
        foreach ($sections as $section) {
            $filename=$dirs['sources_dir'] . "/$dist/$section/Sources.gz";
            print "\nI'm parsing $dist/$section => Sources.gz\n";
            # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5
            $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S{32})\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?(Dm-Upload-Allowed:\s(.+)[^a])?(Homepage:\s(.+)[^a])?(Vcs-Browser:\s(.+)[^a])?(Vcs-([A-Z][a-z]+):\s(.+)[^a])?(Checksums-\S+:\s(.+)[^a])?/Ssi";
            $zp = gzopen($filename, "r") or die_status("Could not open $filename");
            $extracted=tempnam("/tmp/", "$dist_$section_Sources");
            exec("gzip -c -d $filename > $extracted");
            $sourcesize=filesize($extracted);
            unlink($extracted);
            if(filesize($filename)>20) {
                while (!gzeof($zp)) {
                    $line='';
                    while ($line!="\n")  {
                        $line = gzgets ($zp,4096) ;
                        $buff1 .= $line;
                    }
                    if (preg_match($regexp,$buff1,$matches))  {
                        if ($matches[5]) {
                            $matches[5]=iconv("ISO-8859-1","UTF-8",$matches[5]);
                            $matches[5]=pg_escape_string($matches[5]);
                        }
                        if ($matches[10]) {
                            $matches[10]=iconv("ISO-8859-1","UTF-8",$matches[10]);
                            $matches[10]=pg_escape_string($matches[10]);
                        }
                        @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR die_status("Temp table pkgs_atsrc query error");
                        $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders,homepage,vcs_browser,vcs_type,vcs,vcsoversource,lastpkgsourcesupdate) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]','$matches[14]','$matches[16]','$matches[18]','$matches[19]','0',now())");
                        if (!$rst) {
                            $rsql=pg_exec($db, "SELECT pkgs.version FROM pkgs WHERE name='$matches[1]' AND dist='$dist';");
                            $version = $matches[4]; // just for safety
                            while ($res_array=pg_fetch_array($rsql)) {
                                $version = $res_array['version'];
                            }
                            $extra = '';
                            if ($version != $matches[4]) {
                                // only update the dversionmangled field if the versions differ
                                $extra = ",dversionmangled='$matches[4]'";
                                // also reset wwiz_type
                                $extra = ",wwiz_type=NULL";
                                // and update the lastpkgsourcesupdate field
                                $extra = ",lastpkgsourcesupdate=now()";
                            }
                            $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]'$extra,maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]',homepage='$matches[14]',vcs_browser='$matches[16]',vcs_type='$matches[18]',vcs='$matches[19]' WHERE name='$matches[1]' AND dist='$dist'") OR die_status("\nDb adding error =>" . pg_last_error() . "\n");
                        }
                        $bin_names=split(",", $matches[2]);
                        foreach ($bin_names as $bin_name) {
                            pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR die_status("Temp table pkgs_atsrc query error");
                            $rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ;
                        }
                    }
                    else {
                        preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches);
                        if(strncmp($initial,$matches[1],strlen($initial))<0) break;
                    }
                    $left=$sourcesize-gztell($zp);
                    print "\rSource file Left => $left";
                    $buff1="";
                    unset($matches);
                }
            }
            gzclose($zp) ;
        }
    }
    clear_db($db);
    pg_close($db);
}
function db_query($pkg,$dist='%') {
    global $dirs,$dbconn;
    $db = pg_pconnect($dbconn )  or  die_status(pg_last_error($db));
    $rsql=pg_exec($db, "SELECT * FROM  pkgs
                              WHERE name='$pkg' AND dist LIKE '$dist';");
    while ($res_array=pg_fetch_array($rsql)) {
        print "Package=> " . $res_array['name'] . "\n";
        print "Version => " . $res_array['version'] . "\n";
        print "Mangled version => " . $res_array['dversionmangled'] . "\n";
        print "Diff MD5 Downloaded => " . $res_array['md5_diff'] . "\n";
        print "Diff MD5 at Sources => " . $res_array['md5_atsource'] . "\n";
        print "Distribution => " . $res_array['dist'] . "\n";
        print "Section => " . $res_array['section'] . "\n";
        print "KBytes => " . number_format ( $res_array['bytes']/1024, 2, ",",".") . "\n";
        print "Pop Inst=>" . $res_array['pop_inst'] . "\n";
        if ($res_array['watch']) {
            $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['watch']);
            print "Version Upstream => " . $uscan_res['uversion'] . "\n";
            print "Watch => " . trim($res_array['watch'])."\n";
            print "Watch Warnings => " . $uscan_res['warnings'];
        }
        else print "Watch => Not present" . "\n";
        if ($res_array['wwiz_type']) {
            $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['wwiz']);
            print "WWiz Upstream => " . $uscan_res['uversion'] . "\n";
            print "WWiz => " . $res_array['wwiz']."\n";
            print "WWiz Warings => " . $uscan_res['warnings'];
        }
        print "\n";
    }
    if (pg_numrows($rsql)==0) echo "Package=> " . $pkg . "\nNot found\n!!";
    pg_close($db);
}
function byte_total(&$db,$initial) {
    global $dirs;
    $rsql=pg_exec ($db,"Select sum(bytes) as totbytes FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)");
    $res_array=pg_fetch_array($rsql);
    return $res_array[totbytes] ;
}
function download_sources () {
    global $dirs,$dists,$mirrors;
    foreach ($dists as $dist=>$sections) {
        $server=$mirrors[$dist];
        $conn_id = ftp_cn("anonymous","bo@bo.org",$server);
        foreach ($sections as $section) {
            $localfile=$dirs[sources_dir] . "/$dist/$section/Sources.gz";
            $remotefile=remote_dist_dir($dist) . "/$dist/$section/source/Sources.gz";
            $sources_info=md5_source($section,$dist);
            if (!file_exists ($dirs[sources_dir] . "/$dist/")) mkdir ( $dirs[sources_dir] . "/$dist",0740);
            if (!file_exists ($dirs[sources_dir] . "/$dist/$section")) mkdir ($dirs[sources_dir] . "/$dist/$section",0740);
            if (!file_exists($localfile) OR md5_of_file($localfile)!=$sources_info[md5])  {
                print "Downloading $remotefile Bytes $sources_info[bytes]=> ";
                if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
                    print "Done\n";
                    if(md5_of_file($localfile)==$sources_info[md5]) print "MD5 => OK\n";
                    else print "MD5 => no match\n";
                }
                else {
                    unlink($localfile);
                    print "Fail\n";
                    die_status ("Error in source file");
                }
            }
        }
        ftp_quit($conn_id);
    }
}
function md5_source($section,$dist) {
    global $mirrors;
    $release=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die_status("Failed to check MD5 of sources");
    preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $release,$matches);
    return array("md5"=>$matches[1],"bytes"=>$matches[2]);
    # 183dea9ca3c2e35ffd6cf5eee9501c21          2941637 main/binary-i386/Packages.gz
}

function vers_conv($debvers, $extendedConv = true) {
    // Strip off the epoch
    $pos = strpos($debvers, ':');
    if ( $pos !== FALSE) {
        $debvers = substr($debvers, $pos+1);
    }

    // strip off the Debian revision (look from the back of the string)
    $pos = strrpos($debvers, '-');
    if ( $pos !== FALSE) {
        $debvers = substr($debvers, 0, $pos);
    }

    if ($extendedConv) {
        // strip off repacking indicators
        $debvers = preg_replace("/[-.+~]?(ds|dfsg|debian)(.*)/i", "", $debvers);
    }

    return $debvers;
}
function check_db() {
    global $dirs,$dbconn;
    $db = pg_pconnect($dbconn) or die_status(1/*pg_last_notice($db)*/);
    if (!pg_table_exists($db, "pkgs")) {
        pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text, lastcheck timestamp, homepage text, vcs text,  vcs_type text, vcs_browser text,vcs_watch text, vcsoversource bool,vcs_up_version text,vcs_dversionmangled text,vcs_up_url text, vcs_watch_warn text, vcs_updated bool, lastvcsfetch timestamp, lastupvsdebsync timestamp, lastpkgsourcesupdate timestamp, failures_count numeric, pop_inst numeric,pop_vote numeric)") or die_status('Error creating table pkgs\n');
        pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db));
        pg_exec($db, "CREATE INDEX checkx on pkgs (lastcheck)") or die_status('Errore creating index - ' . pg_last_error($db));
        pg_exec($db, "CREATE INDEX updatex on pkgs (lastcheck,lastpkgsourcesupdate);") or die_status('Errore creating index - ' . pg_last_error($db));
        pg_exec($db, "CREATE INDEX maintx on pkgs (maint)") or die_status('Errore creating index - ' . pg_last_error($db));
        $sw=true;
    }
    if (!pg_table_exists($db, "binpkgs")) {
        pg_exec($db, "CREATE TABLE binpkgs (id serial PRIMARY KEY, name text,bin_name text,dist text, pop_inst numeric,pop_vote numeric, up_error numeric,avg_error_date numeric)") or die_status('Error creating table binpkgs\n');
        pg_exec($db, "CREATE UNIQUE INDEX idxbin on binpkgs (name,bin_name,dist)") or die_status('Errore creating index - ' . pg_last_error($db));
        $sw=true;
    }
    if (!pg_table_exists($db, "events")) {
        pg_exec($db, "CREATE TABLE events (id serial PRIMARY KEY, name text,type text, dist text, stamp timestamp, content text)") or die_status('Error creating table events\n');
        pg_exec($db, "CREATE UNIQUE INDEX idxevents on events (name,dist,content)") or die_status('Error creating index - ' . pg_last_error($db));
        pg_exec($db, "CREATE INDEX pkgxeventtype on events (name,type,dist)") or die_status('Error creating index - ' . pg_last_error($db));
        $sw=true;
    }
    if (!pg_table_exists($db, "stamps")) {
        pg_exec($db, "CREATE TABLE stamps (type text PRIMARY KEY, stamp timestamp, pid text)") or die_status('Error creating table stamps\n');
        $sw=true;
    }
    #if (!pg_table_exists($db,"upstream")){
    #  pg_exec($db, "CREATE TABLE upstream (name text,dist text,up_version text,watch_warn text,updated bool)") or die_status('Errore nel creare la tabella');
    #  pg_exec($db, "CREATE UNIQUE INDEX idxupstrm on upstream (name,dist)") or die_status('Errore nel creare gli indici ' . pg_last_error($db));
    #  $sw=true;
    #}
    pg_close($db);
    if($sw) return $sw;
    else return false;
}
function pg_table_exists (&$dblink, $table) {
    $sql = "SELECT relname as tbl FROM pg_class WHERE relname = '$table'";
    if ($res = pg_exec ($dblink, $sql)) {
        return pg_numrows($res) > 0;
    } else {
        return false; // or throw exception
    }
}


function watch_stat() {
    global $dirs,$dbconn;
    $db = pg_pconnect($dbconn) or die_status(pg_last_error($db));
    $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch!=''");
    $tot_watch=pg_fetch_array($rsql);
    print "Total packages with Watch =>$tot_watch[0]\n";
    $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch IS NULL");
    $tot_nowatch=pg_fetch_array($rsql);
    pg_close($db);
    print "Totale packages without Watch => $tot_nowatch[0]\n";
    print "Watch presence rate => " . number_format ($tot_watch[0]/($tot_nowatch[0]+$tot_watch[0])*100, 2, ",",".")  . "%\n";
}
function remote_dist_dir($dist) {
    #if ($dist=="non-US") return "/debian-non-US/dists/unstable";
    #else
    return "/debian/dists";
}
function clear_db(&$db) {
    #global $dbconn;
    #if (check_db()) die_status("Il Database era vuoto\n");
    #download_sources();
    #$db = pg_pconnect($dbconn)  or  die_status(pg_result_error($db));
    $sql1=      "Delete from binpkgs where id NOT IN
                        (select id FROM binpkgs INNER JOIN bin_atsrc USING (name,bin_name,dist))" ;
    $sql2=      "Delete from pkgs where id NOT IN (select id FROM pkgs INNER JOIN pkgs_atsrc USING (name,dist))" ;
    $sql3=      "UPDATE pkgs SET wwiz=NULL, wwiz_type=NULL, wwiz_version=NULL WHERE watch IS NOT NULL and watch!=''" ;
    $sql4=      "UPDATE pkgs SET up_changes=NULL WHERE updated='1'";
    if(pg_exec($db,$sql1) && pg_exec($db,$sql2) && pg_exec($db,$sql3) && pg_exec($db,$sql4)) {
        print "Db clear complete\n";
    }
}
function db_popcon() {
    global $dirs,$dbconn;
    $db = pg_pconnect($dbconn)  or  die_status(pg_last_error($db));
    $filename=$dirs['popcon_dir'] . "/popcon.db";
    $popcon_db = dba_open($filename, 'r-', 'db4');

    $rsql = pg_exec($db, "SELECT name FROM pkgs");

    while ($res_array=pg_fetch_array($rsql)) {
        $package = $res_array['name'];

        if (!dba_exists('si:' .  $package, $popcon_db)) {
            continue;
        }

        $inst = dba_fetch("si:$package", $popcon_db);
        $vote = dba_fetch("sv:$package", $popcon_db);

        // when one of the values is 'missing' set to -1 so the other value
        //  _is_ UPDATEd, but use -1 to catch those bugs
        $inst = (is_numeric($inst))? $inst : -1;
        $vote = (is_numeric($vote))? $vote : -1;

        @pg_exec($db, "UPDATE pkgs SET pop_inst='$inst',pop_vote='$vote' WHERE name='$package'");
    }

    dba_close($popcon_db);
    @pg_close($db);
}
function dl_popcon() {
    global $dirs,$dbconn;

    $localtempdir=$dirs['popcon_dir'] . "/new";
    dir_ctrl($localtempdir);

    $localfile=$dirs['popcon_dir'] . "/popcon.db";
    $remotefile='http://qa.debian.org/data/popcon/popcon.db';

    $return = 0;
    $output = null;
    exec('cd ' . escapeshellarg($localtempdir) . ' && ' .
         '/usr/bin/wget -qN ' . escapeshellarg($remotefile), $output, $return);

    if ($return != 0) {
        echo __FUNCTION__ . "wget exited with status $return!\n";
        return false;
    }

    return copy($localtempdir . '/popcon.db', $localfile);
}
function is_updated ($up_vers,$debian_vers, $dversionAsIs = false) {
    if ($up_vers=='') return 1;

    $extendedConv = true;

    if ($up_vers === $debian_vers) $dversionAsIs = true;

    if (strpos(strtolower($up_vers), 'dfsg') !== false && strpos(strtolower($debian_vers), 'dfsg') !== false) {
        $extendedConv = false;
    }

    passthru("dpkg --compare-versions ".escapeshellarg($up_vers)." le " . escapeshellarg($dversionAsIs?$debian_vers:vers_conv($debian_vers, $extendedConv)),$great);

    return (int)!($great);
}
function db_upstream($initial='%', $checkStamps = false, $limit = null) {
    global $dirs, $dbconn, $minimumAge;

    check_db();
    $db = pg_pconnect($dbconn) ;
    events::DB($db);

    $postWHERE = '';
    if ($checkStamps) {
        $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL OR (lastpkgsourcesupdate>lastcheck))";
    }

    $postORDER = '';
    if ($limit !== null && is_int($limit)) {
        $postORDER = 'LIMIT ' . $limit;
    }

    $rsql=pg_exec($db, "SELECT id, name,dist,up_version,version,watch,lastupvsdebsync,failures_count FROM pkgs WHERE watch is not null AND name LIKE '$initial' $postWHERE ORDER BY lastcheck DESC $postORDER");
    $tot=pg_numrows($rsql);

    while ($res_array=pg_fetch_array($rsql)) {

        $uscan_res=uscan_foo($res_array['name'],$res_array['version'],$res_array['watch']);

        #print $uscan_res['uversion'] . "\n Name => $res_array[name]\n";
        $version         = $res_array['version'];
        $dversionmangled = $res_array['version'];
        $lastupvsdebsync = $res_array['lastupvsdebsync'];
        $mangle = true;

        /* Do not increase the number of failures if uscan didn't throw any warning
            because it might be that the watch file is "empty"
        */
        if (!$uscan_res['uversion'] && $uscan_res['warnings']) {
            $res_array['failures_count']++;
        } else {
            $res_array['failures_count'] = 0;
        }

        print "\rVersion=>" . $uscan_res['dversion'];
        if ($uscan_res['dversion']!=null) $version=$uscan_res['dversion'];
        if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) {
            $dversionmangled=$uscan_res['dversionmangled'];
            $mangle = false;
        }

        $up_url=iconv("ISO-8859-1","UTF-8",$uscan_res['uurl']);
        $up_version=iconv("ISO-8859-1","UTF-8",$uscan_res['uversion']);

        $updated = $uscan_res['updated'];

        /*if (!$updated)*/ {
            $updated=is_updated($uscan_res['uversion'], $dversionmangled, !$mangle);
        }

        if (!$updated) {
            events::newVersion($res_array['name'], $res_array['version'], $up_version, $res_array['dist'], $up_url);
        } else {
            $lastupvsdebsync = sqldate();
        }

        if ($lastupvsdebsync === null) {
            $lastupvsdebsync = sqldate(0);
        }

        $keep_changes = (int) ($up_version != $res_array['up_version']);

        $watch_warn='';
        if($uscan_res['warnings'] != null) {
            $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8", $uscan_res['warnings']));
        }

        pg_exec($db, "UPDATE pkgs SET
                                    up_version='" . pg_escape_string($up_version) . "',
                                    dversionmangled='".pg_escape_string($dversionmangled)."',
                                    up_url='" . pg_escape_string($up_url) . "',
                                    watch_warn='" . $watch_warn . "',
                                    keep_changes='$keep_changes',
                                    wwiz_version=NULL,
                                    wwiz_type=NULL,
                                    wwiz=NULL,
                                    failures_count=$res_array[failures_count],
                                    updated='$updated',
                                    lastcheck=now(),
                                    lastupvsdebsync='$lastupvsdebsync'
                                  WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream");

        #print "\rUpdated " . $res_array['name'] .  " => $updated Up Version => ". $uscan_res['uversion'];
        print "\t Upstream left => " . $tot --;
        #                               }
        #                                                       else print "Added ". $res_array['name'] ."\n";
        #       if ($great==0) {

        #               print "Package=> " . $res_array['name'] . "\n";
        #               print "Version => " . $res_array['version'] . "\n";
        #               print "Upstream Ver=> " . $uscan_res['uversion'] . "\n";
        #print "Watch warings=> " . $uscan_res['warnings']  . "\n";
        #       }
    }
    print "\n";
    pg_close($db);
}
function check_url ($url) {
    $url_p = parse_url ($url);
    if (isset ($url_p["host"])) $host = $url_p["host"];
    else return false;
    preg_match("/(http|ftp)/",$host,$matches);
    if ($matches[1]=='http') $fp = fsockopen ($host, 80, $errno, $errstr, 5);
    elseif ($matches[1]=='ftp') $fp2 = fsockopen ($host, 21, $errno, $errstr, 5);
    if (!$fp AND !$fp2) return false;
    else return true;
}
function keep_remote ($localfile,$remotefile) {
    if (!check_url($remotefile)) return false;
    #   $rfp = @fopen ($remotefile, "r");
    #   if (!$rfp) {
    #           return false;
    #           }
    #     fclose($rfp);
    #   $lfp = fopen ($localfile, "w");
    #   stream_set_blocking($rfp,0);
    #   ini_set ( "memory_limit", "100M");
    #   while (!feof($rfp)) {
    #           $data = fread($rfp, 8192);
    #           $bytes+=8192;
    #           print "Downlaoding $remotefile: $bytes\r";
    #           fwrite($lfp, $data);
    #   } while (true);
    #   ini_set ( "memory_limit", "15M");

    if (!copy($remotefile, $localfile)) {
        echo "Copia di $file non riuscita ...\n";
        return false;
    }
    #   fclose($lfp);
    #   fclose($rfp);

    return true;
}
function up_changes($src_name='%', $force = false) {
    global $dbconn, $fetchUpstreamChanges;

    if (!$fetchUpstreamChanges && !$force) {
        //print "\nSkipping download of upstream sources to grab changelogs\n";
        return;
    }

    $changes_files=array("CHANGES","NEWS","Changelog","ChangeLog","changelog");
    $db = pg_pconnect($dbconn) ;
    $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!=''  AND keep_changes='t' AND name LIKE '$src_name'");
    $tot=pg_numrows($rsql);
    while ($res_array=pg_fetch_array($rsql)) {
        $localfile=tempnam("/tmp/", "dehs-" . $res_array[id] . ".tar.gz");
        print "I'm downloading $res_array[up_url]\r";
        $res=keep_remote($localfile,$res_array['up_url']);
        print "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]";
        if (!$res) {
            pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");
            $notdown++;
            $tot--;
            print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
            #print "\n Not Downlaoded: $res_array[up_url]\n";
            unlink($localfile);
            continue;
        }
        foreach($changes_files as $filename) {
            $command="tar --wildcards -zxOf " . $localfile . " *$filename 2>/dev/null";
            exec($command,$up_changes,$cmdres);
            $up_changes=rtrim(implode("\n",$up_changes));
            if ($cmdres==0 AND strlen($up_changes)>20) {
                $sw=true;
                $conv_res=iconv("ISO-8859-1","UTF-8",$up_changes);
                if ($conv_res) $up_changes=$conv_res;
                $up_changes=pg_escape_string($up_changes);
                $up_changes_sql="UPDATE pkgs SET up_changes='$up_changes', keep_changes='f' WHERE id=$res_array[id]";
                pg_exec($db,$up_changes_sql);
                #print "\nFound $filename on $res_array[up_url]\n";
                $found++;
                break;
            }
        }
        if (!$sw) {
            pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");
            #print "\n Not Found any on $res_array[up_url]\n";
            $notfound++;
        }
        unlink($localfile);
        print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
    }
    pg_close($db);
}
function uscan_foo($name,$version,$watch){
    global $base, $uscan;

    $tmpnam=tempnam('/tmp/', $name.'_watch');
    $fw=fopen($tmpnam,"w");
    fwrite($fw,$watch);
    fclose($fw);

    $uscan_res=shell_exec("$uscan --dehs --package $name --upstream-version " . escapeshellarg(vers_conv($version)) . " --watchfile $tmpnam 2>/dev/null 2>&1");

    unlink($tmpnam);

    $p = xml_parser_create();
    xml_parse_into_struct($p, $uscan_res, $vals, $index);
    xml_parser_free($p);

    $upstream=$vals[$index['UPSTREAM-VERSION'][0]][value];
    $up_url=$vals[$index['UPSTREAM-URL'][0]][value];
    $deb_vers=$vals[$index['DEBIAN-UVERSION'][0]][value];
    $deb_mangled_vers=$vals[$index['DEBIAN-MANGLED-UVERSION'][0]][value];
    $warnings=$vals[$index['WARNINGS'][0]][value];
    $updated=$vals[$index['STATUS'][0]][value];

    if ($updated=="up to date")  $updated=1;
    else $updated=0;

    if ($upstream=='') $upstream=null;
    if ($up_url=='')  $up_url=null;
    if ($deb_vers=='') $deb_vers=null;
    if ($warnings=='') $warnings=null;
    if ($deb_mangled_vers=='') $deb_mangled_vers=null;

    $ret = array();
    $ret['uversion']        = $upstream;
    $ret['warnings']        = $warnings;
    $ret['uurl']            = $up_url;
    $ret['dversion']        = $deb_vers;
    $ret['dversionmangled'] = $deb_mangled_vers;
    $ret['updated']         = $updated;

    return $ret;
}
function db_up_error() {
    global $dbconn;
    check_db();
    $db = pg_pconnect($dbconn) ;
    $bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done";
    print "\nDownloading bugs list =>";
    if (!($ferrors=file_get_contents($bugurl))) {
        /*die_status ("I can't keep error page from bugs.debian.org");*/
        fwrite(STDERR, "I can't keep error page from bugs.debian.org, but not aborting");
        return false;
    }
    print "Done\n";
    $ferrors=explode("<li>",$ferrors);
    $totline=count($ferrors);
    #$errors[]=array();
    #array_pop($errors);
    print "Total lines => $totline\n";
    foreach ($ferrors as $line) {
        # print $line . "\n";
        if (preg_match("/Package:.+?>([^<]+)<\/a>.+?<strong>((\d+)\syears?\sand\s|)(\d+)\sdays?/sS",$line,$matches)) {
            $errors[$matches[1]][err_tot]++;
            $errors[$matches[1]][days]+=$matches[3]*365+$matches[4];
            #if ($matches[3]) print "Anni $matches[3]";
            #print "Giorni $matches[4]\n";
        }
        print "\rLines left => " . $totline--;
    }
    #arsort($errors);
    $total=count($errors);
    foreach ($errors as $pkg_name=>$value) {
        pg_exec($db, "UPDATE binpkgs SET up_error=" . $value[err_tot] . ",avg_error_date=" . $value[days]/$value[err_tot] . " WHERE bin_name='$pkg_name'") or die_status("Error in upstream bug query");
        #print "Pkg: $error Errori: $value[err_tot] Average Days: " . $value[days]/$value[err_tot]  . "\n";
        print "\rBugs left => " . $total--;
    }
    pg_close($db);
}
function watch_wizard($src_name='%', $checkStamps = false, $limit = null){
    global $dirs,$dbconn,$sfIgnore,$watchFileExts,$watchFileDversionMangle,$watchFormat,$minimumAge;
    $db = pg_pconnect($dbconn) or die_status ("Db error");
    $checked=$errors=$notfound=$verok=$notmatch=0;

    // prepare some stuff that needs to be done once per run
    $opts       = array();
    $exts       = '';
    $dvmanglers = '';

    foreach ($watchFileExts as $ext) {
        $exts .= $ext . '|';
    }
    unset($ext);

    if ($exts != '') {
        $opts[] = 'uversionmangle=s/\.(' . substr($exts, 0, -1) . ')$//i';
        $exts = '(' . $exts . ')';
    }

    foreach ($watchFileDversionMangle as $dvmangler) {
        $dvmanglers .= $dvmangler . '|';
    }
    unset($dvmangler);

    if ($dvmanglers != '') {
        $opts[] = 'dversionmangle=s/[-.+~]?(' . substr($dvmanglers, 0, -1) . ')(.*)$//i';
    }

    $opts[] = 'pasv';

    if (is_array($opts)) {
        $_opts = '';
        foreach ($opts as $opt) {
            $_opts .= $opt . ',';
        }
        $opts = '';
        if ($_opts != ',') {
            $opts = 'opts=' . substr($_opts, 0, -1) . ' \\';
        }
    }

    $postWHERE = '';
    if ($checkStamps) {
        $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL)"
                   . " AND wwiz_type!='no_cright' AND wwiz_type!='notmatch'";
    }

    $postORDER = '';
    if ($limit !== null && is_int($limit)) {
        $postORDER = 'LIMIT ' . $limit;
    }

    $sqlstring="SELECT name,version,dversionmangled,section,dist, wwiz_version FROM pkgs
        WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name' $postWHERE ORDER BY lastcheck DESC $postORDER";
    $rsql=pg_exec($db, $sqlstring);

    while ($res_array=pg_fetch_array($rsql)) {

        if (substr($res_array["name"],0,3)=='lib')
            $initial=substr($res_array["name"],0,4);
        else
            $initial=substr($res_array["name"],0,1);

        $url="http://packages.debian.org/changelogs/pool/" .
        $res_array["section"] . "/" .
        $initial . "/" .
        $res_array["name"] ."/" .
        $res_array["name"] . "_" .
        $res_array["version"] . "/copyright";
        $contents=@file_get_contents($url) or '';
        $generatedWatches = array();

        if (!$contents) {
            $url="http://packages.debian.org/changelogs/pool/" .
            $res_array["section"] . "/" .
            $initial . "/" .
            $res_array["name"] ."/" .
            $res_array["name"] . "_" .
            $res_array["version"] . "/" . $res_array["name"] . ".copyright";
            $contents=@file_get_contents($url) or '';
        }

        if (!$contents) {
            $url="http://packages.debian.org/changelogs/pool/" .
            $res_array["section"] . "/" .
            $initial . "/" .
            $res_array["name"] ."/" .
            "current/copyright";
            $contents=@file_get_contents($url) or '';
        }

        if ($contents) {

            if (preg_match ('/(sf\.net|sourceforge\.net)/i', $contents, $matches) ||
                (preg_match ('/([\w\-]+)\.(sf\.net|sourceforge\.net)/i', $contents, $matches)
                 && !in_array($matches[1], $sfIgnore))) {
                preg_match("/([^\d]+)/",$res_array['name'],$dmatches);
                $depured_debian_name=$dmatches[1];
                /*$matches[1]="ftp://download.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/";*/
                $generatedWatches[]=array('url'=>"http://sf.net/" . $depured_debian_name . "/");
                $generatedWatches[]=array('url'=>"http://sf.net/" . $depured_debian_name . "/",'name'=>$depured_debian_name);

            }

            if (preg_match ('#((ftp://|http://|https://)\S+(html|htm|/(?(?=.+(tar|tgz|zip|gz|bz2))|[^>\s\)\a]+)))#i',$contents,$matches)) {
                $final=substr($matches[1],-1);
                if ($final=='.') {
                    $matches[1]=substr($matches[1],0,-1);
                    $final=substr($matches[1],-1);
                }
                if ($matches[2]=="http://" || $matches[2]=="https://") {
                    $space=' ' . "(?:.*/)?";
                } elseif ($final!='/') {
                    $matches[1].='/';
                }
                $generatedWatches[]=array('url' => preg_replace("#((http|https|ftp)://.+):(.*)#i",'$1' . '$3', $matches[1],1), 'space' => $space);
            }

            if (preg_match ('#((ftp|http)\.\S+(html|htm|/(?(?=.+(tar|tgz|zip|gz|bz2))|[^>\s\)\a]+)))#i',$contents,$matches)) {
                $final=substr($matches[1],-1);
                if ($final=='.') {
                    $matches[1]=substr($matches[1],0,-1);
                    $final=substr($matches[1],-1);
                }
                $space = '';
                if ($matches[2]=="http") {
                    $space=' ' . "(?:.*/)?";
                } elseif ($final!='/') {
                    $matches[1].='/';
                }
                $generatedWatches[]=array('url'=>preg_replace('#(http|ftp)\.(.+)#i','$1://$0', $matches[1],1), 'space'=>$space);
            }

            if (empty($generatedWatches)) {
                $notmatch++;
                $contents=iconv("ISO-8859-1","UTF-8",$contents);
                pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($contents) . "', wwiz_type='notmatch', up_url='', up_changes='', lastcheck=now(), keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'");

            } else {

                $defs = array('name' => $res_array['name'], 'url' => '', 'space' => '');

                foreach ($generatedWatches as $tWatch) {

                    if (!is_array($tWatch)) {
                        $tWatch = array($tWatch);
                    }

                    $tWatch = array_merge($defs, $tWatch);

                    $watch = sprintf($watchFormat, $tWatch['url'], $tWatch['space'], $tWatch['name'], $exts, $opts);

                    $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch);

                    if ($uscan_res['uversion'] && levenshtein($uscan_res['uversion'], $res_array['version']) < strlen($res_array['version'])) {
                        break;
                    }
                }

                if ($uscan_res['warnings'] && !$uscan_res['uversion']) {
                    pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($uscan_res['warnings']) . "', wwiz_type='error', up_url='', up_changes='', lastcheck=now(), keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'");
                    $errors++;
                } else {
                    $verok++;
                    if ($uscan_res['uversion'] != $res_array['wwiz_version']) {
                        $keep_changes = 1;
                    } else {
                        $keep_changes = 0;
                    }

                    $dversionmangled = $res_array[version];
                    if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) $dversionmangled=$uscan_res['dversionmangled'];

                    $updated = $uscan_res['updated'];
                    /*if ($updated)*/ {
                        $updated = is_updated($uscan_res['uversion'], $dversionmangled, $dversionmangled != $uscan_res['dversionmangled']);
                    }

                    /*if (!$updated) {
                        events::newVersion($res_array['name'], $res_array['version'], $uscan_res['uversion'], $res_array['dist']);
                    }*/

                    pg_exec($db, "UPDATE pkgs SET wwiz='" . pg_escape_string($watch) ."', wwiz_type='watch', wwiz_version='" . pg_escape_string($uscan_res['uversion']) . "',dversionmangled='".pg_escape_string($dversionmangled)."', up_url='" . pg_escape_string($uscan_res['uurl']) . "', keep_changes='$keep_changes', updated='$updated', lastcheck=now() where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'");
                }
            }
        } else {
            $notfound++;
            pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz_type='no_cright' where name='" . pg_escape_string($res_array['name']) . "' AND dist='" . $res_array['dist'] ."'");
        }
        print "Package checked: " . $checked++ .
        " OK: $verok Error: $errors Not Matched: $notmatch\r";
    }
    print "\nCopyright files not found: $notfound\n";
    print "Automatic generated watch file failures: $errors\n";
    pg_close($db);
}
function dbqa_conn ($dbname,$mode) {
    $id = dba_open($dbname, $mode, "db4");
    if (!$id) {
        die_status("dba_open failed\n");
    }
    return $id;
}
function dehsqa_db () {
    global $dirs,$dbconn;
    $dir1='/org/alioth.debian.org/chroot/home/groups/dehs/htdocs/';
    $dir2="/org/qa.debian.org/data/dehs/";
    $dists=array("unstable","experimental");
    foreach ($dists as $dist) {
        $id=dbqa_conn($dir1 . "dehs_qa_" . $dist . ".db",'n');
        $db = pg_pconnect($dbconn) or die_status ("Db error");
        $sql="SELECT name, id, up_version, wwiz_version, watch, wwiz_type, watch_warn, version, dversionmangled, updated FROM pkgs WHERE dist='$dist'";
        $rsql=pg_exec($db, $sql);
        while($res_array=pg_fetch_array($rsql)) {

            if (!$res_array[watch]) { if ($res_array['wwiz_type']==null) $version="N/A"; else $version='-';}
            elseif (!$res_array[up_version]) $version="Error";
            else $version=$res_array[up_version];

            if ($res_array['wwiz_type']=="watch") $wwiz=$res_array['wwiz_version'];
            else $wwiz=$res_array[wwiz_type];

            // the updated flag is set to true even when up_version could not be determined
            $updated = ($res_array['updated'] && ($res_array[up_version] || $res_array[wwiz_version]))? 'yes' : 'no';

            $xml="<data><id>$res_array[id]</id>" .
            "<up_version>$version</up_version>" .
            "<dversion>$res_array[version]</dversion>" .
            "<dversionmangled>$res_array[dversionmangled]</dversionmangled>" .
            "<uptodate>$updated</uptodate>" .
            "<wwiz>$wwiz</wwiz>" .
            "</data>";
            $value=addslashes($xml);
            dba_replace($res_array[name],$xml,$id);
        }
        dba_optimize($id);
        dba_close($id);
        pg_close($db);
    }
}
function md5_of_file($inFile) {
    if (file_exists($inFile)) {
        return md5_file($inFile);
    } else {
        return false;
    }
}
function update_all() {
    global $email, $timeStamp_basedCheck;
    $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
    echo __FUNCTION__ . ' started the ' . $intime . "\n";
    db_add();
    dl_diffs();
    dl_from_vcs('%','%',in_array('vcs',$timeStamp_basedCheck));
    db_up_error();
    db_upstream('%', in_array('all', $timeStamp_basedCheck));
    watch_wizard('%', in_array('all', $timeStamp_basedCheck));
    up_changes();
    dehsqa_db();
    $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
    $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";
    mail($email, "DEHS report", $messaggio,
    "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
}
function update_new() {
    global $email_all, $timeStamp_basedCheck;
    $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
    echo __FUNCTION__ . ' started the ' . $intime . "\n";
    db_add();

    global $dbconn;
    check_db();
    $db = pg_pconnect($dbconn);
    $sql = "SELECT name FROM pkgs WHERE"
          ." (md5_diff!=md5_atsource or md5_diff is null)"
          ." or (lastcheck is null and watch is not null)"
          ." or (lastpkgsourcesupdate>lastcheck)";
    $rsql=pg_exec($db, $sql);
    $res_array=pg_fetch_all($rsql);
    pg_close($db);

    dl_diffs();
    db_up_error();
    if ($res_array !== false) {
        foreach ($res_array as $entry) {
            dl_from_vcs($entry['name'],'%',in_array('vcs',$timeStamp_basedCheck));
            db_upstream($entry['name']);
            up_changes($entry['name']);
            watch_wizard($entry['name']);
        }
        dehsqa_db();
    }
    $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
    $messaggio="Dehs executed successful\nBegin: $intime\nPackages processed: ".((!is_array($res_array))?'none':count($res_array))."\nEnd: $endtime";
    mail($email_all, "DEHS update_new report", $messaggio,
    "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
}
function update_bogus() {
    global $email_all,$dbconn;
    $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
    echo __FUNCTION__ . ' started the ' . $intime . "\n";
    check_db();
    $db = pg_pconnect($dbconn);
    $sql="SELECT name FROM pkgs WHERE up_version='' AND watch!=''";
    $rsql=pg_exec($db, $sql);
    $res_array=pg_fetch_all($rsql);
    $orig_count = ((!is_array($res_array))?0:count($res_array));
    if ($res_array !== false) {
        foreach ($res_array as $entry) {
            db_upstream($entry['name']);
            up_changes($entry['name']);
        }
        dehsqa_db();
    }
    $sql="SELECT COUNT(name) FROM pkgs WHERE up_version='' AND watch!=''";
    $rsql=pg_exec($db, $sql);
    $res_array=pg_fetch_array($rsql);
    $new_count = (int)$res_array[0];
    pg_close($db);
    $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
    $messaggio="Dehs executed successful\nBegin: $intime\n".
    "Packages processed: $orig_count\n" .
    "New bogus watches count: $new_count\nEnd: $endtime";
    mail($email_all, "DEHS update_bogus report", $messaggio,
    "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
}
function die_status($msg = '', $status = 1) {
    if (empty($msg)) {
        die($status);
    } else if (is_int($msg) && $status == 1 /* default */) {
        die($msg);
    } else {
        fwrite(STDERR, $msg);
        die($status);
    }
}

function sqldate($timestamp=null) {
        if ($timestamp===null) $timestamp=time();
        return gmdate("Y-m-d H:i:s", $timestamp);
}

function dl_from_vcs($package = '%', $dist = '%', $checkStamps = false, $db = null) {
    global $minimumVCSAge;
    $local_db = false;

    if (!$fetchWatchFromVCS) {
        return false;
    }

    if ($db === null) {
        $local_db = true;
        $db = pg_pconnect($dbconn)  or  die_status(pg_last_error($db));
    }

    $sql="SELECT name,dist,vcs_type,vcs FROM pkgs WHERE name LIKE '$package' AND "
    . "dist LIKE '$dist' AND vcs!='' AND vcs IS NOT NULL AND vcs_type!='' AND vcs_type IS NOT NULL "
    . ($checkStamps? "AND (lastvcsfetch IS NULL OR lastvcsfetch <='"
                    . (sqldate(time() - ($minimumVCSAge*24*3600)))."') "
                    : '');
    $rsql=pg_exec($db, $sql);
    while ($res = pg_fetch_array($rsql)) {
        // avoid miss-parsed entries:
        if (   count(explode('\n',$res['vcs_type'])) > 1
            || count(explode('\n',$res['vcs'])) > 1)
            continue;

        print "Fetching watch file of $res[name] (type: $res[vcs_type]) from $[vcs]\n";

        $tmp_dir = shell_exec('mktemp -d') or die_status("mktemp failed to create directory!");
        $vcs_watch = null;
        if (exec('debcheckout.pl -t ' . escapeshellarg($res['vcs_type'])
                             . ' -f debian/watch '
                             . escapeshellarg($res['vcs'])
                             . ' ' . escapeshellarg($tmp_dir))) {
            $vcs_watch = implode('\n', file($tmp_dir . '/debian/watch'));
        }

        shell_exec('rm -rf ' . escapeshellarg($tmp_dir));

        $dehs_opts = array();
        $dehs_opts['vcs'] = 0;

        if (preg_match('/#\s*DEHS\s*:\s*([\w\s,]+)/i', $vcs_watch, $matches)) {
            $_dehs_opts = array_map('trim',explode(',', $matches[1]));
            switch ($_dehs_opts) {
                case 'vcs':
                    $dehs_opts['vcs'] = 1;
                    break;
                case 'novcs':
                    $dehs_opts['vcs'] = 0;
                    break;
            }
        }

        if ($vcs_watch != null) {
            $vcs_watch = iconv('ISO-8859-1', 'UTF-8', $vcs_watch);
            $vcs_watch = pg_escape_string($vcs_watch);
        }

        pg_exec($db, "UPDATE pkgs SET vcs_watch='" . $vcs_watch . "', lastvcsfetch = now(), vcsoversource='$dehs_opts[vcs]' WHERE name='$res[name]' AND dist='$res[dist]'") or die_status("Query error");
    }

    if ($local_db) pg_close($db);
    return true;
}

?>