--- trunk/dehs_pg.php 2007/03/28 23:54:07 43 +++ trunk/dehs_pg.php 2007/12/16 01:53:08 77 @@ -1,24 +1,26 @@ #!/usr/bin/php4 -q - - Copyright 2004, Stefano Fabri - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Originally written by Stefano Fabri +Copyright 2004, Stefano Fabri +Copyright 2007 by Raphael Geissert + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + // stabilire una connessione #ini_set ( "error_reporting", "E_COMPILE_ERROR|E_ERROR|E_CORE_ERROR"); ini_set ("html_errors", "off"); @@ -28,767 +30,1016 @@ require "dehs.config.inc"; dir_ctrl($dirs); if (!function_exists('file_get_contents')) { - function file_get_contents($filename) { - if (!($file = fopen($filename,"r"))) return false; - else { - $contents = ""; - do { - $data = fread($file, 4096); - if (strlen($data) == 0) { - break; - } - $contents .= $data; - } while (true); - fclose($file); - return $contents; - } - } -} -if (in_array("-db_add",$argv)) db_add($argv[2]); - elseif (in_array("-db_query",$argv)) db_query($argv[2],$argv[3] ); - elseif (in_array("-keep_diff",$argv)) keep_diff($argv[2]); - elseif (in_array("-keep_sources",$argv)) keep_sources(); - elseif (in_array("-db_popcon",$argv)) db_popcon($argv[2]); - elseif (in_array("-keep_popcon",$argv)) keep_popcon($argv[2]); - elseif (in_array("-db_upstream",$argv)) db_upstream($argv[2]); - elseif (in_array("-update_all",$argv)) update_all(); - elseif (in_array("-check_db",$argv)) check_db(); - elseif (in_array("-dehsqa_db",$argv)) dehsqa_db(); - elseif (in_array("-clear_db",$argv)) clear_db(); - elseif (in_array("-db_up_error",$argv)) db_up_error(); - elseif (in_array("-up_changes",$argv)) up_changes($argv[2]); - elseif (in_array("-watchf",$argv)) watch_fill($argv[2]); - elseif (in_array("-kb_total",$argv)) print "Diff + function file_get_contents($filename) { + if (!($file = fopen($filename,"rb"))) return false; + else { + $contents = ""; + do { + $data = fread($file, 4096); + if (strlen($data) == 0) { + break; + } + $contents .= $data; + } while (true); + fclose($file); + return $contents; + } + } +} + +if (!function_exists('file_put_contents')) { + function file_put_contents($filename, $content, $flags = 0) { + if (!($file = fopen($filename,$flags))) return false; + else { + $n = fwrite($file, $content); + fclose($file); + return true; + } + } +} + +if (in_array("-db_add",$argv)) db_add((!empty($argv[2])?$argv[2]:''), false); + elseif (in_array("-db_query",$argv)) db_query($argv[2],(!empty($argv[3])?$argv[3]:'%') ); + elseif (in_array("-dl_diffs",$argv) || in_array("-keep_diff",$argv)) dl_diffs($argv[2]); + elseif (in_array("-keep_sources",$argv) || in_array("-download_sources",$argv)) download_sources(); + elseif (in_array("-db_popcon",$argv)) db_popcon($argv[2]); + elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv)) dl_popcon($argv[2]); + elseif (in_array("-db_upstream",$argv)) db_upstream($argv[2]); + elseif (in_array("-update_all",$argv)) update_all(); + elseif (in_array("-update_new",$argv)) update_new(); + elseif (in_array("-update_bogus",$argv)) update_bogus(); + elseif (in_array("-check_db",$argv)) check_db(); + elseif (in_array("-dehsqa_db",$argv)) dehsqa_db(); + elseif (in_array("-clear_db",$argv)) clear_db(); + elseif (in_array("-db_up_error",$argv)) db_up_error(); + elseif (in_array("-up_changes",$argv)) up_changes($argv[2], true); + elseif (in_array("-watchf",$argv) || in_array("-wwiz",$argv)) watch_wizard((!empty($argv[2])?$argv[2]:'')); + elseif (in_array("-kb_total",$argv)) print "Diff #stat_up_error(); #stat_up_error();to download Kb => " . kb_total($argv[2]) . "\n" ; - elseif (in_array("-watch_stat",$argv)) watch_stat(); - else print "Command $argv[1] not found\n"; + elseif (in_array("-watch_stat",$argv)) watch_stat(); + else print "Command $argv[1] not found\n"; #stat_up_error(); function dir_ctrl($dirs) { - foreach ($dirs as $dir) { - if (!is_dir($dir)) exec("mkdir -m 0740 -p $dir",$output,$res); - if ($res!=0) die("Error creating basic dir: $dir\n"); - } + foreach ($dirs as $dir) { + if (!is_dir($dir)) exec("mkdir -m 0740 -p " . escapeshellarg($dir), $output, $res); + if ($res != 0) die_status("Error creating basic dir: $dir\n"); + } } -function ext_watch ($file,$pkg,$version){ - if (!file_exists($file)) return ""; - $zp=gzopen($file,"r") or die("Fail to open $file"); - while (!gzeof($zp)) { - $line = gzgets ($zp,4096); - if (preg_match("/\+\+\+\s.+\/debian\/watch/",$line)) { - $line = gzgets ($zp,4096); - preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght); - for ($i = 1; $i <= $watch_lenght[1]; $i++) { - $line=gzgets ($zp,4096); - $watch .=substr($line,1); - } - break; +function ext_watch ($file,$pkg,$version) { + if (!file_exists($file)) return ""; + $zp=gzopen($file,"r") or die_status("Fail to open $file"); + while (!gzeof($zp)) { + $line = gzgets ($zp,4096); + if (preg_match('/\+\+\+\s.+\/debian\/watch$/',$line)) { + $line = gzgets ($zp,4096); + preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght); + for ($i = 1; $i <= $watch_lenght[1]; $i++) { + $line=gzgets ($zp,4096); + $watch .=substr($line,1); } + break; } -if ($watch!=null) return $watch; -else return NULL; + } + if ($watch!=null) return $watch; + else return NULL; } function ftp_cn($login,$password,$ftp_server) { -$conn_id = @ftp_connect($ftp_server) or DIE("Error: Cannot contact ftp server $ftp_server\n"); -// login con user name e password -ftp_pasv($conn_id,1); -$login_result = ftp_login($conn_id, "anonymous","bo@bo.org"); -// controllo della connessione -if ((!$conn_id) || (!$login_result)) { + $conn_id = @ftp_connect($ftp_server) or die_status("Error: Cannot contact ftp server $ftp_server\n"); + // login con user name e password + ftp_pasv($conn_id,1); + $login_result = ftp_login($conn_id, "anonymous","bo@bo.org"); + // controllo della connessione + if ((!$conn_id) || (!$login_result)) { echo "Ftp connection failed!\n"; echo "Failed connection to $ftp_server\n"; - die(); + die_status(); } else { return $conn_id; } -} -function keep_diff($initial) { - global $dirs,$dbconn,$mirrors; - check_db(); - $conn_normal=ftp_cn("anonymous","bo@bo.org",$mirrors['unstable']); - #$conn_non_us=ftp_cn("anonymous","bo@bo.org",$mirrors["non-US"]); - $db = pg_pconnect($dbconn) ; - $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR DIE("Error in diff query\n"); - $total=pg_numrows($rsql); - $btotal=byte_total($db,$initial); - print "Total Diff to downlaod Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n"; - while ($res_array=pg_fetch_array($rsql)) { - #if ($res_array['dist']=='non-US') $conn_id = $conn_non_us; - #else - $conn_id = $conn_normal; - if ($localfile=download_diffs($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) { - $watch=ext_watch($localfile,$res_array[name],$res_array[version]); - unlink($localfile); - if ($watch!=NULL) { - $watch=iconv("ISO-8859-1","UTF-8",$watch); - $watch= addslashes($watch); - pg_exec($db, "UPDATE pkgs SET watch='" . $watch . "', md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Query error"); - } - else pg_exec($db, "UPDATE pkgs SET watch=NULL, md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Errore nella query"); - } - $btotal=$btotal-$res_array['bytes']; - number_format ($btotal/1024, 2, ",","."); - print "\rDiffs left => " . $total-- . " Bytes left => " . number_format ($btotal/1024, 2, ",",".") . " - " . $res_array[name]; - } - ftp_quit($conn_normal); - pg_close($db); - return "Fatto\n"; +} +function dl_diffs($initial = '') { + global $dirs,$dbconn,$mirrors; + check_db(); + $conn_normal=ftp_cn("anonymous","bo@bo.org",$mirrors['unstable']); + #$conn_non_us=ftp_cn("anonymous","bo@bo.org",$mirrors["non-US"]); + $db = pg_pconnect($dbconn) ; + $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR die_status("Error in diff query\n"); + $total=pg_numrows($rsql); + $btotal=byte_total($db,$initial); + print "Total Diff to downlaod Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n"; + while ($res_array=pg_fetch_array($rsql)) { + #if ($res_array['dist']=='non-US') $conn_id = $conn_non_us; + #else + $conn_id = $conn_normal; + if ($localfile=download_diffs($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) { + $watch=ext_watch($localfile,$res_array[name],$res_array[version]); + unlink($localfile); + if ($watch!=NULL) { + $watch=iconv("ISO-8859-1","UTF-8",$watch); + $watch= pg_escape_string($watch); + pg_exec($db, "UPDATE pkgs SET watch='" . $watch . "', md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or die_status("Query error"); + } + else pg_exec($db, "UPDATE pkgs SET watch=NULL, md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or die_status("Errore nella query"); + } + $btotal=$btotal-$res_array['bytes']; + number_format ($btotal/1024, 2, ",","."); + print "\rDiffs left => " . $total-- . " Bytes left => " . number_format ($btotal/1024, 2, ",",".") . " - " . $res_array[name]; + } + ftp_quit($conn_normal); + pg_close($db); + return "Fatto\n"; } function download_diffs(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) { -global $dirs,$mirrors; - $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz"; - #if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz"; - #else - $remotefile="/debian/$directory/$pkg" . "_" . "$version.diff.gz"; - #print "$localfile - $remotefile\n"; - if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) { - # print "$pkg Diff.gz => Downloaded\n"; - if(md5_of_file($localfile)==$md5_atsource) { - #print "MD5 => OK\n"; - #unlink($localfile); - return $localfile; - } - } - print "Download Diff.gz of package $pkg => Fallito\n"; - return false; -} -function db_add($initial=''){ -global $dirs,$dists,$dbconn; -check_db(); -keep_source(); -$db = pg_connect($dbconn) or die(pg_errormessage($db)); -pg_exec($db, "CREATE TEMP TABLE pkgs_atsrc (name text,dist text)") or die('Error creating temp table pkgs_atsrc'); -pg_exec($db, "CREATE UNIQUE INDEX idxdis on pkgs_atsrc (name,dist)") or die('Error creating index on temp table'); -pg_exec($db, "CREATE TEMP TABLE bin_atsrc (name text,bin_name text, dist text)") or die('Error creating temp table pkgs_atsrc'); -pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die('Error creating index on temp table bin_atsrc'); - foreach ($dists as $dist=>$sections) { + global $dirs,$mirrors; + $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz"; + #if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz"; + #else + $remotefile="/debian/$directory/$pkg" . "_" . "$version.diff.gz"; + #print "$localfile - $remotefile\n"; + if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) { + # print "$pkg Diff.gz => Downloaded\n"; + if(md5_of_file($localfile)==$md5_atsource) { + #print "MD5 => OK\n"; + #unlink($localfile); + return $localfile; + } + } + print "Download Diff.gz of package $pkg => Fallito\n"; + return false; +} +function db_add($initial='', $clear_db = true) { + global $dirs,$dists,$dbconn; + check_db(); + download_sources(); + + $db = pg_pconnect($dbconn) or die_status(pg_last_error($db)); + pg_exec($db, "CREATE TEMP TABLE pkgs_atsrc (name text,dist text)") or die_status('Error creating temp table pkgs_atsrc'); + pg_exec($db, "CREATE UNIQUE INDEX idxdis on pkgs_atsrc (name,dist)") or die_status('Error creating index on temp table'); + pg_exec($db, "CREATE TEMP TABLE bin_atsrc (name text,bin_name text, dist text)") or die_status('Error creating temp table pkgs_atsrc'); + pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die_status('Error creating index on temp table bin_atsrc'); + + foreach ($dists as $dist=>$sections) { foreach ($sections as $section) { - $filename=$dirs[sources_dir] . "/$dist/$section/Sources.gz"; - print "\nI'm parsing $dist/$section => Sources.gz\n"; - # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5 - $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S+)\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?/Ssi"; - $zp = gzopen($filename, "r") or die("Could not open $filename"); - $extracted=tempnam("/tmp/", "$dist_$section_Sources"); - exec("gzip -c -d $filename > $extracted"); - $sourcesize=filesize($extracted); - unlink($extracted); - if(filesize($filename)>20) { - while (!gzeof($zp)) { - $line=''; - while ($line!="\n") { - $line = gzgets ($zp,4096) ; - $buff1 .= $line; + $filename=$dirs['sources_dir'] . "/$dist/$section/Sources.gz"; + print "\nI'm parsing $dist/$section => Sources.gz\n"; + # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5 + $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S+)\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?/Ssi"; + $zp = gzopen($filename, "r") or die_status("Could not open $filename"); + $extracted=tempnam("/tmp/", "$dist_$section_Sources"); + exec("gzip -c -d $filename > $extracted"); + $sourcesize=filesize($extracted); + unlink($extracted); + if(filesize($filename)>20) { + while (!gzeof($zp)) { + $line=''; + while ($line!="\n") { + $line = gzgets ($zp,4096) ; + $buff1 .= $line; + } + if (preg_match($regexp,$buff1,$matches)) { + if ($matches[5]) { + $matches[5]=iconv("ISO-8859-1","UTF-8",$matches[5]); + $matches[5]=pg_escape_string($matches[5]); + } + if ($matches[10]) { + $matches[10]=iconv("ISO-8859-1","UTF-8",$matches[10]); + $matches[10]=pg_escape_string($matches[10]); + } + @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR die_status("Temp table pkgs_atsrc query error"); + $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]')") ; + if (!$rst) { + $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]',dversionmangled='$matches[4]',maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]' WHERE name='$matches[1]' AND dist='$dist'") OR die_status("\nDb adding error =>" . pg_last_error() . "\n"); } - if (preg_match($regexp,$buff1,$matches)) { - if ($matches[10]) { - $matches[10]=iconv("ISO-8859-1","UTF-8",$matches[10]); - $matches[10]=addslashes($matches[10]); - } - @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR DIE("Temp table pkgs_atsrc query error"); - $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,maint,dir,md5_atsource,bytes,dist,section,uploaders) VALUES ('$matches[1]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]')") ; - if (!$rst) { - $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]',maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]' WHERE name='$matches[1]' AND dist='$dist'") OR DIE ("\nDb adding error =>" . pg_errormessage() . "\n"); - } - $bin_names=split(",", $matches[2]); - foreach ($bin_names as $bin_name) { - pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR DIE("Temp table pkgs_atsrc query error"); - $rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ; - } - } - else { - preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches); - if(strncmp($initial,$matches[1],strlen($initial))<0) break; - } - $left=$sourcesize-gztell($zp); - //print "\rSource file Left => $left"; - $buff1=""; - unset($matches); - } + $bin_names=split(",", $matches[2]); + foreach ($bin_names as $bin_name) { + pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR die_status("Temp table pkgs_atsrc query error"); + $rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ; + } + } + else { + preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches); + if(strncmp($initial,$matches[1],strlen($initial))<0) break; + } + $left=$sourcesize-gztell($zp); + print "\rSource file Left => $left"; + $buff1=""; + unset($matches); + } } gzclose($zp) ; } - } - clear_db($db); - pg_close($db); -} -function db_query($pkg,$dist) { - global $dirs,$dbconn; - $db = pg_pconnect($dbconn ) or die(pg_errormessage($db)); - $rsql=pg_exec($db, "SELECT pkgs.* ,mpop_inst FROM (SELECT name,MAX(binpkgs.pop_inst) AS mpop_inst FROM binpkgs GROUP BY name) - AS binpkgs INNER JOIN pkgs ON pkgs.name=binpkgs.name - WHERE pkgs.name='$pkg' AND dist='$dist';"); - if (pg_numrows($rsql)==1) { - $res_array=pg_fetch_array($rsql); - print "Package=> " . $res_array['name'] . "\n"; - print "Version => " . $res_array['version'] . "\n"; - print "Diff MD5 Downloaded => " . $res_array['md5_diff'] . "\n"; - print "Diff MD5 at Sources => " . $res_array['md5_atsource'] . "\n"; - print "Distribution => " . $res_array['dist'] . "\n"; - print "Section => " . $res_array['section'] . "\n"; - print "KBytes => " . number_format ( $res_array['bytes']/1024, 2, ",",".") . "\n"; - print "Pop Inst=>" . $res_array['mpop_inst'] . "\n"; + } + if ($clear_db) + clear_db($db); + pg_close($db); +} +function db_query($pkg,$dist='%') { + global $dirs,$dbconn; + $db = pg_pconnect($dbconn ) or die_status(pg_last_error($db)); + $rsql=pg_exec($db, "SELECT pkgs.* ,mpop_inst FROM (SELECT name,MAX(binpkgs.pop_inst) AS mpop_inst FROM binpkgs GROUP BY name) + AS binpkgs INNER JOIN pkgs ON pkgs.name=binpkgs.name + WHERE pkgs.name='$pkg' AND dist='$dist';"); + while ($res_array=pg_fetch_array($rsql)) { + print "Package=> " . $res_array['name'] . "\n"; + print "Version => " . $res_array['version'] . "\n"; + print "Mangled version => " . $res_array['dversionmangled'] . "\n"; + print "Diff MD5 Downloaded => " . $res_array['md5_diff'] . "\n"; + print "Diff MD5 at Sources => " . $res_array['md5_atsource'] . "\n"; + print "Distribution => " . $res_array['dist'] . "\n"; + print "Section => " . $res_array['section'] . "\n"; + print "KBytes => " . number_format ( $res_array['bytes']/1024, 2, ",",".") . "\n"; + print "Pop Inst=>" . $res_array['mpop_inst'] . "\n"; if ($res_array['watch']) { - $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['watch']); - print "Version Upstream => " . $uscan_res[0] . "\n"; - print "Watch => " . $res_array['watch']; - print "Watch Warings => " . $uscan_res[1]; + $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['watch']); + print "Version Upstream => " . $uscan_res[0] . "\n"; + print "Watch => " . trim($res_array['watch'])."\n"; + print "Watch Warnings => " . $uscan_res[1]; } else print "Watch => Not present" . "\n"; + if ($res_array['wwiz_type']) { + $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['wwiz']); + print "WWiz Upstream => " . $uscan_res[0] . "\n"; + print "WWiz => " . $res_array['wwiz']."\n"; + print "WWiz Warings => " . $uscan_res[1]; } - else echo "Package=> " . $pkg . "\nNot found\n!!"; - pg_close($db); + print "\n"; + } + if (pg_numrows($rsql)==0) echo "Package=> " . $pkg . "\nNot found\n!!"; + pg_close($db); } -function byte_total(&$db,$initial){ -global $dirs; -$rsql=pg_exec ($db,"Select sum(bytes) as totbytes FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)"); -$res_array=pg_fetch_array($rsql); -return $res_array[totbytes] ; -} -function keep_source (){ - global $dirs,$dists,$mirrors; - foreach ($dists as $dist=>$sections) { - $server=$mirrors[$dist]; - $conn_id = ftp_cn("anonymous","bo@bo.org",$server); - foreach ($sections as $section) { - $localfile=$dirs[sources_dir] . "/$dist/$section/Sources.gz"; - $remotefile=remote_dist_dir($dist) . "/$dist/$section/source/Sources.gz"; - $sources_info=md5_source($section,$dist); - if (!file_exists ($dirs[sources_dir] . "/$dist/")) mkdir ( $dirs[sources_dir] . "/$dist",0740); - if (!file_exists ($dirs[sources_dir] . "/$dist/$section")) mkdir ($dirs[sources_dir] . "/$dist/$section",0740); - if (!file_exists($localfile) OR md5_of_file($localfile)!=$sources_info[md5]) { - print "Downloading $remotefile Bytes $sources_info[bytes]=> "; - if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) { - print "Done\n"; - if(md5_of_file($localfile)==$sources_info[md5]) print "MD5 => OK\n"; - } - else { - unlink("$localfile"); - print "Fail\n"; - die ("Error in source file"); - } - } +function byte_total(&$db,$initial) { + global $dirs; + $rsql=pg_exec ($db,"Select sum(bytes) as totbytes FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)"); + $res_array=pg_fetch_array($rsql); + return $res_array[totbytes] ; +} +function download_sources () { + global $dirs,$dists,$mirrors; + foreach ($dists as $dist=>$sections) { + $server=$mirrors[$dist]; + $conn_id = ftp_cn("anonymous","bo@bo.org",$server); + foreach ($sections as $section) { + $localfile=$dirs[sources_dir] . "/$dist/$section/Sources.gz"; + $remotefile=remote_dist_dir($dist) . "/$dist/$section/source/Sources.gz"; + $sources_info=md5_source($section,$dist); + if (!file_exists ($dirs[sources_dir] . "/$dist/")) mkdir ( $dirs[sources_dir] . "/$dist",0740); + if (!file_exists ($dirs[sources_dir] . "/$dist/$section")) mkdir ($dirs[sources_dir] . "/$dist/$section",0740); + if (!file_exists($localfile) OR md5_of_file($localfile)!=$sources_info[md5]) { + print "Downloading $remotefile Bytes $sources_info[bytes]=> "; + if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) { + print "Done\n"; + if(md5_of_file($localfile)==$sources_info[md5]) print "MD5 => OK\n"; + else print "MD5 => no match\n"; + } + else { + unlink($localfile); + print "Fail\n"; + die_status ("Error in source file"); + } + } } - ftp_quit($conn_id); - } + ftp_quit($conn_id); + } } function md5_source($section,$dist) { -global $mirrors; -$relase=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die("Failed to check MD5 of sources"); -preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $relase,$matches); -return array("md5"=>$matches[1],"bytes"=>$matches[2]); - # 183dea9ca3c2e35ffd6cf5eee9501c21 2941637 main/binary-i386/Packages.gz + global $mirrors; + $relase=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die_status("Failed to check MD5 of sources"); + preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $relase,$matches); + return array("md5"=>$matches[1],"bytes"=>$matches[2]); + # 183dea9ca3c2e35ffd6cf5eee9501c21 2941637 main/binary-i386/Packages.gz } - + function vers_conv($debvers) { - preg_match("/(.+:)?([^-]+)(ds|dsfg|debian)/",$debvers,$matches); - if (!$matches[3]) { - unset($matches); - preg_match("/(.+:)?([^-]+)/",$debvers,$matches); - } - if (substr($matches[2],-1)=='.') $matches[2]=substr($matches[2],0,-1); - return $matches[2]; + // Strip off the epoch + $pos = strpos($debvers, ':'); + if ( $pos !== FALSE) { + $debvers = substr($debvers, $pos+1); + } + + // strip off the Debian revision (look from the back of the string) + $pos = strrpos($debvers, '-'); + if ( $pos !== FALSE) { + $debvers = substr($debvers, 0, $pos); + } + + // strip off repacking indicators + $debvers = preg_replace("/[-.+~]?(ds|dfsg|debian)(.*)/i", "", $debvers); + return $debvers; } function check_db() { -global $dirs,$dbconn; -$db = @pg_connect($dbconn) or die(pg_last_notice()); - if (!pg_table_exists($db,"pkgs")){ - pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text)") or die('Error creating table pkgs\n'); - pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die('Errore creating index - ' . pg_last_error($db)); - $sw=true; - } - if (!pg_table_exists($db,"binpkgs")){ - pg_exec($db, "CREATE TABLE binpkgs (id serial PRIMARY KEY, name text,bin_name text,dist text, pop_inst numeric,pop_vote numeric, up_error numeric,avg_error_date numeric)") or die('Error creating table binpkgs\n'); - pg_exec($db, "CREATE UNIQUE INDEX idxbin on binpkgs (name,bin_name,dist)") or die('Errore creating index - ' . pg_last_error($db)); - $sw=true; -} - #if (!pg_table_exists($db,"upstream")){ - # pg_exec($db, "CREATE TABLE upstream (name text,dist text,up_version text,watch_warn text,updated bool)") or die('Errore nel creare la tabella'); - # pg_exec($db, "CREATE UNIQUE INDEX idxupstrm on upstream (name,dist)") or die('Errore nel creare gli indici ' . pg_last_error($db)); - # $sw=true; -#} -pg_close($db); -if($sw) return $sw; -else return false; -} + global $dirs,$dbconn; + $db = pg_pconnect($dbconn) or die_status(1/*pg_last_notice($db)*/); + if (!pg_table_exists($db,"pkgs")){ + pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text)") or die_status('Error creating table pkgs\n'); + pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db)); + $sw=true; + } + if (!pg_table_exists($db,"binpkgs")){ + pg_exec($db, "CREATE TABLE binpkgs (id serial PRIMARY KEY, name text,bin_name text,dist text, pop_inst numeric,pop_vote numeric, up_error numeric,avg_error_date numeric)") or die_status('Error creating table binpkgs\n'); + pg_exec($db, "CREATE UNIQUE INDEX idxbin on binpkgs (name,bin_name,dist)") or die_status('Errore creating index - ' . pg_last_error($db)); + $sw=true; + } + #if (!pg_table_exists($db,"upstream")){ + # pg_exec($db, "CREATE TABLE upstream (name text,dist text,up_version text,watch_warn text,updated bool)") or die_status('Errore nel creare la tabella'); + # pg_exec($db, "CREATE UNIQUE INDEX idxupstrm on upstream (name,dist)") or die_status('Errore nel creare gli indici ' . pg_last_error($db)); + # $sw=true; + #} + pg_close($db); + if($sw) return $sw; + else return false; +} function pg_table_exists (&$dblink, $table) { - $sql = "SELECT relname as tbl FROM pg_class WHERE relname = '$table'"; - if ($res = pg_exec ($dblink, $sql)) { - return pg_numrows($res) > 0; - } else { - return false; // or throw exception - } - } + $sql = "SELECT relname as tbl FROM pg_class WHERE relname = '$table'"; + if ($res = pg_exec ($dblink, $sql)) { + return pg_numrows($res) > 0; + } else { + return false; // or throw exception + } +} function watch_stat() { -global $dirs,$dbconn; -$db = pg_pconnect($dbconn) or die(pg_errormessage($db)); -$rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch!=''"); -$tot_watch=pg_fetch_array($rsql); -print "Total packages with Watch =>$tot_watch[0]\n"; -$rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch IS NULL"); -$tot_nowatch=pg_fetch_array($rsql); -pg_close($db); -print "Totale packages without Watch => $tot_nowatch[0]\n"; -print "Watch presence rate => " . number_format ($tot_watch[0]/($tot_nowatch[0]+$tot_watch[0])*100, 2, ",",".") . "%\n"; + global $dirs,$dbconn; + $db = pg_pconnect($dbconn) or die_status(pg_last_error($db)); + $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch!=''"); + $tot_watch=pg_fetch_array($rsql); + print "Total packages with Watch =>$tot_watch[0]\n"; + $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch IS NULL"); + $tot_nowatch=pg_fetch_array($rsql); + pg_close($db); + print "Totale packages without Watch => $tot_nowatch[0]\n"; + print "Watch presence rate => " . number_format ($tot_watch[0]/($tot_nowatch[0]+$tot_watch[0])*100, 2, ",",".") . "%\n"; } function remote_dist_dir($dist) { -#if ($dist=="non-US") return "/debian-non-US/dists/unstable"; -#else -return "/debian/dists"; + #if ($dist=="non-US") return "/debian-non-US/dists/unstable"; + #else + return "/debian/dists"; } function clear_db(&$db) { - #global $dbconn; - #if (check_db()) die("Il Database era vuoto\n"); - #keep_source(); - #$db = pg_pconnect($dbconn) or die(pg_result_error($db)); - $sql1= "Delete from binpkgs where id NOT IN + #global $dbconn; + #if (check_db()) die_status("Il Database era vuoto\n"); + #download_sources(); + #$db = pg_pconnect($dbconn) or die_status(pg_result_error($db)); + $sql1= "Delete from binpkgs where id NOT IN (select id FROM binpkgs INNER JOIN bin_atsrc USING (name,bin_name,dist))" ; - $sql2= "Delete from pkgs where id NOT IN (select id FROM pkgs INNER JOIN pkgs_atsrc USING (name,dist))" ; - $sql3= "UPDATE pkgs SET wwiz=NULL, wwiz_type=NULL, wwiz_version=NULL WHERE watch IS NOT NULL or watch!=''" ; - $sql4= "UPDATE pkgs SET up_changes=NULL WHERE wwiz_version=up_version OR up_version=version" ; -if(pg_exec($db,$sql1) AND pg_exec($db,$sql2) AND pg_exec($db,$sql3) AND pg_exec($db,$sql4)) print "Db clear complete\n"; + $sql2= "Delete from pkgs where id NOT IN (select id FROM pkgs INNER JOIN pkgs_atsrc USING (name,dist))" ; + $sql3= "UPDATE pkgs SET wwiz=NULL, wwiz_type=NULL, wwiz_version=NULL WHERE watch IS NOT NULL or watch!=''" ; + $sql4= "UPDATE pkgs SET up_changes=NULL WHERE wwiz_version=up_version OR up_version=version" ; + if(pg_exec($db,$sql1) AND pg_exec($db,$sql2) AND pg_exec($db,$sql3) AND pg_exec($db,$sql4)) print "Db clear complete\n"; } function db_popcon($initial='') { -global $dirs,$dbconn; -$db = pg_pconnect($dbconn) or die(pg_errormessage($db)); -$filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; -$regexp="/Package:\s($initial\S*)\s+(\d+)\s+(\d+)/iS"; - $zp = gzopen($filename, "r") or die("Could not open $filename"); - $tin=time(); - if(filesize($filename)>20) { - #pg_exec($db, "BEGIN TRANSACTION") OR DIE("Impossibile iniziare la transazione"); - while (!gzeof($zp)) { - $line = gzgets ($zp,4096) ; - if (preg_match($regexp,$line,$matches)) { - #print "\rName => $matches[1] Install => $matches[2] Vote => $matches[3]"; - @pg_exec($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'"); - $processed+=1; - $elapsed_time=time()-$tin; - if (($elapsed_time % 20)==0 AND $updtime!=$elapsed_time) { - $updtime=$elapsed_time; - $wprocessed=$processed-$wprocessed; - $avg_time=$wprocessed/20; - $wprocessed=$processed; - $countdown=(34701- $processed)/$avg_time; - $last_time=time(); - } - //print "\rProcessing Popcon: " . date("i:s",$countdown=$countdown-(time()-$last_time)) . " =>" . (34701-$processed) . " Media $avg_time"; - $last_time=time(); - } - } - #pg_exec($db, "COMMIT") OR DIE("Impossibile commsionare la transazione"); - } -@pg_close($db); + global $dirs,$dbconn; + $db = pg_pconnect($dbconn) or die_status(pg_last_error($db)); + $filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; + $regexp="/Package:\s($initial\S*)\s+(\d+)\s+(\d+)/iS"; + $zp = gzopen($filename, "r") or die_status("Could not open $filename"); + $tin=time(); + if(filesize($filename)>20) { + #pg_exec($db, "BEGIN TRANSACTION") OR die_status("Impossibile iniziare la transazione"); + while (!gzeof($zp)) { + $line = gzgets ($zp,4096) ; + if (preg_match($regexp,$line,$matches)) { + #print "\rName => $matches[1] Install => $matches[2] Vote => $matches[3]"; + @pg_exec($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'"); + $prcessed++; + /*$elapsed_time=time()-$tin; + if (($elapsed_time % 20)==0 AND $updtime!=$elapsed_time) { + $updtime=$elapsed_time; + $wprocessed=$processed-$wprocessed; + $avg_time=$wprocessed/20; + $wprocessed=$processed; + $countdown=(34701- $processed)/$avg_time; + $last_time=time(); + } + print "\rProcessing Popcon: " . date("i:s",$countdown=$countdown-(time()-$last_time)) . " =>" . (34701-$processed) . " Media $avg_time"; + $last_time=time();*/ + } + } + #pg_exec($db, "COMMIT") OR die_status("Impossibile commsionare la transazione"); + } + @pg_close($db); } function db_popcon_test() { -global $dirs,$dbconn; -$db = pg_pconnect($dbconn) or die(pg_errormessage($db)); -$filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; -$regexp="/Package:\s(\S+)\s+(\d+)\s+(\d+)/"; - $zp = gzfile($filename, "r") or die("Could not open $filename"); - if(filesize($filename)>20) { - foreach ($zp as $line) { - if (preg_match($regexp,$line,$matches)) { + global $dirs,$dbconn; + $db = pg_pconnect($dbconn) or die_status(pg_last_error($db)); + $filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; + $regexp="/Package:\s(\S+)\s+(\d+)\s+(\d+)/"; + $zp = gzfile($filename, "r") or die_status("Could not open $filename"); + if(filesize($filename)>20) { + foreach ($zp as $line) { + if (preg_match($regexp,$line,$matches)) { # print "Name => $matches[1] \n Install => $matches[2] \n Vote => $matches[3]\n"; - print "\r" . $i++; - #pg_unbuffered_query($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'"); - } - } - } -@pg_close($db); -} -function file_put_contents($filename, $content, $flags = 0) { - if (!($file = fopen($filename,$flags))) return false; - else { - $n = fwrite($file, $content); - fclose($file); - return true; - } - } -function keep_popcon() { -global $dirs,$dbconn; -$localfile=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; -$remotefile="http://popcon.debian.org/all-popcon-results.txt.gz"; -@file_put_contents($localfile,file_get_contents($remotefile),'w') or die("Errore nello scaricare il file popcon\n"); + print "\r" . $i++; + #pg_unbuffered_query($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'"); + } + } + } + @pg_close($db); +} +function dl_popcon() { + global $dirs,$dbconn; + $localfile=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; + $remotefile="http://popcon.debian.org/all-popcon-results.txt.gz"; + @file_put_contents($localfile,file_get_contents($remotefile),'w') or die_status("Errore nello scaricare il file popcon\n"); } function is_updated ($up_vers,$debian_vers) { -if ($up_vers=='') return 0; -passthru("dpkg --compare-versions '$up_vers' eq '" . vers_conv($debian_vers) . "'",$great); -if ($great!=0) return 1; -else return 0; + if ($up_vers=='') return 0; + passthru("dpkg --compare-versions '".escapeshellarg($up_vers)."' le '" . escapeshellarg(vers_conv($debian_vers)) . "'",$great); + if ($great!=0) return 1; + else return 0; } function db_upstream($initial='%') { - global $dirs,$dbconn; - check_db(); - $db = pg_connect($dbconn) ; - $rsql=pg_exec($db, "SELECT id, pkgs.name,pkgs.dist,up_version,version,watch FROM pkgs WHERE watch is not null AND name LIKE '$initial%' ORDER BY name DESC"); - $tot=pg_numrows($rsql); - while ($res_array=pg_fetch_array($rsql)) { - $uscan_res=uscan_foo($res_array['name'],$res_array['version'],$res_array['watch']); - if($uscan_res[1] !=null) $uscan_res[1]=addslashes($uscan_res[1]); - #print $uscan_res[0] . "\n Name => $res_array[name]\n"; - $version=$res_array[version]; - print "Version=>" . $uscan_res[3] . "\n"; - if ($uscan_res[3]!=null) $version=$uscan_res[3]; - $updated=is_updated($uscan_res[0],$res_array[version]); - if ($uscan_res[0]!=$res_array[up_version]) ($keep_changes=1); - else $keep_changes=0; - pg_exec($db, "UPDATE pkgs SET version='". $version . "', up_version='" . $uscan_res[0] . "',up_url='" . $uscan_res[2] . "', watch_warn='" . $uscan_res[1] . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated' WHERE id=$res_array[id]") or Die("Error in update pkgs upstream"); - #print "\rUpdated " . $res_array['name'] . " => $updated Up Version => ". $uscan_res[0]; - print "\rUpstream left => " . $tot --; -# } -# else print "Added ". $res_array['name'] ."\n"; - # if ($great==0) { - - # print "Package=> " . $res_array['name'] . "\n"; - # print "Version => " . $res_array['version'] . "\n"; - # print "Upstream Ver=> " . $uscan_res[0] . "\n"; - #print "Watch warings=> " . $uscan_res[1] . "\n"; - # } - } - print "\n"; - pg_close($db); + global $dirs,$dbconn; + check_db(); + $db = pg_pconnect($dbconn) ; + $rsql=pg_exec($db, "SELECT id, pkgs.name,pkgs.dist,up_version,version,watch FROM pkgs WHERE watch is not null AND name LIKE '$initial%' ORDER BY name DESC"); + $tot=pg_numrows($rsql); + while ($res_array=pg_fetch_array($rsql)) { + $uscan_res=uscan_foo($res_array['name'],$res_array['version'],$res_array['watch']); + #print $uscan_res[0] . "\n Name => $res_array[name]\n"; + $version=$res_array[version]; + $dversionmangled=$res_array['dversionmangled']; + print "\rVersion=>" . $uscan_res[3]; + if ($uscan_res[3]!=null) $version=$uscan_res[3]; + if ($uscan_res[5]!=null) $dversionmangled=$uscan_res[5]; + $updated=$uscan_res[4]; + if ($updated) { + $updated=is_updated($uscan_res[0],$dversionmangled); + } + + if (!$updated) { + events::newVersion($res_array['name'], $res_array['version'], $uscan_res[0], $res_array['dist']); + } + + if ($uscan_res[0]!=$res_array[up_version]) ($keep_changes=1); + else $keep_changes=0; + $up_url=iconv("ISO-8859-1","UTF-8",$uscan_res[2]); + $up_version=iconv("ISO-8859-1","UTF-8",$uscan_res[0]); + $watch_warn=''; + if($uscan_res[1] != null) { + $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8",$uscan_res[1])); + } + pg_exec($db, "UPDATE pkgs SET up_version='" . pg_escape_string($up_version) . "', dversionmangled='".pg_escape_string($dversionmangled)."',up_url='" . pg_escape_string($up_url) . "', watch_warn='" . $watch_warn . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated' WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream"); + #print "\rUpdated " . $res_array['name'] . " => $updated Up Version => ". $uscan_res[0]; + print "\tUpstream left => " . $tot --; + # } + # else print "Added ". $res_array['name'] ."\n"; + # if ($great==0) { + + # print "Package=> " . $res_array['name'] . "\n"; + # print "Version => " . $res_array['version'] . "\n"; + # print "Upstream Ver=> " . $uscan_res[0] . "\n"; + #print "Watch warings=> " . $uscan_res[1] . "\n"; + # } + } + print "\n"; + pg_close($db); } function check_url ($url) { - $url_p = parse_url ($url); - if (isset ($url_p["host"])) $host = $url_p["host"]; - else return false; - preg_match("/(http|ftp)/",$host,$matches); - if ($matches[1]=='http') $fp = fsockopen ($host, 80, $errno, $errstr, 10); - elseif ($matches[1]=='ftp') $fp2 = fsockopen ($host, 21, $errno, $errstr, 10); - if (!$fp AND !$fp2) return false; - else return true; + $url_p = parse_url ($url); + if (isset ($url_p["host"])) $host = $url_p["host"]; + else return false; + preg_match("/(http|ftp)/",$host,$matches); + if ($matches[1]=='http') $fp = fsockopen ($host, 80, $errno, $errstr, 5); + elseif ($matches[1]=='ftp') $fp2 = fsockopen ($host, 21, $errno, $errstr, 5); + if (!$fp AND !$fp2) return false; + else return true; } function keep_remote ($localfile,$remotefile) { - #if (!check_url($remotefile)) return false; - $rfp = @fopen ($remotefile, "rb"); - if (!$rfp) { - return false; - } - $lfp = fopen ($localfile, "w"); -# ini_set ( "memory_limit", "100M"); - while (!feof($rfp)) { - $data = fread($rfp, 1024); - $bytes+=1024; - #print "Downlaoding $remotefile: $bytes\r"; - fwrite($lfp, $data); - } -# ini_set ( "memory_limit", "15M"); - fclose($lfp); - fclose($rfp); - - return true; + if (!check_url($remotefile)) return false; + # $rfp = @fopen ($remotefile, "r"); + # if (!$rfp) { + # return false; + # } + # fclose($rfp); + # $lfp = fopen ($localfile, "w"); + # stream_set_blocking($rfp,0); + # ini_set ( "memory_limit", "100M"); + # while (!feof($rfp)) { + # $data = fread($rfp, 8192); + # $bytes+=8192; + # print "Downlaoding $remotefile: $bytes\r"; + # fwrite($lfp, $data); + # } while (true); + # ini_set ( "memory_limit", "15M"); + + if (!copy($remotefile, $localfile)) { + echo "Copia di $file non riuscita ...\n"; + return false; + } + # fclose($lfp); + # fclose($rfp); + + return true; } -function up_changes($src_name='%') { - global $dbconn; - $changes_files=array("CHANGES","NEWS","Changelog","ChangeLog","changelog"); - $db = pg_connect($dbconn) ; - $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!='' AND keep_changes='t' AND name LIKE '$src_name%'"); - $tot=pg_numrows($rsql); - while ($res_array=pg_fetch_array($rsql)) { - $localfile=tempnam("/tmp/", "dehs-" . $res_array[id] . ".tar.gz"); - #print "I'm downloading $res_array[up_url]\r"; - $res=keep_remote($localfile,$res_array['up_url']); - #print "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]"; - if (!$res) { - pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]"); - $notdown++; - $tot--; - print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r"; - #print "\n Not Downlaoded: $res_array[up_url]\n"; - continue; - } - foreach($changes_files as $filename) { - $command="tar -zxOf " . $localfile . " *$filename 2>/dev/null"; - exec($command,$up_changes,$cmdres); - $up_changes=rtrim(implode("\n",$up_changes)); - if ($cmdres==0 AND strlen($up_changes)>20) { - $sw=true; - $up_changes=addslashes($up_changes); - $conv_res=iconv("ISO-8859-1","UTF-8",$up_changes); - if ($conv_res) $up_changes=$conv_res; - $up_changes_sql="UPDATE pkgs SET up_changes='$up_changes', keep_changes='f' WHERE id=$res_array[id]"; - pg_exec($db,$up_changes_sql); - #print "\nFounded $filename on $res_array[up_url]\n"; - $found++; - break; - } - } - if (!$sw) { - pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]"); - #print "\n Not Founded any on $res_array[up_url]\n"; - $notfound++; - } - unlink($localfile); - print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r"; - } - pg_close($db); +function up_changes($src_name='%', $force = false) { + global $dbconn, $fetchUpstreamChanges; + + if (!$fetchUpstreamChanges && !$force) { + print "\nSkipping download of upstream sources to grab changelogs\n"; + return; + } + + $changes_files=array("CHANGES","NEWS","Changelog","ChangeLog","changelog"); + $db = pg_pconnect($dbconn) ; + $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!='' AND keep_changes='t' AND name LIKE '$src_name%'"); + $tot=pg_numrows($rsql); + while ($res_array=pg_fetch_array($rsql)) { + $localfile=tempnam("/tmp/", "dehs-" . $res_array[id] . ".tar.gz"); + print "I'm downloading $res_array[up_url]\r"; + $res=keep_remote($localfile,$res_array['up_url']); + print "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]"; + if (!$res) { + pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]"); + $notdown++; + $tot--; + print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r"; + #print "\n Not Downlaoded: $res_array[up_url]\n"; + unlink($localfile); + continue; + } + foreach($changes_files as $filename) { + $command="tar --wildcards -zxOf " . $localfile . " *$filename 2>/dev/null"; + exec($command,$up_changes,$cmdres); + $up_changes=rtrim(implode("\n",$up_changes)); + if ($cmdres==0 AND strlen($up_changes)>20) { + $sw=true; + $conv_res=iconv("ISO-8859-1","UTF-8",$up_changes); + if ($conv_res) $up_changes=$conv_res; + $up_changes=pg_escape_string($up_changes); + $up_changes_sql="UPDATE pkgs SET up_changes='$up_changes', keep_changes='f' WHERE id=$res_array[id]"; + pg_exec($db,$up_changes_sql); + #print "\nFound $filename on $res_array[up_url]\n"; + $found++; + break; + } + } + if (!$sw) { + pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]"); + #print "\n Not Found any on $res_array[up_url]\n"; + $notfound++; + } + unlink($localfile); + print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r"; + } + pg_close($db); } function uscan_foo($name,$version,$watch){ - global $base; + global $base, $uscan; + $tmpnam=tempnam('/tmp/', $name.'_watch'); $fw=fopen($tmpnam,"w"); + fwrite($fw,$watch); + fclose($fw); + + $uscan_res=shell_exec("$uscan --dehs --package $name --upstream-version " . escapeshellarg(vers_conv($version)) . " --watchfile $tmpnam 2>/dev/null 2>&1"); - fwrite($fw,$watch); - fclose($fw); - $uscan_res=shell_exec($base . "uscan --dehs --package $name --upstream-version " . vers_conv($version) . " --watchfile $tmpnam 2>/dev/null 2>&1"); unlink($tmpnam); - $p = xml_parser_create(); - xml_parse_into_struct($p, $uscan_res, $vals, $index); - xml_parser_free($p); - $upstream=$vals[$index['UPSTREAM-VERSION'][0]][value]; - $up_url=$vals[$index['UPSTREAM-URL'][0]][value]; - $deb_vers=$vals[$index['DEBIAN-UVERSION'][0]][value]; - $warnings=$vals[$index['WARNINGS'][0]][value]; - if ($upstream=='') $upstream=null; - if ($up_url=='') $up_url=null; - if ($deb_vers=='') $deb_vers=null; - if ($warnings=='') $warnings=null; - return array($upstream,$warnings,$up_url,$deb_vers); -} + + $p = xml_parser_create(); + xml_parse_into_struct($p, $uscan_res, $vals, $index); + xml_parser_free($p); + + $upstream=$vals[$index['UPSTREAM-VERSION'][0]][value]; + $up_url=$vals[$index['UPSTREAM-URL'][0]][value]; + $deb_vers=$vals[$index['DEBIAN-UVERSION'][0]][value]; + $deb_mangled_vers=$vals[$index['DEBIAN-MANGLED-UVERSION'][0]][value]; + $warnings=$vals[$index['WARNINGS'][0]][value]; + $updated=$vals[$index['STATUS'][0]][value]; + + if ($updated=="up to date") $updated=0; + else $updated=1; + + if ($upstream=='') $upstream=null; + if ($up_url=='') $up_url=null; + if ($deb_vers=='') $deb_vers=null; + if ($warnings=='') $warnings=null; + if ($deb_mangled_vers=='') $deb_mangled_vers=null; + + return array($upstream, $warnings, $up_url, $deb_vers, $updated, $deb_mangled_vers); +} function db_up_error() { global $dbconn; - check_db(); - $db = pg_connect($dbconn) ; - $bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done"; - print "Downloading bugs list =>"; - $ferrors=file_get_contents($bugurl) or die ("I can't keep error page from bugs.debian.org"); - print "Done\n"; - $ferrors=explode("
  • ",$ferrors); - $totline=count($ferrors); - #$errors[]=array(); - #array_pop($errors); - print "Total line => $totline\n"; - foreach ($ferrors as $line) { - # print $line . "\n"; - $founded=preg_match("/Package:.+([^<]+)<\/strong><\/a>.+((\d+)\syear\sand\s|)(\d+)\sdays/sS",$line,$matches); - if ($founded) { - $errors[$matches[1]][err_tot]++; - $errors[$matches[1]][days]+=$matches[3]*365+$matches[4]; - #if ($matches[3]) print "Anni $matches[3]"; - #print "Giorni $matches[4]\n"; - } - print "\rLine left => " . $totline--; - } - #arsort($errors); - $total=count($errors); - foreach ($errors as $error=>$value) { - pg_exec($db, "UPDATE binpkgs SET up_error=" . $value[err_tot] . ",avg_error_date=" . $value[days]/$value[err_tot] . " WHERE bin_name='$error'") or Die("Error in upstream error query"); - #print "Pkg: $error Errori: $value[err_tot] Average Days: " . $value[days]/$value[err_tot] . "\n"; - print "\rErrors left => " . $total--; - } - pg_close($db); -} -function watch_fill($src_name='%'){ - global $dirs,$dbconn; - $db = pg_pconnect($dbconn) or DIE ("Db error"); - $checked=$errors=$notfound=$verok=$notmatch=0; - $sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN - (Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist) - WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name%' ORDER BY mpop_inst DESC"; - $rsql=pg_exec($db, $sqlstring); - while ($res_array=pg_fetch_array($rsql)) { - if (substr($res_array["name"],0,3)=='lib') $initial=substr($res_array["name"],0,4); - else $initial=substr($res_array["name"],0,1); - $url="http://packages.debian.org/changelogs/pool/" . - $res_array["section"] . "/" . - $initial . "/" . - $res_array["name"] ."/" . - $res_array["name"] . "_" . - $res_array["version"] . "/copyright"; - $contents=@file_get_contents($url) or ''; - $space=''; - if (!$contents) { - $url="http://packages.debian.org/changelogs/pool/" . - $res_array["section"] . "/" . - $initial . "/" . - $res_array["name"] ."/" . - $res_array["name"] . "_" . - $res_array["version"] . "/" . $res_array["name"] . ".copyright"; - $contents=@file_get_contents($url) or ''; - } - if (!$contents) { - $notfound++; - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz_type='no_cright' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - } - else { - $res=preg_match ('/(sf\.net|sourceforge\.net)/',$contents,$matches); - if ($res) { - preg_match("/([^\d]+)/",$res_array[name],$dmatches); - $depured_debian_name=$dmatches[1]; - $matches[1]="ftp://download.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/"; - } - else { - unset($matches); - $res=preg_match ('/((ftp:\/\/|http:\/\/)\S+(html|htm|\/(?(?=.+(tar|tgz))|[^>\s\)\a]+)))/',$contents,$matches); - $final=substr($matches[1],-1); - if ($final=='.') { - $matches[1]=substr($matches[1],0,-1); - $final=substr($matches[1],-1); - } - if ($matches[2]=="http://") $space=' ' . "(?:.*/)?"; - elseif ($final!='/') $matches[1].='/'; - $matches[1]=preg_replace("/((http|ftp):\/\/.+):(.*)/",'$1' . '$3', $matches[1],1); - } - if (!$res) { - $notmatch++; - $contents=iconv("ISO-8859-1","UTF-8",$contents); - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - } - else { - if (!preg_match("/$res_array[name]/",$matches[1])) { - preg_match("/([^\d]+)/",$res_array[name],$dmatches); - $depured_debian_name=$dmatches[1]; - } - else $depured_debian_name=$res_array[name]; - $watch="version=3\n" . - $matches[1] . $space . $depured_debian_name . "-?_?([\d+\.]+|\d+)\.tar.* debian uupdate"; - $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch); - if ($uscan_res[1] AND !$uscan_res[0]) { - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - $errors++; - } - else { - $verok++; - if ($uscan_res[0]!=$res_array[wwiz_version]) ($keep_changes=1); - else $keep_changes=0; - pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($watch) ."', wwiz_type='watch', wwiz_version='" . $uscan_res[0] . "', up_url='" . $uscan_res[2] . "', keep_changes='$keep_changes' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - } - } - } - print "Package checked: " . $checked++ . - " OK: $verok Error: $errors Not Matched: $notmatch\r"; - } - print "\nCopyright files not founded: $notfound\n"; - print "Automatic genereted watch file failure: $errors\n"; - pg_close($db); -} -function watch_fill_fre($src_name='%'){ - global $dirs,$dbconn; - $db = pg_pconnect($dbconn) or DIE ("Db error"); - $checked=$errors=$notfound=$verok=$notmatch=0; - $sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN - (Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist) + check_db(); + $db = pg_pconnect($dbconn) ; + $bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done"; + print "\nDownloading bugs list =>"; + $ferrors=file_get_contents($bugurl) or die_status ("I can't keep error page from bugs.debian.org"); + print "Done\n"; + $ferrors=explode("
  • ",$ferrors); + $totline=count($ferrors); + #$errors[]=array(); + #array_pop($errors); + print "Total lines => $totline\n"; + foreach ($ferrors as $line) { + # print $line . "\n"; + if (preg_match("/Package:.+?>([^<]+)<\/a>.+((\d+)\syear\sand\s|)(\d+)\sdays/sS",$line,$matches)) { + $errors[$matches[1]][err_tot]++; + $errors[$matches[1]][days]+=$matches[3]*365+$matches[4]; + #if ($matches[3]) print "Anni $matches[3]"; + #print "Giorni $matches[4]\n"; + } + print "\rLines left => " . $totline--; + } + #arsort($errors); + $total=count($errors); + foreach ($errors as $pkg_name=>$value) { + pg_exec($db, "UPDATE binpkgs SET up_error=" . $value[err_tot] . ",avg_error_date=" . $value[days]/$value[err_tot] . " WHERE bin_name='$pkg_name'") or die_status("Error in upstream bug query"); + #print "Pkg: $error Errori: $value[err_tot] Average Days: " . $value[days]/$value[err_tot] . "\n"; + print "\rBugs left => " . $total--; + } + pg_close($db); +} +function watch_wizard($src_name='%'){ + global $dirs,$dbconn,$sfIgnore,$watchFileExts,$watchFileDversionMangle,$watchFormat; + $db = pg_pconnect($dbconn) or die_status ("Db error"); + $checked=$errors=$notfound=$verok=$notmatch=0; + + // prepare some stuff that needs to be done once per run + $opts = array(); + $exts = ''; + $dvmanglers = ''; + + foreach ($watchFileExts as $ext) { + $exts .= $ext . '|'; + } + unset($ext); + + if ($exts != '') { + $opts[] = 'uversionmangle=s/\.(' . substr($exts, 0, -1) . ')$//i'; + } + + foreach ($watchFileDversionMangle as $dvmangler) { + $dvmanglers .= $dvmangler . '|'; + } + unset($dvmangler); + + if ($dvmanglers != '') { + $opts[] = 'dversionmangle=s/[-.+~]?(' . substr($dvmanglers, 0, -1) . ')(.*)$//i'; + } + + if (is_array($opts)) { + $_opts = ''; + foreach ($opts as $opt) { + $_opts .= $opt . ','; + } + $opts = ''; + if ($_opts != ',') { + $opts = 'opts=' . substr($_opts, 0, -1) . ' \\'; + } + } + + $sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.dversionmangled,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN + (Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist) WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name%' ORDER BY mpop_inst DESC"; - $rsql=pg_exec($db, $sqlstring); - while ($res_array=pg_fetch_array($rsql)) { - $url="http://freshmeat.net/projects/" . - $res_array["name"] ; - $contents=@file_get_contents($url) or ''; - $space=''; - if (!$contents) { - $notfound++; - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz_type='no_cright' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - } - else { - $res=preg_match ('/(tgz|zip).+>(\S+\/).*<\//',$contents,$matches); - if ($res) { - $durl=$matches[2]; - $res=preg_match ('/(sf\.net|sourceforge\.net)/',$durl,$matches); - if ($res) { - preg_match("/([^\d]+)/",$res_array[name],$dmatches); - $depured_debian_name=$dmatches[1]; - $durl="ftp://download.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/"; - } - } - if (!durl) { - $notmatch++; - $contents=iconv("ISO-8859-1","UTF-8",$contents); - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - } - else { - if (!preg_match("/$res_array[name]/",$durl)) { - preg_match("/([^\d]+)/",$res_array[name],$dmatches); - $depured_debian_name=$dmatches[1]; - } - else $depured_debian_name=$res_array[name]; - $space=" "; - $watch="version=3\n" . - $durl . $space . $depured_debian_name . "-?_?([\d+\.]+|\d+)\.tar.* debian uupdate"; - $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch); - if ($uscan_res[1] AND !$uscan_res[0]) { - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - $errors++; - } - else { - $verok++; - if ($uscan_res[0]!=$res_array[wwiz_version]) ($keep_changes=1); - else $keep_changes=0; - pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($watch) ."', wwiz_type='watch', wwiz_version='" . $uscan_res[0] . "', up_url='" . $uscan_res[2] . "', keep_changes='$keep_changes' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'"); - } - } - } - #print $watch; - print "Package checked: " . $checked++ . - " OK: $verok Error: $errors Not Matched: $notmatch\r"; - } - print "\nCopyright files not founded: $notfound\n"; - print "Automatic genereted watch file failure: $errors\n"; - pg_close($db); + $rsql=pg_exec($db, $sqlstring); + + while ($res_array=pg_fetch_array($rsql)) { + + if (substr($res_array["name"],0,3)=='lib') + $initial=substr($res_array["name"],0,4); + else + $initial=substr($res_array["name"],0,1); + + $url="http://packages.debian.org/changelogs/pool/" . + $res_array["section"] . "/" . + $initial . "/" . + $res_array["name"] ."/" . + $res_array["name"] . "_" . + $res_array["version"] . "/copyright"; + $contents=@file_get_contents($url) or ''; + $generatedWatches = array(); + + if (!$contents) { + $url="http://packages.debian.org/changelogs/pool/" . + $res_array["section"] . "/" . + $initial . "/" . + $res_array["name"] ."/" . + $res_array["name"] . "_" . + $res_array["version"] . "/" . $res_array["name"] . ".copyright"; + $contents=@file_get_contents($url) or ''; + } + + if (!$contents) { + $url="http://packages.debian.org/changelogs/pool/" . + $res_array["section"] . "/" . + $initial . "/" . + $res_array["name"] ."/" . + "current/copyright"; + $contents=@file_get_contents($url) or ''; + } + + if ($contents) { + + if (preg_match ('/(sf\.net|sourceforge\.net)/i', $contents, $matches) || + (preg_match ('/([\w\-]+)\.(sf\.net|sourceforge\.net)/i', $contents, $matches) + && !in_array($matches[1], $sfIgnore))) { + preg_match("/([^\d]+)/",$res_array['name'],$dmatches); + $depured_debian_name=$dmatches[1]; + /*$matches[1]="ftp://download.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/";*/ + $generatedWatches[]=array('url'=>"http://sf.net/" . $depured_debian_name . "/"); + $generatedWatches[]=array('url'=>"http://sf.net/" . $depured_debian_name . "/",'name'=>$depured_debian_name); + + } + + if (preg_match ('#((ftp://|http://|https://)\S+(html|htm|/(?(?=.+(tar|tgz|zip|gz|bz2))|[^>\s\)\a]+)))#i',$contents,$matches)) { + $final=substr($matches[1],-1); + if ($final=='.') { + $matches[1]=substr($matches[1],0,-1); + $final=substr($matches[1],-1); + } + if ($matches[2]=="http://" || $matches[2]=="https://") { + $space=' ' . "(?:.*/)?"; + } elseif ($final!='/') { + $matches[1].='/'; + } + $generatedWatches[]=array('url' => preg_replace("#((http|https|ftp)://.+):(.*)#i",'$1' . '$3', $matches[1],1), 'space' => $space); + } + + if (preg_match ('#((ftp|http)\.\S+(html|htm|/(?(?=.+(tar|tgz|zip|gz|bz2))|[^>\s\)\a]+)))#i',$contents,$matches)) { + $final=substr($matches[1],-1); + if ($final=='.') { + $matches[1]=substr($matches[1],0,-1); + $final=substr($matches[1],-1); + } + $space = ''; + if ($matches[2]=="http") { + $space=' ' . "(?:.*/)?"; + } elseif ($final!='/') { + $matches[1].='/'; + } + $generatedWatches[]=array('url'=>preg_replace('#(http|ftp)\.(.+)#i','$1://$0', $matches[1],1), 'space'=>$space); + } + + if (empty($generatedWatches)) { + $notmatch++; + $contents=iconv("ISO-8859-1","UTF-8",$contents); + pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); + + } else { + + $defs = array('name' => $res_array['name'], 'url' => '', 'space' => ''); + + foreach ($generatedWatches as $tWatch) { + + if (!is_array($tWatch)) { + $tWatch = array($tWatch); + } + + $tWatch = array_merge($defs, $tWatch); + + + $watch = sprintf($watchFormat, $tWatch['url'], $tWatch['space'], $tWatch['name'], $exts, $opts); + + $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch); + + if ($uscan_res[0]) { + break; + } + } + + if ($uscan_res[1] && !$uscan_res[0]) { + pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); + $errors++; + } else { + $verok++; + if ($uscan_res[0] != $res_array['wwiz_version']) { + $keep_changes = 1; + } else { + $keep_changes = 0; + } + + $dversionmangled = $res_array[version]; + if ($uscan_res[5]!=null) $dversionmangled=$uscan_res[5]; + + $updated=$uscan_res[4]; + if ($updated) { + $updated=is_updated($uscan_res[0],$dversionmangled); + } + + if (!$updated) { + events::newVersion($res_array['name'], $res_array['version'], $uscan_res[0], $res_array['dist']); + } + + pg_exec($db, "UPDATE pkgs SET wwiz='" . pg_escape_string($watch) ."', wwiz_type='watch', wwiz_version='" . pg_escape_string($uscan_res[0]) . "',dversionmangled='".pg_escape_string($dversionmangled)."', up_url='" . pg_escape_string($uscan_res[2]) . "', keep_changes='$keep_changes', updated='$updated' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); + } + } + } else { + $notfound++; + pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz_type='no_cright' where name='" . pg_escape_string($res_array['name']) . "' AND dist='" . $res_array['dist'] ."'"); + } + print "Package checked: " . $checked++ . + " OK: $verok Error: $errors Not Matched: $notmatch\r"; + } + print "\nCopyright files not found: $notfound\n"; + print "Automatic generated watch file failures: $errors\n"; + pg_close($db); } function dbqa_conn ($dbname,$mode) { -$id = dba_open($dbname, $mode, "db4"); -if (!$id) { - echo "dba_open failed\n"; - exit; -} -return $id; + $id = dba_open($dbname, $mode, "db4"); + if (!$id) { + die_status("dba_open failed\n"); + } + return $id; } function dehsqa_db () { -#to complete!!! -#print phpinfo(); - global $dirs,$dbconn; - $dir1='/org/alioth.debian.org/chroot/home/groups/dehs/htdocs/'; - $dir2="/org/qa.debian.org/data/dehs/"; - $dists=array("unstable","experimental"); - foreach ($dists as $dist) { - $id=dbqa_conn($dir1 . "dehs_qa_" . $dist . ".db",'n'); - $db = pg_pconnect($dbconn) or DIE ("Db error"); - $sql="SELECT name, id, up_version, wwiz_version, watch, wwiz_type, watch_warn, updated FROM pkgs WHERE dist='$dist'"; - $rsql=pg_exec($db, $sql); - while($res_array=pg_fetch_array($rsql)) { - if (!$res_array[watch]) $version="N/A"; - elseif (!$res_array[up_version]) $version="Error"; - else $version=$res_array[up_version]; - if ($res_array[wwiz_type]=="watch") $wwiz=$res_array[wwiz_version]; - else $wwiz=$res_array[wwiz_type]; - $xml="$res_array[id]" . - "$version" . - "$wwiz" . - ""; - $value=addslashes($xml); - dba_replace($res_array[name],$xml,$id); - } - dba_optimize($id); - dba_close($id); - pg_close($db); - } + global $dirs,$dbconn; + $dir1='/org/alioth.debian.org/chroot/home/groups/dehs/htdocs/'; + $dir2="/org/qa.debian.org/data/dehs/"; + $dists=array("unstable","experimental"); + foreach ($dists as $dist) { + $id=dbqa_conn($dir1 . "dehs_qa_" . $dist . ".db",'n'); + $db = pg_pconnect($dbconn) or die_status ("Db error"); + $sql="SELECT name, id, up_version, wwiz_version, watch, wwiz_type, watch_warn, version, dversionmangled FROM pkgs WHERE dist='$dist'"; + $rsql=pg_exec($db, $sql); + while($res_array=pg_fetch_array($rsql)) { + + if (!$res_array[watch]) { if ($res_array['wwiz_type']==null) $version="N/A"; else $version='-';} + elseif (!$res_array[up_version]) $version="Error"; + else $version=$res_array[up_version]; + + if ($res_array['wwiz_type']=="watch") $wwiz=$res_array['wwiz_version']; + else $wwiz=$res_array[wwiz_type]; + + $xml="$res_array[id]" . + "$version" . + "$res_array[version]" . + "$res_array[dversionmangled]" . + "$wwiz" . + ""; + $value=addslashes($xml); + dba_replace($res_array[name],$xml,$id); + } + dba_optimize($id); + dba_close($id); + pg_close($db); + } } function md5_of_file($inFile) { - if (file_exists($inFile)) { - return md5_file($inFile); - } else { - return false; - } + if (file_exists($inFile)) { + return md5_file($inFile); + } else { + return false; + } } function update_all() { -global $email; -$intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); -db_add(); -keep_popcon(); -db_popcon(); -keep_diff(); -#db_up_error(); -db_upstream(); -watch_fill(); -up_changes(); -dehsqa_db(); -$endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); -$messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime"; -mail($email, "DEHS report", $messaggio, - "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . - "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . - "X-Mailer: PHP/" . phpversion()); + global $email; + $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + db_add(); + dl_popcon(); + db_popcon(); + dl_diffs(); + db_up_error(); + db_upstream(); + watch_wizard(); + up_changes(); + dehsqa_db(); + $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime"; + mail($email, "DEHS report", $messaggio, + "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . + "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . + "X-Mailer: PHP/" . phpversion()); +} +function update_new() { + global $email_all; + $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + db_add('',false); + dl_popcon(); + db_popcon(); + + global $dbconn; + check_db(); + $db = pg_pconnect($dbconn); + $sql="SELECT name FROM pkgs WHERE (md5_diff!=md5_atsource or md5_diff is null)"; + $rsql=pg_exec($db, $sql); + $res_array=pg_fetch_all($rsql); + pg_close($db); + + dl_diffs(); + db_up_error(); + if ($res_array !== false) { + foreach ($res_array as $entry) { + db_upstream($entry['name']); + up_changes($entry['name']); + watch_wizard($entry['name']); + } + dehsqa_db(); + } + $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + $messaggio="Dehs executed successful\nBegin: $intime\nPackages processed: ".((!is_array($res_array))?'none':count($res_array))."\nEnd: $endtime"; + mail($email_all, "DEHS update_new report", $messaggio, + "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . + "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . + "X-Mailer: PHP/" . phpversion()); +} +function update_bogus() { + global $email_all,$dbconn; + $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + check_db(); + $db = pg_pconnect($dbconn); + $sql="SELECT name FROM pkgs WHERE up_version='' AND watch!=''"; + $rsql=pg_exec($db, $sql); + $res_array=pg_fetch_all($rsql); + $orig_count = ((!is_array($res_array))?0:count($res_array)); + if ($res_array !== false) { + foreach ($res_array as $entry) { + db_upstream($entry['name']); + up_changes($entry['name']); + } + dehsqa_db(); + } + $sql="SELECT COUNT(name) FROM pkgs WHERE up_version='' AND watch!=''"; + $rsql=pg_exec($db, $sql); + $res_array=pg_fetch_array($rsql); + $new_count = (int)$res_array[0]; + pg_close($db); + $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + $messaggio="Dehs executed successful\nBegin: $intime\n". + "Packages processed: $orig_count\n" . + "New bogus watches count: $new_count\nEnd: $endtime"; + mail($email_all, "DEHS update_bogus report", $messaggio, + "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . + "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" . + "X-Mailer: PHP/" . phpversion()); +} +function die_status($msg = '', $status = 1) { + if (empty($msg)) { + die($status); + } else if (is_int($msg) && $status == 1 /* default */) { + die($msg); + } else { + echo $msg; + die($status); + } +} + +class events +{ + function newVersion($sourcePackage, $currentVersion, $newVersion, $distribution, $downloadLink = '') + { + global $base, $sendNotifications; + + if (!$sendNotifications) { + return null; + } + + $tpl = file_get_contents($base . '/tpls/new_version.mail.tpl'); + + $search = array(); + $replace = array(); + + $search [] = '#PACKAGE#'; + $replace[] = $sourcePackage; + $search [] = '#DVERSION#'; + $replace[] = $currentVersion; + $search [] = '#UVERSION#'; + $replace[] = $newVersion; + $search [] = '#DISTRO#'; + $replace[] = $distribution; + + $message = str_replace($search, $replace, $tpl); + $subject = events::_extractHeader('subject', $message); + $to = events::_extractHeader('to', $message); + + $extraHeaders = events::_getTefaultExtraHeaders(); + $extraHeaders[] = 'X-DEHS-UVERSION: ' . $newVersion; + + return mail($to, $subject, $message, implode("\r\n", $extraHeaders)); + } + + function _extractHeader($header, &$message) + { + $regex = '/^' . preg_quote($header, '/') . ':([ \t]*)(.*)$/i'; + + $matches = array(); + + if (preg_match($regex, $message, $matches)) { + + $spacer = $matches[1]; + $value = $matches[2]; + + $message = ltrim(preg_replace($regex, '', $message)); + + return $value; + + } + return false; + } + + function _getTefaultExtraHeaders() + { + global $notificationsMailFrom; + $headers = array(); + + $headers[] = 'From: ' . $notificationsMailFrom; + $headers[] = 'X-PTS-Approved: sure'; + $headers[] = 'X-BY-DEHS: sure'; + + return $headers; + } + } -?> + +?> \ No newline at end of file