#!/usr/bin/php4 -q
<?
/*
 
 Originally written by Stefano Fabri <bluefuture@nospam@email.it>
 Copyright 2004, Stefano Fabri
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.
 
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/
// stabilire una connessione
#ini_set ( "error_reporting", "E_COMPILE_ERROR|E_ERROR|E_CORE_ERROR");
ini_set ("html_errors", "off");
set_time_limit(0);
ini_set ( "memory_limit", "40M");
#print_R($_SERVER);
require "dehs.config.inc";
dir_ctrl($dirs);
if (!function_exists('file_get_contents')) {
	function file_get_contents($filename) {
       if (!($file = fopen($filename,"rb"))) return false;
	   else {
       		$contents = "";
			do {
   				$data = fread($file, 4096);
   				if (strlen($data) == 0) {
       			break;
   				}
   				$contents .= $data;
			} while (true);
		fclose($file);
		return $contents;
		}
   }
}   
if (in_array("-db_add",$argv))  db_add($argv[2]);
        elseif (in_array("-db_query",$argv))  db_query($argv[2],$argv[3] );
        elseif (in_array("-keep_diff",$argv))  keep_diff($argv[2]);
		elseif (in_array("-keep_sources",$argv))  keep_sources();
		elseif (in_array("-db_popcon",$argv))  db_popcon($argv[2]);
		elseif (in_array("-keep_popcon",$argv))  keep_popcon($argv[2]);
		elseif (in_array("-db_upstream",$argv))  db_upstream($argv[2]);
		elseif (in_array("-update_all",$argv))  update_all();
		elseif (in_array("-check_db",$argv))  check_db();
		elseif (in_array("-dehsqa_db",$argv))  dehsqa_db();
		elseif (in_array("-clear_db",$argv))  clear_db();
		elseif (in_array("-db_up_error",$argv))  db_up_error();
		elseif (in_array("-up_changes",$argv))  up_changes($argv[2]);
		elseif (in_array("-watchf",$argv))  watch_fill($argv[2]);
        elseif (in_array("-kb_total",$argv))  print "Diff 
#stat_up_error();
#stat_up_error();to download Kb => " . kb_total($argv[2]) . "\n" ;
		elseif (in_array("-watch_stat",$argv))  watch_stat();
        else print "Command $argv[1] not found\n";
#stat_up_error();
function dir_ctrl($dirs) {
        foreach ($dirs as $dir) {
                if (!is_dir($dir)) exec("mkdir -m 0740 -p $dir",$output,$res);
				if ($res!=0) die("Error creating basic dir: $dir\n");
        }
}
function ext_watch ($file,$pkg,$version){
        if (!file_exists($file)) return "";
        $zp=gzopen($file,"r") or die("Fail to open $file");
        while (!gzeof($zp)) {
                $line = gzgets ($zp,4096);
        if (preg_match("/\+\+\+\s.+\/debian\/watch/",$line)) {
                    $line = gzgets ($zp,4096);
                        preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght);
                        for ($i = 1; $i <= $watch_lenght[1]; $i++)      {
                                $line=gzgets ($zp,4096);
                                $watch .=substr($line,1);
                        }		
                break;
            }
        }
if ($watch!=null) return $watch;
else return NULL;
}
function ftp_cn($login,$password,$ftp_server) {
$conn_id = @ftp_connect($ftp_server) or DIE("Error: Cannot contact ftp server $ftp_server\n");
// login con user name e password
ftp_pasv($conn_id,1);
$login_result = ftp_login($conn_id, "anonymous","bo@bo.org");
// controllo della connessione
if ((!$conn_id) || (!$login_result)) {
        echo "Ftp connection failed!\n";
        echo "Failed connection to $ftp_server\n";
        die();
    } else {
        return $conn_id;
    }
}                                                                               
function keep_diff($initial) {
        global $dirs,$dbconn,$mirrors;
	    check_db();	
	 $conn_normal=ftp_cn("anonymous","bo@bo.org",$mirrors['unstable']);
	 #$conn_non_us=ftp_cn("anonymous","bo@bo.org",$mirrors["non-US"]);
       $db = pg_pconnect($dbconn) ;
       $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR DIE("Error in diff query\n"); 
       		$total=pg_numrows($rsql);
			$btotal=byte_total($db,$initial);
			print "Total Diff to downlaod Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n";
	    	while ($res_array=pg_fetch_array($rsql)) {
			#if ($res_array['dist']=='non-US') $conn_id = $conn_non_us;
			#else 
				$conn_id = $conn_normal;
 	       		if   ($localfile=download_diffs($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) {
		   			$watch=ext_watch($localfile,$res_array[name],$res_array[version]);						
					unlink($localfile);
					if ($watch!=NULL) {
					        $watch=iconv("ISO-8859-1","UTF-8",$watch);
						$watch= addslashes($watch);
						pg_exec($db, "UPDATE pkgs SET watch='" . $watch . "', md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Query error"); 
					}
					else pg_exec($db, "UPDATE pkgs SET watch=NULL, md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Errore nella query"); 
				}
			$btotal=$btotal-$res_array['bytes'];		
			number_format ($btotal/1024, 2, ",",".");
		print "\rDiffs left =>	" . $total-- . " Bytes left => " . number_format ($btotal/1024, 2, ",",".") . " - " . $res_array[name];
 			}
	ftp_quit($conn_normal);					
	pg_close($db);
	return "Fatto\n";
}
function download_diffs(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) {
global $dirs,$mirrors;
			$localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz";
			#if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz";
			#else 
			$remotefile="/debian/$directory/$pkg" . "_" . "$version.diff.gz";
			#print "$localfile - $remotefile\n";
        	if      (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
        	    #    print "$pkg Diff.gz => Downloaded\n";
            	if(md5_of_file($localfile)==$md5_atsource) {
				#print "MD5 => OK\n";		
				#unlink($localfile);	
				return $localfile;
				}
			}
        	print "Download Diff.gz of package $pkg => Fallito\n";
			return false;
}
function db_add($initial=''){
global $dirs,$dists,$dbconn;
check_db();
keep_source();
$db = pg_connect($dbconn)  or  die(pg_errormessage($db));
pg_exec($db, "CREATE TEMP TABLE pkgs_atsrc (name text,dist text)") or die('Error creating temp table pkgs_atsrc');
pg_exec($db, "CREATE UNIQUE INDEX idxdis on pkgs_atsrc (name,dist)") or die('Error creating index on temp table');
pg_exec($db, "CREATE TEMP TABLE bin_atsrc (name text,bin_name text, dist text)") or die('Error creating temp table pkgs_atsrc');
pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die('Error creating index on temp table bin_atsrc');
	foreach ($dists as $dist=>$sections) {	
        foreach ($sections as $section) {
        	$filename=$dirs[sources_dir] . "/$dist/$section/Sources.gz";
			print "\nI'm parsing $dist/$section => Sources.gz\n";
        # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5
        	$regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S+)\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?/Ssi";	
    		$zp = gzopen($filename, "r") or die("Could not open $filename");
			$extracted=tempnam("/tmp/", "$dist_$section_Sources");
			exec("gzip -c -d $filename > $extracted");
			$sourcesize=filesize($extracted);
			unlink($extracted);
			if(filesize($filename)>20) {
            	while (!gzeof($zp)) {
                   $line='';
                        while ($line!="\n")  {
                                $line = gzgets ($zp,4096) ;
                                $buff1 .= $line;
                        }
                        if (preg_match($regexp,$buff1,$matches))  {
							if ($matches[10]) {
								$matches[10]=iconv("ISO-8859-1","UTF-8",$matches[10]);
								$matches[10]=addslashes($matches[10]);
							}
							@pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR DIE("Temp table pkgs_atsrc query error");
							$rst=@pg_exec($db, "INSERT INTO pkgs (name,version,maint,dir,md5_atsource,bytes,dist,section,uploaders) VALUES ('$matches[1]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]')") ;
							if (!$rst) {
								$rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]',maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]' WHERE name='$matches[1]' AND dist='$dist'") OR DIE ("\nDb adding error =>" . pg_errormessage() . "\n");
									}	
							$bin_names=split(",", $matches[2]);
							foreach ($bin_names as $bin_name) {
								pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR DIE("Temp table pkgs_atsrc query error");
								$rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ;
							}
						}
					else {
					preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches);
					if(strncmp($initial,$matches[1],strlen($initial))<0) break;	
					}
					$left=$sourcesize-gztell($zp);
					print "\rSource file Left => $left";
					$buff1="";
					unset($matches);
				}
            }
            gzclose($zp) ;
        }
    } 
	clear_db($db);
	   pg_close($db);	
}
function db_query($pkg,$dist) {
        global $dirs,$dbconn;
		$db = pg_pconnect($dbconn )  or  die(pg_errormessage($db));
		$rsql=pg_exec($db, "SELECT pkgs.* ,mpop_inst FROM (SELECT name,MAX(binpkgs.pop_inst) AS mpop_inst FROM binpkgs GROUP BY name) 
									AS binpkgs INNER JOIN pkgs ON pkgs.name=binpkgs.name 
		              WHERE pkgs.name='$pkg' AND dist='$dist';"); 
       if (pg_numrows($rsql)==1) {
		        $res_array=pg_fetch_array($rsql);
                print "Package=> " . $res_array['name'] . "\n";
                print "Version => " . $res_array['version'] . "\n";
                print "Diff MD5 Downloaded => " . $res_array['md5_diff'] . "\n";
				print "Diff MD5 at Sources => " . $res_array['md5_atsource'] . "\n";
				print "Distribution => " . $res_array['dist'] . "\n";
				print "Section => " . $res_array['section'] . "\n";
                print "KBytes => " . number_format ( $res_array['bytes']/1024, 2, ",",".") . "\n";
				print "Pop Inst=>" . $res_array['mpop_inst'] . "\n";
        if ($res_array['watch']) {
		       $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['watch']);              
                print "Version Upstream => " . $uscan_res[0] . "\n";              
			    print "Watch => " . $res_array['watch'];
				print "Watch Warings => " . $uscan_res[1];
        }
        else print "Watch => Not present" . "\n";
        }
     else echo "Package=> " . $pkg . "\nNot found\n!!";
   pg_close($db);
}
function byte_total(&$db,$initial){
global $dirs;
$rsql=pg_exec ($db,"Select sum(bytes) as totbytes FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)");
$res_array=pg_fetch_array($rsql);
return $res_array[totbytes] ;
}
function keep_source (){
		global $dirs,$dists,$mirrors;
	foreach ($dists as $dist=>$sections) {	
				$server=$mirrors[$dist];
	            $conn_id = ftp_cn("anonymous","bo@bo.org",$server);
        foreach ($sections as $section) {		         
                $localfile=$dirs[sources_dir] . "/$dist/$section/Sources.gz";
               $remotefile=remote_dist_dir($dist) . "/$dist/$section/source/Sources.gz";
                $sources_info=md5_source($section,$dist);
                if (!file_exists ($dirs[sources_dir] . "/$dist/")) mkdir ( $dirs[sources_dir] . "/$dist",0740);
				if (!file_exists ($dirs[sources_dir] . "/$dist/$section")) mkdir ($dirs[sources_dir] . "/$dist/$section",0740);
        	if (!file_exists($localfile) OR md5_of_file($localfile)!=$sources_info[md5])  {                                                                                
        		print "Downloading $remotefile Bytes $sources_info[bytes]=> ";
        		if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
                	print "Done\n";
                	if(md5_of_file($localfile)==$sources_info[md5]) print "MD5 => OK\n";
        		}
        		else {
        			unlink("$localfile");
        			print "Fail\n";
        			die ("Error in source file");
        		}	
    		}
        }
		ftp_quit($conn_id);
	}
}
function md5_source($section,$dist) {
global $mirrors;
$relase=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die("Failed to check MD5 of sources");
preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $relase,$matches);
return array("md5"=>$matches[1],"bytes"=>$matches[2]);
 # 183dea9ca3c2e35ffd6cf5eee9501c21          2941637 main/binary-i386/Packages.gz
}
                                                                                
function vers_conv($debvers) {
/*	preg_match("/(.+:)?([^-]+)(ds|dsfg|debian)/",$debvers,$matches);
	if (!$matches[3]) {
		unset($matches);
		preg_match("/(.+:)?([^-]+)/",$debvers,$matches);
	}
	if (substr($matches[2],-1)=='.') $matches[2]=substr($matches[2],0,-1);
	return $matches[2];
*/
        // Strip off the epoch
        $pos = strpos($debvers, ':');
        if ( $pos !== FALSE) {
                $debvers = substr($debvers, $pos+1);
        }

        // strip off the Debian revision (look from the back of the string)
        $pos = strrpos($debvers, '-');
        if ( $pos !== FALSE) {
                $debvers = substr($debvers, 0, $pos);
        }

        // strip off repacking indicators
        $debvers = preg_replace("/[-.+~]?(ds|dfsg|debian)(.*)/i", "", $debvers);
        return $debvers;
}
function check_db() {
global $dirs,$dbconn;
$db = @pg_connect($dbconn) or die(pg_last_notice());
   if (!pg_table_exists($db,"pkgs")){
   pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text)") or die('Error creating table pkgs\n');
   pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die('Errore creating index - ' . pg_last_error($db));
   $sw=true;
  } 
 if (!pg_table_exists($db,"binpkgs")){
   pg_exec($db, "CREATE TABLE binpkgs (id serial PRIMARY KEY, name text,bin_name text,dist text, pop_inst numeric,pop_vote numeric, up_error numeric,avg_error_date numeric)") or die('Error creating table binpkgs\n');
   pg_exec($db, "CREATE UNIQUE INDEX idxbin on binpkgs (name,bin_name,dist)") or die('Errore creating index - ' . pg_last_error($db));
   $sw=true;
}
 #if (!pg_table_exists($db,"upstream")){
 #  pg_exec($db, "CREATE TABLE upstream (name text,dist text,up_version text,watch_warn text,updated bool)") or die('Errore nel creare la tabella');
 #  pg_exec($db, "CREATE UNIQUE INDEX idxupstrm on upstream (name,dist)") or die('Errore nel creare gli indici ' . pg_last_error($db));
 #  $sw=true;
#}
pg_close($db);
if($sw) return $sw;
else return false;
}                                                                                
function pg_table_exists (&$dblink, $table) {
       $sql = "SELECT relname as tbl FROM pg_class WHERE relname = '$table'";
       if ($res = pg_exec ($dblink, $sql)) {
           return pg_numrows($res) > 0;
       } else {
           return false; // or throw exception
       }
   }


function watch_stat() {
global $dirs,$dbconn;
$db = pg_pconnect($dbconn) or die(pg_errormessage($db));
$rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch!=''");
$tot_watch=pg_fetch_array($rsql);
print "Total packages with Watch =>$tot_watch[0]\n";
$rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch IS NULL");
$tot_nowatch=pg_fetch_array($rsql);
pg_close($db);
print "Totale packages without Watch => $tot_nowatch[0]\n";
print "Watch presence rate => " . number_format ($tot_watch[0]/($tot_nowatch[0]+$tot_watch[0])*100, 2, ",",".")  . "%\n";
}
function remote_dist_dir($dist) {
#if ($dist=="non-US") return "/debian-non-US/dists/unstable";
#else 
return "/debian/dists";
}
function clear_db(&$db) {
	#global $dbconn;
	#if (check_db()) die("Il Database era vuoto\n");
	#keep_source();
	#$db = pg_pconnect($dbconn)  or  die(pg_result_error($db));
	$sql1=	"Delete from binpkgs where id NOT IN 
			(select id FROM binpkgs INNER JOIN bin_atsrc USING (name,bin_name,dist))" ;
	$sql2=	"Delete from pkgs where id NOT IN (select id FROM pkgs INNER JOIN pkgs_atsrc USING (name,dist))" ;		
	$sql3=	"UPDATE pkgs SET wwiz=NULL, wwiz_type=NULL, wwiz_version=NULL WHERE watch IS NOT NULL or watch!=''" ;					
	$sql4=	"UPDATE pkgs SET up_changes=NULL WHERE wwiz_version=up_version OR up_version=version" ;					
if(pg_exec($db,$sql1) AND pg_exec($db,$sql2) AND pg_exec($db,$sql3) AND pg_exec($db,$sql4)) print "Db clear complete\n";			   
}
function db_popcon($initial='') {
global $dirs,$dbconn;
$db = pg_pconnect($dbconn)  or  die(pg_errormessage($db));
$filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
$regexp="/Package:\s($initial\S*)\s+(\d+)\s+(\d+)/iS";	
    		$zp = gzopen($filename, "r") or die("Could not open $filename");
			$tin=time();
			if(filesize($filename)>20) {
			#pg_exec($db, "BEGIN TRANSACTION") OR DIE("Impossibile iniziare la transazione");
            	while (!gzeof($zp)) {
				$line = gzgets ($zp,4096) ;
				 if (preg_match($regexp,$line,$matches)) {
                 #print "\rName => $matches[1] Install => $matches[2] Vote => $matches[3]";
				 @pg_exec($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'");
				 $processed+=1;
				 $elapsed_time=time()-$tin;
				 if (($elapsed_time % 20)==0 AND $updtime!=$elapsed_time) {
				 	$updtime=$elapsed_time;
				    $wprocessed=$processed-$wprocessed;
					$avg_time=$wprocessed/20;
					$wprocessed=$processed;
				 	$countdown=(34701- $processed)/$avg_time; 
					$last_time=time();
				 }
				 //print "\rProcessing Popcon: " .  date("i:s",$countdown=$countdown-(time()-$last_time))   . " =>" . (34701-$processed) . " Media $avg_time";
				 $last_time=time();
				}		
				}
			#pg_exec($db, "COMMIT") OR DIE("Impossibile commsionare la transazione");	
			}
@pg_close($db);			
}
function db_popcon_test() {
global $dirs,$dbconn;
$db = pg_pconnect($dbconn)  or  die(pg_errormessage($db));
$filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
$regexp="/Package:\s(\S+)\s+(\d+)\s+(\d+)/";	
    		$zp = gzfile($filename, "r") or die("Could not open $filename");
			if(filesize($filename)>20) {
            	foreach ($zp as $line) {
				 if (preg_match($regexp,$line,$matches)) {
                # print "Name => $matches[1] \n Install => $matches[2] \n Vote => $matches[3]\n";
				 print "\r" . $i++;
				 #pg_unbuffered_query($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'");
				}		
				}	
			}
@pg_close($db);			
}
function file_put_contents($filename, $content, $flags = 0) {
       if (!($file = fopen($filename,$flags))) return false;
	   else {
       	$n = fwrite($file, $content);
       	fclose($file);
       	return true;
	   }
   }
function keep_popcon() {
global $dirs,$dbconn;
$localfile=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
$remotefile="http://popcon.debian.org/all-popcon-results.txt.gz";
@file_put_contents($localfile,file_get_contents($remotefile),'w') or die("Errore nello scaricare il file popcon\n");
}
function is_updated ($up_vers,$debian_vers) {
if ($up_vers=='') return 0;
passthru("dpkg --compare-versions '$up_vers' eq '" . vers_conv($debian_vers) . "'",$great);
if ($great!=0) return 1;
else return 0;
}
function db_upstream($initial='%') {
       global $dirs,$dbconn;
	    check_db();	
       $db = pg_connect($dbconn) ;
       $rsql=pg_exec($db, "SELECT id, pkgs.name,pkgs.dist,up_version,version,watch FROM pkgs WHERE watch is not null AND name LIKE '$initial%' ORDER BY name DESC"); 
	   $tot=pg_numrows($rsql);
        	while ($res_array=pg_fetch_array($rsql)) {
				$uscan_res=uscan_foo($res_array['name'],$res_array['version'],$res_array['watch']);
				if($uscan_res[1] !=null) $uscan_res[1]=addslashes($uscan_res[1]);
				#print $uscan_res[0] . "\n Name => $res_array[name]\n";
				$version=$res_array[version];
				print "Version=>" . $uscan_res[3] . "\n";
				if ($uscan_res[3]!=null) $version=$uscan_res[3];
	            #$updated=is_updated($uscan_res[0],$res_array[version]);
	            $updated=$uscan_res[4];
				if ($uscan_res[0]!=$res_array[up_version]) ($keep_changes=1);
				else $keep_changes=0;
                $up_url=iconv("ISO-8859-1","UTF-8",$uscan_res[2]);
				$up_version=iconv("ISO-8859-1","UTF-8",$uscan_res[0]);
				$watch_warn=iconv("ISO-8859-1","UTF-8",$uscan_res[1]);
				pg_exec($db, "UPDATE pkgs SET up_version='" . $up_version . "',up_url='" . $up_url . "', watch_warn='" . $watch_warn . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated' WHERE id=$res_array[id]") or Die("Error in update pkgs upstream");
				#print "\rUpdated " . $res_array['name'] .  " => $updated Up Version => ". $uscan_res[0];
				print "\rUpstream left => " . $tot --;
#				}	
#							else print "Added ". $res_array['name'] ."\n";
			#	if ($great==0) {
				
			#		print "Package=> " . $res_array['name'] . "\n";
            #		print "Version => " . $res_array['version'] . "\n";		
            #		print "Upstream Ver=> " . $uscan_res[0] . "\n";
					#print "Watch warings=> " . $uscan_res[1]  . "\n";
			#	}
				}	
	print "\n";			
	pg_close($db);
}
function check_url ($url) {
   $url_p = parse_url ($url);
   if (isset ($url_p["host"])) $host = $url_p["host"];
   else return false;
   preg_match("/(http|ftp)/",$host,$matches);
   if ($matches[1]=='http') $fp = fsockopen ($host, 80, $errno, $errstr, 5);
   elseif ($matches[1]=='ftp') $fp2 = fsockopen ($host, 21, $errno, $errstr, 5);
   if (!$fp AND !$fp2) return false;
   else return true;
}
function keep_remote ($localfile,$remotefile) {
	if (!check_url($remotefile)) return false;
#	$rfp = @fopen ($remotefile, "r");
#	if (!$rfp) {
#		return false;
#		}
#     fclose($rfp);
#	$lfp = fopen ($localfile, "w");
#	stream_set_blocking($rfp,0);
#	ini_set ( "memory_limit", "100M");
#	while (!feof($rfp)) {
#		$data = fread($rfp, 8192);
#		$bytes+=8192;
#		print "Downlaoding $remotefile: $bytes\r";
#		fwrite($lfp, $data);
#	} while (true);
#	ini_set ( "memory_limit", "15M");
    
	if (!copy($remotefile, $localfile)) {
    echo "Copia di $file non riuscita ...\n";
    return false;
}
#	fclose($lfp);
#	fclose($rfp);
	
	return true;
}
function up_changes($src_name='%') {
    global $dbconn;
	$changes_files=array("CHANGES","NEWS","Changelog","ChangeLog","changelog");
    $db = pg_pconnect($dbconn) ;
    $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!=''  AND keep_changes='t' AND name LIKE '$src_name%'"); 
	$tot=pg_numrows($rsql);
      	while ($res_array=pg_fetch_array($rsql)) {
		    $localfile=tempnam("/tmp/", "dehs-" . $res_array[id] . ".tar.gz");
			print "I'm downloading $res_array[up_url]\r";
			$res=keep_remote($localfile,$res_array['up_url']);
			print "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]";
			if (!$res) {
				pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");				
				$notdown++;
				$tot--; 
				print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
				#print "\n Not Downlaoded: $res_array[up_url]\n";
				unlink($localfile);
				continue;
			}
			foreach($changes_files as $filename) {
			    $command="tar -zxOf " . $localfile . " *$filename 2>/dev/null";
				exec($command,$up_changes,$cmdres);
				$up_changes=rtrim(implode("\n",$up_changes));
				if ($cmdres==0 AND strlen($up_changes)>20) {
					$sw=true;
					$up_changes=addslashes($up_changes);
					$conv_res=iconv("ISO-8859-1","UTF-8",$up_changes);
					if ($conv_res) $up_changes=$conv_res;
					$up_changes_sql="UPDATE pkgs SET up_changes='$up_changes', keep_changes='f' WHERE id=$res_array[id]";
					pg_exec($db,$up_changes_sql);
					#print "\nFound $filename on $res_array[up_url]\n";
					$found++;
					break;
				}	
			}
			if (!$sw) {
			pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");
			#print "\n Not Found any on $res_array[up_url]\n";
			$notfound++;
			}
			unlink($localfile);
			print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
		}
	pg_close($db);	
}
function uscan_foo($name,$version,$watch){
    global $base, $uscan;
    $tmpnam=tempnam('/tmp/', $name.'_watch');
    $fw=fopen($tmpnam,"w");
	fwrite($fw,$watch);
	fclose($fw);
    $uscan_res=shell_exec("$uscan --dehs --package $name --upstream-version " . vers_conv($version) . " --watchfile $tmpnam 2>/dev/null 2>&1");
    unlink($tmpnam);
	$p = xml_parser_create();
	xml_parse_into_struct($p, $uscan_res, $vals, $index);
	xml_parser_free($p);
	$upstream=$vals[$index['UPSTREAM-VERSION'][0]][value];
	$up_url=$vals[$index['UPSTREAM-URL'][0]][value];
	$deb_vers=$vals[$index['DEBIAN-UVERSION'][0]][value];
	$warnings=$vals[$index['WARNINGS'][0]][value];
	$updated=$vals[$index['STATUS'][0]][value];
	if ($updated=="up to date")  $updated=0;
	else $updated=1;
	if ($upstream=='') $upstream=null;
	if ($up_url=='')  $up_url=null;
	if ($deb_vers=='') $deb_vers=null;
	if ($warnings=='') $warnings=null;
	return array($upstream,$warnings,$up_url,$deb_vers,$updated);
}	
function db_up_error() {
    global $dbconn;
	check_db();
	$db = pg_connect($dbconn) ;
	$bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done";
	print "Downloading bugs list =>";
	$ferrors=file_get_contents($bugurl) or die ("I can't keep error page from bugs.debian.org");
	print "Done\n";
	$ferrors=explode("<li>",$ferrors);
	$totline=count($ferrors);
	#$errors[]=array();
	#array_pop($errors);
	print "Total line => $totline\n";
	foreach ($ferrors as $line) {
	   # print $line . "\n";
		$founded=preg_match("/Package:.+?>([^<]+)<\/a>.+<strong>((\d+)\syear\sand\s|)(\d+)\sdays/sS",$line,$matches);
		if ($founded) {
			$errors[$matches[1]][err_tot]++;
			$errors[$matches[1]][days]+=$matches[3]*365+$matches[4];
			#if ($matches[3]) print "Anni $matches[3]";
			#print "Giorni $matches[4]\n";
		}
		print "\rLine left => " . $totline--;
	}
	#arsort($errors);
	$total=count($errors);
	foreach ($errors as $error=>$value) {
		pg_exec($db, "UPDATE binpkgs SET up_error=" . $value[err_tot] . ",avg_error_date=" . $value[days]/$value[err_tot] . " WHERE bin_name='$error'") or Die("Error in upstream error query");
		#print "Pkg: $error Errori: $value[err_tot] Average Days: " . $value[days]/$value[err_tot]  . "\n";
		print "\rErrors left => " . $total--;
	}
	pg_close($db);
}
function watch_fill($src_name='%'){
	global $dirs,$dbconn;
	$db = pg_pconnect($dbconn) or DIE ("Db error");
	$checked=$errors=$notfound=$verok=$notmatch=0;
	$sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN
	(Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist) 
	WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name%' ORDER BY mpop_inst DESC";
	$rsql=pg_exec($db, $sqlstring);
	while ($res_array=pg_fetch_array($rsql)) {
		if (substr($res_array["name"],0,3)=='lib') $initial=substr($res_array["name"],0,4);
		else $initial=substr($res_array["name"],0,1);
		$url="http://packages.debian.org/changelogs/pool/" . 
		$res_array["section"] . "/" .
		$initial . "/" . 
		$res_array["name"] ."/" . 
		$res_array["name"] . "_" .
		$res_array["version"] . "/copyright";
		$contents=@file_get_contents($url) or '';
		$space='';
		if (!$contents) {
			$url="http://packages.debian.org/changelogs/pool/" . 
			$res_array["section"] . "/" .
			$initial . "/" . 
			$res_array["name"] ."/" . 
			$res_array["name"] . "_" .
			$res_array["version"] . "/" . $res_array["name"] . ".copyright";
			$contents=@file_get_contents($url) or '';
		}
		if (!$contents) {
			$notfound++;
			pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz_type='no_cright' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
		}
		else {
			$res=preg_match ('/(sf\.net|sourceforge\.net)/',$contents,$matches);
			if ($res) {
				preg_match("/([^\d]+)/",$res_array[name],$dmatches);
				$depured_debian_name=$dmatches[1];
				/*$matches[1]="ftp://download.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/";*/
				$matches[1]="http://sf.net/" . $depured_debian_name . "/";
			}	
			else {
				unset($matches);
				$res=preg_match ('/((ftp:\/\/|http:\/\/)\S+(html|htm|\/(?(?=.+(tar|tgz))|[^>\s\)\a]+)))/',$contents,$matches);	
				$final=substr($matches[1],-1);
				if ($final=='.') {
					$matches[1]=substr($matches[1],0,-1);
					$final=substr($matches[1],-1);
				}	
				if ($matches[2]=="http://") $space=' ' . "(?:.*/)?";
		    	elseif ($final!='/') $matches[1].='/';
				$matches[1]=preg_replace("/((http|ftp):\/\/.+):(.*)/",'$1' . '$3', $matches[1],1);
				}
			if (!$res) {
				$notmatch++;
				$contents=iconv("ISO-8859-1","UTF-8",$contents);
				pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
			}
			else {
				if (!preg_match("/$res_array[name]/",$matches[1])) {
				preg_match("/([^\d]+)/",$res_array[name],$dmatches);
				$depured_debian_name=$dmatches[1];
				}
				else $depured_debian_name=$res_array[name];
				$watch="version=3\n" . 
				$matches[1]  . $space . $depured_debian_name . "-?_?([\d+\.]+|\d+)\.tar.* debian uupdate";
				$uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch);
                                
			if ($uscan_res[1] AND !$uscan_res[0]) {
				$watch="version=3\n" . 
				$matches[1]  . $space . $depured_debian_name . "-?_?([\d+\.]+|\d+)\.tgz debian uupdate";
				$uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch);
                        }
                                
			if ($uscan_res[1] AND !$uscan_res[0]) {
                                
				pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
				$errors++;
                        }
			else {
				$verok++;
				if ($uscan_res[0]!=$res_array[wwiz_version]) ($keep_changes=1);
				else $keep_changes=0;
				pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($watch) ."', wwiz_type='watch', wwiz_version='" . $uscan_res[0] . "', up_url='" . $uscan_res[2] . "', keep_changes='$keep_changes' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
			}
		}
		}
		print "Package checked: " . $checked++ . 
		      " OK: $verok Error: $errors Not Matched: $notmatch\r";
	}
	print "\nCopyright files not found: $notfound\n"; 
	print "Automatic genereted watch file failure: $errors\n";
	pg_close($db);
}
// This doesn't seem to be used:
function watch_fill_fre($src_name='%'){
	global $dirs,$dbconn;
	$db = pg_pconnect($dbconn) or DIE ("Db error");
	$checked=$errors=$notfound=$verok=$notmatch=0;
	$sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN
	(Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist) 
	WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name%' ORDER BY mpop_inst DESC";
	$rsql=pg_exec($db, $sqlstring);
	while ($res_array=pg_fetch_array($rsql)) {
		$url="http://freshmeat.net/projects/" . 
		$res_array["name"] ;
		$contents=@file_get_contents($url) or '';
		$space='';
		if (!$contents) {
			$notfound++;
			pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz_type='no_cright' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
		}
		else {
			$res=preg_match ('/(tgz|zip).+>(\S+\/).*<\//',$contents,$matches);
			if ($res)  {
				$durl=$matches[2];
				$res=preg_match ('/(sf\.net|sourceforge\.net)/',$durl,$matches);
				if ($res) {
					preg_match("/([^\d]+)/",$res_array[name],$dmatches);
					$depured_debian_name=$dmatches[1];
					$durl="ftp://download.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/";
				}
			}	
			if (!durl) {
				$notmatch++;
				$contents=iconv("ISO-8859-1","UTF-8",$contents);
				pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
			}
			else {
				if (!preg_match("/$res_array[name]/",$durl)) {
				preg_match("/([^\d]+)/",$res_array[name],$dmatches);
				$depured_debian_name=$dmatches[1];
				}
				else $depured_debian_name=$res_array[name];
				$space=" ";
				$watch="version=3\n" . 
				$durl  . $space . $depured_debian_name . "-?_?([\d+\.]+|\d+)\.tar.* debian uupdate";
				$uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch);
			if ($uscan_res[1] AND !$uscan_res[0]) {
				pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
				$errors++;
			}
			else {
				$verok++;
				if ($uscan_res[0]!=$res_array[wwiz_version]) ($keep_changes=1);
				else $keep_changes=0;
				pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($watch) ."', wwiz_type='watch', wwiz_version='" . $uscan_res[0] . "', up_url='" . $uscan_res[2] . "', keep_changes='$keep_changes' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
			}
		}
		}
		#print $watch;
		print "Package checked: " . $checked++ . 
		      " OK: $verok Error: $errors Not Matched: $notmatch\r";
	}
	print "\nCopyright files not found: $notfound\n"; 
	print "Automatic genereted watch file failure: $errors\n";
	pg_close($db);
}
function dbqa_conn ($dbname,$mode) {
$id = dba_open($dbname, $mode, "db4");
if (!$id) {
   echo "dba_open failed\n";
   exit;
}
return $id;
}
function dehsqa_db () {
#to complete!!!
#print phpinfo();
	global $dirs,$dbconn;
	$dir1='/org/alioth.debian.org/chroot/home/groups/dehs/htdocs/';
	$dir2="/org/qa.debian.org/data/dehs/";
	$dists=array("unstable","experimental");
	foreach ($dists as $dist) {
		$id=dbqa_conn($dir1 . "dehs_qa_" . $dist . ".db",'n');
		$db = pg_pconnect($dbconn) or DIE ("Db error");
		$sql="SELECT name, id, up_version, wwiz_version, watch, wwiz_type, watch_warn, updated FROM pkgs WHERE dist='$dist'"; 
		$rsql=pg_exec($db, $sql);
		while($res_array=pg_fetch_array($rsql)) {
			if (!$res_array[watch]) $version="N/A";
			elseif (!$res_array[up_version]) $version="Error";
			else $version=$res_array[up_version];
			if ($res_array[wwiz_type]=="watch") $wwiz=$res_array[wwiz_version];
			else $wwiz=$res_array[wwiz_type];
			$xml="<data><id>$res_array[id]</id>" .
			 	"<up_version>$version</up_version>" .
			 	"</data>";	 
			$value=addslashes($xml);
			dba_replace($res_array[name],$xml,$id);
		}
		dba_optimize($id);
		dba_close($id);
		pg_close($db);
	}	
}
function md5_of_file($inFile) {
   if (file_exists($inFile)) {
       return md5_file($inFile);
   } else {
       return false;
   }
}
function update_all() {
global $email;
$intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
db_add();
keep_popcon();
db_popcon();
keep_diff();
db_up_error();
db_upstream();
watch_fill();
up_changes();
dehsqa_db();
$endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
$messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";
mail($email, "DEHS report", $messaggio,
     "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .
     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .
     "X-Mailer: PHP/" . phpversion());
}
?>