--- trunk/dehs_pg.php 2007/12/18 20:49:31 84 +++ trunk/dehs_pg.php 2008/04/19 00:28:32 132 @@ -4,7 +4,7 @@ Originally written by Stefano Fabri Copyright 2004, Stefano Fabri -Copyright 2007 by Raphael Geissert +Copyright 2007, 2008 by Raphael Geissert This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -28,6 +28,7 @@ ini_set ( "memory_limit", "40M"); #print_R($_SERVER); require "dehs.config.inc"; +require "events.class.php"; dir_ctrl($dirs); if (!function_exists('file_get_contents')) { function file_get_contents($filename) { @@ -63,8 +64,8 @@ elseif (in_array("-dl_diffs",$argv) || in_array("-keep_diff",$argv)) dl_diffs($argv[2]); elseif (in_array("-keep_sources",$argv) || in_array("-download_sources",$argv)) download_sources(); elseif (in_array("-db_popcon",$argv)) db_popcon($argv[2]); - elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv)) dl_popcon($argv[2]); - elseif (in_array("-db_upstream",$argv)) db_upstream($argv[2]); + elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv)) dl_popcon(); + elseif (in_array("-db_upstream",$argv)) db_upstream((!empty($argv[2])?$argv[2]:'%')); elseif (in_array("-update_all",$argv)) update_all(); elseif (in_array("-update_new",$argv)) update_new(); elseif (in_array("-update_bogus",$argv)) update_bogus(); @@ -72,8 +73,8 @@ elseif (in_array("-dehsqa_db",$argv)) dehsqa_db(); elseif (in_array("-clear_db",$argv)) clear_db(); elseif (in_array("-db_up_error",$argv)) db_up_error(); - elseif (in_array("-up_changes",$argv)) up_changes($argv[2], true); - elseif (in_array("-watchf",$argv) || in_array("-wwiz",$argv)) watch_wizard((!empty($argv[2])?$argv[2]:'')); + elseif (in_array("-up_changes",$argv)) up_changes((!empty($argv[2])?$argv[2]:'%'), true); + elseif (in_array("-watchf",$argv) || in_array("-wwiz",$argv)) watch_wizard((!empty($argv[2])?$argv[2]:'%')); elseif (in_array("-kb_total",$argv)) print "Diff #stat_up_error(); #stat_up_error();to download Kb => " . kb_total($argv[2]) . "\n" ; @@ -81,6 +82,9 @@ else print "Command $argv[1] not found\n"; #stat_up_error(); function dir_ctrl($dirs) { + if (!is_array($dirs)) { + $dirs = array($dirs); + } foreach ($dirs as $dir) { if (!is_dir($dir)) exec("mkdir -m 0740 -p " . escapeshellarg($dir), $output, $res); if ($res != 0) die_status("Error creating basic dir: $dir\n"); @@ -132,7 +136,7 @@ #if ($res_array['dist']=='non-US') $conn_id = $conn_non_us; #else $conn_id = $conn_normal; - if ($localfile=download_diffs($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) { + if ($localfile=download_diff($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) { $watch=ext_watch($localfile,$res_array[name],$res_array[version]); unlink($localfile); if ($watch!=NULL) { @@ -150,7 +154,7 @@ pg_close($db); return "Fatto\n"; } -function download_diffs(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) { +function download_diff(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) { global $dirs,$mirrors; $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz"; #if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz"; @@ -168,7 +172,7 @@ print "Download Diff.gz of package $pkg => Fallito\n"; return false; } -function db_add($initial='', $clear_db = true) { +function db_add($initial='') { global $dirs,$dists,$dbconn; check_db(); download_sources(); @@ -184,7 +188,7 @@ $filename=$dirs['sources_dir'] . "/$dist/$section/Sources.gz"; print "\nI'm parsing $dist/$section => Sources.gz\n"; # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5 - $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S+)\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?/Ssi"; + $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S{32})\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?(Dm-Upload-Allowed:\s(.+)[^a])?(Homepage:\s(.+)[^a])?(Vcs-Browser:\s(.+)[^a])?(Vcs-([A-Z][a-z]+):\s(.+)[^a])?(Checksums-\S+:\s(.+)[^a])?/Ssi"; $zp = gzopen($filename, "r") or die_status("Could not open $filename"); $extracted=tempnam("/tmp/", "$dist_$section_Sources"); exec("gzip -c -d $filename > $extracted"); @@ -207,9 +211,21 @@ $matches[10]=pg_escape_string($matches[10]); } @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR die_status("Temp table pkgs_atsrc query error"); - $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]')") ; + $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders,homepage,vcs_browser,vcs_type,vcs) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]','$matches[14]','$matches[16]','$matches[18]','$matches[19]')"); if (!$rst) { - $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]',dversionmangled='$matches[4]',maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]' WHERE name='$matches[1]' AND dist='$dist'") OR die_status("\nDb adding error =>" . pg_last_error() . "\n"); + $rsql=pg_exec($db, "SELECT pkgs.version FROM pkgs WHERE name='$matches[1]' AND dist='$dist';"); + $version = $matches[4]; // just for safety + while ($res_array=pg_fetch_array($rsql)) { + $version = $res_array['version']; + } + $extra = ''; + if ($version != $matches[4]) { + // only update the dversionmangled field if the versions differ + $extra = ",dversionmangled='$matches[4]'"; + // and reset wwiz_type + $extra = ",wwiz_type=NULL"; + } + $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]'$extra,maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]',homepage='$matches[14]',vcs_browser='$matches[16]',vcs_type='$matches[18]',vcs='$matches[19]' WHERE name='$matches[1]' AND dist='$dist'") OR die_status("\nDb adding error =>" . pg_last_error() . "\n"); } $bin_names=split(",", $matches[2]); foreach ($bin_names as $bin_name) { @@ -230,16 +246,14 @@ gzclose($zp) ; } } - if ($clear_db) clear_db($db); pg_close($db); } function db_query($pkg,$dist='%') { global $dirs,$dbconn; $db = pg_pconnect($dbconn ) or die_status(pg_last_error($db)); - $rsql=pg_exec($db, "SELECT pkgs.* ,mpop_inst FROM (SELECT name,MAX(binpkgs.pop_inst) AS mpop_inst FROM binpkgs GROUP BY name) - AS binpkgs INNER JOIN pkgs ON pkgs.name=binpkgs.name - WHERE pkgs.name='$pkg' AND dist='$dist';"); + $rsql=pg_exec($db, "SELECT * FROM pkgs + WHERE name='$pkg' AND dist LIKE '$dist';"); while ($res_array=pg_fetch_array($rsql)) { print "Package=> " . $res_array['name'] . "\n"; print "Version => " . $res_array['version'] . "\n"; @@ -249,19 +263,19 @@ print "Distribution => " . $res_array['dist'] . "\n"; print "Section => " . $res_array['section'] . "\n"; print "KBytes => " . number_format ( $res_array['bytes']/1024, 2, ",",".") . "\n"; - print "Pop Inst=>" . $res_array['mpop_inst'] . "\n"; + print "Pop Inst=>" . $res_array['pop_inst'] . "\n"; if ($res_array['watch']) { $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['watch']); - print "Version Upstream => " . $uscan_res[0] . "\n"; + print "Version Upstream => " . $uscan_res['uversion'] . "\n"; print "Watch => " . trim($res_array['watch'])."\n"; - print "Watch Warnings => " . $uscan_res[1]; + print "Watch Warnings => " . $uscan_res['warnings']; } else print "Watch => Not present" . "\n"; if ($res_array['wwiz_type']) { $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['wwiz']); - print "WWiz Upstream => " . $uscan_res[0] . "\n"; + print "WWiz Upstream => " . $uscan_res['uversion'] . "\n"; print "WWiz => " . $res_array['wwiz']."\n"; - print "WWiz Warings => " . $uscan_res[1]; + print "WWiz Warings => " . $uscan_res['warnings']; } print "\n"; } @@ -310,7 +324,7 @@ # 183dea9ca3c2e35ffd6cf5eee9501c21 2941637 main/binary-i386/Packages.gz } -function vers_conv($debvers) { +function vers_conv($debvers, $extendedConv = true) { // Strip off the epoch $pos = strpos($debvers, ':'); if ( $pos !== FALSE) { @@ -323,28 +337,38 @@ $debvers = substr($debvers, 0, $pos); } - // strip off repacking indicators - $debvers = preg_replace("/[-.+~]?(ds|dfsg|debian)(.*)/i", "", $debvers); + if ($extendedConv) { + // strip off repacking indicators + $debvers = preg_replace("/[-.+~]?(ds|dfsg|debian)(.*)/i", "", $debvers); + } + return $debvers; } function check_db() { global $dirs,$dbconn; $db = pg_pconnect($dbconn) or die_status(1/*pg_last_notice($db)*/); - if (!pg_table_exists($db,"pkgs")){ - pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text)") or die_status('Error creating table pkgs\n'); + if (!pg_table_exists($db, "pkgs")) { + pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text, lastcheck timestamp, homepage text, vcs text, vcs_type text, vcs_browser text, lastupvsdebsync timestamp, pop_inst numeric,pop_vote numeric)") or die_status('Error creating table pkgs\n'); pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db)); + pg_exec($db, "CREATE INDEX checkx on pkgs (lastcheck)") or die_status('Errore creating index - ' . pg_last_error($db)); + pg_exec($db, "CREATE INDEX maintx on pkgs (maint)") or die_status('Errore creating index - ' . pg_last_error($db)); $sw=true; } - if (!pg_table_exists($db,"binpkgs")){ + if (!pg_table_exists($db, "binpkgs")) { pg_exec($db, "CREATE TABLE binpkgs (id serial PRIMARY KEY, name text,bin_name text,dist text, pop_inst numeric,pop_vote numeric, up_error numeric,avg_error_date numeric)") or die_status('Error creating table binpkgs\n'); pg_exec($db, "CREATE UNIQUE INDEX idxbin on binpkgs (name,bin_name,dist)") or die_status('Errore creating index - ' . pg_last_error($db)); $sw=true; } - /*if (!pg_table_exists($db,"events")){ - pg_exec($db, "CREATE TABLE events (id serial PRIMARY KEY, name text,type text, dist text)") or die_status('Error creating table binpkgs\n'); - pg_exec($db, "CREATE UNIQUE INDEX idxevents on events (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db)); + if (!pg_table_exists($db, "events")) { + pg_exec($db, "CREATE TABLE events (id serial PRIMARY KEY, name text,type text, dist text, stamp timestamp, content text)") or die_status('Error creating table events\n'); + pg_exec($db, "CREATE UNIQUE INDEX idxevents on events (name,dist,content)") or die_status('Error creating index - ' . pg_last_error($db)); + pg_exec($db, "CREATE INDEX pkgxeventtype on events (name,type,dist)") or die_status('Error creating index - ' . pg_last_error($db)); + $sw=true; + } + if (!pg_table_exists($db, "stamps")) { + pg_exec($db, "CREATE TABLE stamps (type text PRIMARY KEY, stamp timestamp, pid text)") or die_status('Error creating table stamps\n'); $sw=true; - }*/ + } #if (!pg_table_exists($db,"upstream")){ # pg_exec($db, "CREATE TABLE upstream (name text,dist text,up_version text,watch_warn text,updated bool)") or die_status('Errore nel creare la tabella'); # pg_exec($db, "CREATE UNIQUE INDEX idxupstrm on upstream (name,dist)") or die_status('Errore nel creare gli indici ' . pg_last_error($db)); @@ -389,87 +413,138 @@ $sql1= "Delete from binpkgs where id NOT IN (select id FROM binpkgs INNER JOIN bin_atsrc USING (name,bin_name,dist))" ; $sql2= "Delete from pkgs where id NOT IN (select id FROM pkgs INNER JOIN pkgs_atsrc USING (name,dist))" ; - $sql3= "UPDATE pkgs SET wwiz=NULL, wwiz_type=NULL, wwiz_version=NULL WHERE watch IS NOT NULL or watch!=''" ; - $sql4= "UPDATE pkgs SET up_changes=NULL WHERE wwiz_version=up_version OR up_version=version" ; - if(pg_exec($db,$sql1) AND pg_exec($db,$sql2) AND pg_exec($db,$sql3) AND pg_exec($db,$sql4)) print "Db clear complete\n"; + $sql3= "UPDATE pkgs SET wwiz=NULL, wwiz_type=NULL, wwiz_version=NULL WHERE watch IS NOT NULL and watch!=''" ; + $sql4= "UPDATE pkgs SET up_changes=NULL WHERE updated='1'"; + if(pg_exec($db,$sql1) && pg_exec($db,$sql2) && pg_exec($db,$sql3) && pg_exec($db,$sql4)) { + print "Db clear complete\n"; + } } function db_popcon($initial='') { global $dirs,$dbconn; $db = pg_pconnect($dbconn) or die_status(pg_last_error($db)); - $filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; - $regexp="/Package:\s($initial\S*)\s+(\d+)\s+(\d+)/iS"; - $zp = gzopen($filename, "r") or die_status("Could not open $filename"); - $tin=time(); - if(filesize($filename)>20) { - #pg_exec($db, "BEGIN TRANSACTION") OR die_status("Impossibile iniziare la transazione"); - while (!gzeof($zp)) { - $line = gzgets ($zp,4096) ; - if (preg_match($regexp,$line,$matches)) { - #print "\rName => $matches[1] Install => $matches[2] Vote => $matches[3]"; - @pg_exec($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'"); - /*$processed++; - $elapsed_time=time()-$tin; - if (($elapsed_time % 20)==0 AND $updtime!=$elapsed_time) { - $updtime=$elapsed_time; - $wprocessed=$processed-$wprocessed; - $avg_time=$wprocessed/20; - $wprocessed=$processed; - $countdown=(34701- $processed)/$avg_time; - $last_time=time(); - } - print "\rProcessing Popcon: " . date("i:s",$countdown=$countdown-(time()-$last_time)) . " =>" . (34701-$processed) . " Media $avg_time"; - $last_time=time();*/ - } + $filename=$dirs[popcon_dir] . "/popcon.db"; + $popcon_db = dba_open("$filename", 'r-', 'db4'); + + $rsql = pg_exec($db, "SELECT name FROM pkgs"); + + while ($res_array=pg_fetch_array($rsql)) { + $package = $res_array['name']; + + if (!dba_exists('si:' . $package, $popcon_db)) { + continue; } - #pg_exec($db, "COMMIT") OR die_status("Impossibile commsionare la transazione"); + + $inst = dba_fetch("si:$package", $popcon_db); + $vote = dba_fetch("sv:$package", $popcon_db); + + @pg_exec($db, "UPDATE pkgs SET pop_inst='$inst',pop_vote='$vote' WHERE name='$package'"); } + + dba_close($popcon_db); @pg_close($db); } function dl_popcon() { global $dirs,$dbconn; - $localfile=$dirs[popcon_dir] . "/all-popcon-results.txt.gz"; - $remotefile="http://popcon.debian.org/all-popcon-results.txt.gz"; - @file_put_contents($localfile,file_get_contents($remotefile),'w') or die_status("Errore nello scaricare il file popcon\n"); -} -function is_updated ($up_vers,$debian_vers) { - if ($up_vers=='') return 0; - passthru("dpkg --compare-versions '".escapeshellarg($up_vers)."' le '" . escapeshellarg(vers_conv($debian_vers)) . "'",$great); - if ($great!=0) return 1; - else return 0; + + $localtempdir=$dirs[popcon_dir] . "/new"; + dir_ctrl($localtempdir); + + $localfile=$dirs[popcon_dir] . "/popcon.db"; + $remotefile='http://qa.debian.org/data/popcon/popcon.db'; + + $return = 0; + $output = null; + exec('cd ' . escapeshellarg($localtempdir) . ' && ' . + '/usr/bin/wget -qNc ' . escapeshellarg($remotefile), $output, $return); + + if ($return != 0) { + echo __FUNCTION__ . "wget exited with status $return!\n"; + return false; + } + + return copy($localtempdir . '/popcon.db', $localfile); } -function db_upstream($initial='%') { - global $dirs,$dbconn; +function is_updated ($up_vers,$debian_vers, $dversionAsIs = false) { + if ($up_vers=='') return 1; + + $extendedConv = true; + + if ($up_vers === $debian_vers) $dversionAsIs = true; + + if (strpos(strtolower($up_vers), 'dfsg') !== false && strpos(strtolower($debian_vers), 'dfsg') !== false) { + $extendedConv = false; + } + + passthru("dpkg --compare-versions ".escapeshellarg($up_vers)." le " . escapeshellarg($dversionAsIs?$debian_vers:vers_conv($debian_vers, $extendedConv)),$great); + + return (int)!($great); +} +function db_upstream($initial='%', $checkStamps = false, $limit = null) { + global $dirs, $dbconn, $minimumAge; + check_db(); $db = pg_pconnect($dbconn) ; - $rsql=pg_exec($db, "SELECT id, pkgs.name,pkgs.dist,up_version,version,watch FROM pkgs WHERE watch is not null AND name LIKE '$initial%' ORDER BY name DESC"); + events::DB($db); + + $postWHERE = ''; + if ($checkStamps) { + $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL)"; + } + + $postORDER = ''; + if ($limit !== null && is_int($limit)) { + $postORDER = 'LIMIT ' . $limit; + } + + $rsql=pg_exec($db, "SELECT id, name,dist,up_version,version,watch,lastupvsdebsync FROM pkgs WHERE watch is not null AND name LIKE '$initial' $postWHERE ORDER BY lastcheck DESC $postORDER"); $tot=pg_numrows($rsql); + while ($res_array=pg_fetch_array($rsql)) { + $uscan_res=uscan_foo($res_array['name'],$res_array['version'],$res_array['watch']); - #print $uscan_res[0] . "\n Name => $res_array[name]\n"; - $version=$res_array[version]; - $dversionmangled=$res_array['version']; - print "\rVersion=>" . $uscan_res[3]; - if ($uscan_res[3]!=null) $version=$uscan_res[3]; - if ($uscan_res[5]!=null && $uscan_res[5] != vers_conv($res_array['version'])) $dversionmangled=$uscan_res[5]; - $updated=$uscan_res[4]; - if ($updated) { - $updated=is_updated($uscan_res[0],$dversionmangled); + + #print $uscan_res['uversion'] . "\n Name => $res_array[name]\n"; + $version = $res_array['version']; + $dversionmangled = $res_array['version']; + $lastupvsdebsync = $res_array['lastupvsdebsync']; + $mangle = true; + + print "\rVersion=>" . $uscan_res['dversion']; + if ($uscan_res['dversion']!=null) $version=$uscan_res['dversion']; + if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) { + $dversionmangled=$uscan_res['dversionmangled']; + $mangle = false; + } + + $up_url=iconv("ISO-8859-1","UTF-8",$uscan_res['uurl']); + $up_version=iconv("ISO-8859-1","UTF-8",$uscan_res['uversion']); + + $updated = $uscan_res['updated']; + + /*if (!$updated)*/ { + $updated=is_updated($uscan_res['uversion'], $dversionmangled, !$mangle); } if (!$updated) { - events::newVersion($res_array['name'], $res_array['version'], $uscan_res[0], $res_array['dist']); + events::newVersion($res_array['name'], $res_array['version'], $uscan_res['uversion'], $res_array['dist'], $up_url); + } else { + $lastupvsdebsync = sqldate(); + } + + if ($lastupvsdebsync === null) { + $lastupvsdebsync = sqldate(0); } - if ($uscan_res[0]!=$res_array[up_version]) ($keep_changes=1); + if ($uscan_res['uversion']!=$res_array[up_version]) ($keep_changes=1); else $keep_changes=0; - $up_url=iconv("ISO-8859-1","UTF-8",$uscan_res[2]); - $up_version=iconv("ISO-8859-1","UTF-8",$uscan_res[0]); + $watch_warn=''; - if($uscan_res[1] != null) { - $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8",$uscan_res[1])); + if($uscan_res['warnings'] != null) { + $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8", $uscan_res['warnings'])); } - pg_exec($db, "UPDATE pkgs SET up_version='" . pg_escape_string($up_version) . "', dversionmangled='".pg_escape_string($dversionmangled)."',up_url='" . pg_escape_string($up_url) . "', watch_warn='" . $watch_warn . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated' WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream"); - #print "\rUpdated " . $res_array['name'] . " => $updated Up Version => ". $uscan_res[0]; + + pg_exec($db, "UPDATE pkgs SET up_version='" . pg_escape_string($up_version) . "', dversionmangled='".pg_escape_string($dversionmangled)."',up_url='" . pg_escape_string($up_url) . "', watch_warn='" . $watch_warn . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated', lastcheck=now(), lastupvsdebsync='$lastupvsdebsync' WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream"); + #print "\rUpdated " . $res_array['name'] . " => $updated Up Version => ". $uscan_res['uversion']; print "\t Upstream left => " . $tot --; # } # else print "Added ". $res_array['name'] ."\n"; @@ -477,8 +552,8 @@ # print "Package=> " . $res_array['name'] . "\n"; # print "Version => " . $res_array['version'] . "\n"; - # print "Upstream Ver=> " . $uscan_res[0] . "\n"; - #print "Watch warings=> " . $uscan_res[1] . "\n"; + # print "Upstream Ver=> " . $uscan_res['uversion'] . "\n"; + #print "Watch warings=> " . $uscan_res['warnings'] . "\n"; # } } print "\n"; @@ -525,13 +600,13 @@ global $dbconn, $fetchUpstreamChanges; if (!$fetchUpstreamChanges && !$force) { - print "\nSkipping download of upstream sources to grab changelogs\n"; + //print "\nSkipping download of upstream sources to grab changelogs\n"; return; } $changes_files=array("CHANGES","NEWS","Changelog","ChangeLog","changelog"); $db = pg_pconnect($dbconn) ; - $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!='' AND keep_changes='t' AND name LIKE '$src_name%'"); + $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!='' AND keep_changes='t' AND name LIKE '$src_name'"); $tot=pg_numrows($rsql); while ($res_array=pg_fetch_array($rsql)) { $localfile=tempnam("/tmp/", "dehs-" . $res_array[id] . ".tar.gz"); @@ -596,8 +671,8 @@ $warnings=$vals[$index['WARNINGS'][0]][value]; $updated=$vals[$index['STATUS'][0]][value]; - if ($updated=="up to date") $updated=0; - else $updated=1; + if ($updated=="up to date") $updated=1; + else $updated=0; if ($upstream=='') $upstream=null; if ($up_url=='') $up_url=null; @@ -605,7 +680,15 @@ if ($warnings=='') $warnings=null; if ($deb_mangled_vers=='') $deb_mangled_vers=null; - return array($upstream, $warnings, $up_url, $deb_vers, $updated, $deb_mangled_vers); + $ret = array(); + $ret['uversion'] = $upstream; + $ret['warnings'] = $warnings; + $ret['uurl'] = $up_url; + $ret['dversion'] = $deb_vers; + $ret['dversionmangled'] = $deb_mangled_vers; + $ret['updated'] = $updated; + + return $ret; } function db_up_error() { global $dbconn; @@ -639,8 +722,8 @@ } pg_close($db); } -function watch_wizard($src_name='%'){ - global $dirs,$dbconn,$sfIgnore,$watchFileExts,$watchFileDversionMangle,$watchFormat; +function watch_wizard($src_name='%', $checkStamps = false, $limit = null){ + global $dirs,$dbconn,$sfIgnore,$watchFileExts,$watchFileDversionMangle,$watchFormat,$minimumAge; $db = pg_pconnect($dbconn) or die_status ("Db error"); $checked=$errors=$notfound=$verok=$notmatch=0; @@ -681,9 +764,19 @@ } } - $sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.dversionmangled,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN - (Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist) - WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name%' ORDER BY mpop_inst DESC"; + $postWHERE = ''; + if ($checkStamps) { + $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL)" + . " AND wwiz_type!='no_cright' AND wwiz_type!='notmatch'"; + } + + $postORDER = ''; + if ($limit !== null && is_int($limit)) { + $postORDER = 'LIMIT ' . $limit; + } + + $sqlstring="SELECT name,version,dversionmangled,section,dist, wwiz_version FROM pkgs + WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name' $postWHERE ORDER BY lastcheck DESC $postORDER"; $rsql=pg_exec($db, $sqlstring); while ($res_array=pg_fetch_array($rsql)) { @@ -766,7 +859,7 @@ if (empty($generatedWatches)) { $notmatch++; $contents=iconv("ISO-8859-1","UTF-8",$contents); - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); + pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($contents) . "', wwiz_type='notmatch', up_url='', up_changes='', lastcheck=now(), keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); } else { @@ -784,35 +877,35 @@ $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch); - if ($uscan_res[0] && levenshtein($uscan_res[0], $res_array['version']) < strlen($res_array['version'])) { + if ($uscan_res['uversion'] && levenshtein($uscan_res['uversion'], $res_array['version']) < strlen($res_array['version'])) { break; } } - if ($uscan_res[1] && !$uscan_res[0]) { - pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); + if ($uscan_res['warnings'] && !$uscan_res['uversion']) { + pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . pg_escape_string($uscan_res['warnings']) . "', wwiz_type='error', up_url='', up_changes='', lastcheck=now(), keep_changes='f' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); $errors++; } else { $verok++; - if ($uscan_res[0] != $res_array['wwiz_version']) { + if ($uscan_res['uversion'] != $res_array['wwiz_version']) { $keep_changes = 1; } else { $keep_changes = 0; } $dversionmangled = $res_array[version]; - if ($uscan_res[5]!=null && $uscan_res[5] != vers_conv($res_array['version'])) $dversionmangled=$uscan_res[5]; + if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) $dversionmangled=$uscan_res['dversionmangled']; - $updated=$uscan_res[4]; - if ($updated) { - $updated=is_updated($uscan_res[0],$dversionmangled); + $updated = $uscan_res['updated']; + /*if ($updated)*/ { + $updated = is_updated($uscan_res['uversion'], $dversionmangled, $dversionmangled != $uscan_res['dversionmangled']); } /*if (!$updated) { - events::newVersion($res_array['name'], $res_array['version'], $uscan_res[0], $res_array['dist']); + events::newVersion($res_array['name'], $res_array['version'], $uscan_res['uversion'], $res_array['dist']); }*/ - pg_exec($db, "UPDATE pkgs SET wwiz='" . pg_escape_string($watch) ."', wwiz_type='watch', wwiz_version='" . pg_escape_string($uscan_res[0]) . "',dversionmangled='".pg_escape_string($dversionmangled)."', up_url='" . pg_escape_string($uscan_res[2]) . "', keep_changes='$keep_changes', updated='$updated' where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); + pg_exec($db, "UPDATE pkgs SET wwiz='" . pg_escape_string($watch) ."', wwiz_type='watch', wwiz_version='" . pg_escape_string($uscan_res['uversion']) . "',dversionmangled='".pg_escape_string($dversionmangled)."', up_url='" . pg_escape_string($uscan_res['uurl']) . "', keep_changes='$keep_changes', updated='$updated', lastcheck=now() where name='" . pg_escape_string($res_array[name]) . "' AND dist='" . $res_array[dist] ."'"); } } } else { @@ -841,7 +934,7 @@ foreach ($dists as $dist) { $id=dbqa_conn($dir1 . "dehs_qa_" . $dist . ".db",'n'); $db = pg_pconnect($dbconn) or die_status ("Db error"); - $sql="SELECT name, id, up_version, wwiz_version, watch, wwiz_type, watch_warn, version, dversionmangled FROM pkgs WHERE dist='$dist'"; + $sql="SELECT name, id, up_version, wwiz_version, watch, wwiz_type, watch_warn, version, dversionmangled, updated FROM pkgs WHERE dist='$dist'"; $rsql=pg_exec($db, $sql); while($res_array=pg_fetch_array($rsql)) { @@ -852,10 +945,13 @@ if ($res_array['wwiz_type']=="watch") $wwiz=$res_array['wwiz_version']; else $wwiz=$res_array[wwiz_type]; + $updated = $res_array['updated']? 'yes' : 'no'; + $xml="$res_array[id]" . "$version" . "$res_array[version]" . "$res_array[dversionmangled]" . + "$updated" . "$wwiz" . ""; $value=addslashes($xml); @@ -874,15 +970,14 @@ } } function update_all() { - global $email; + global $email, $timeStamp_basedCheck; $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + echo __FUNCTION__ . ' started the ' . $intime . "\n"; db_add(); - dl_popcon(); - db_popcon(); dl_diffs(); db_up_error(); - db_upstream(); - watch_wizard(); + db_upstream('%', in_array('all', $timeStamp_basedCheck)); + watch_wizard('%', in_array('all', $timeStamp_basedCheck)); up_changes(); dehsqa_db(); $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); @@ -895,14 +990,13 @@ function update_new() { global $email_all; $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); - db_add('',false); - dl_popcon(); - db_popcon(); + echo __FUNCTION__ . ' started the ' . $intime . "\n"; + db_add(); global $dbconn; check_db(); $db = pg_pconnect($dbconn); - $sql="SELECT name FROM pkgs WHERE (md5_diff!=md5_atsource or md5_diff is null)"; + $sql="SELECT name FROM pkgs WHERE (md5_diff!=md5_atsource or md5_diff is null) or (lastcheck is null and watch is not null)"; $rsql=pg_exec($db, $sql); $res_array=pg_fetch_all($rsql); pg_close($db); @@ -927,6 +1021,7 @@ function update_bogus() { global $email_all,$dbconn; $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2); + echo __FUNCTION__ . ' started the ' . $intime . "\n"; check_db(); $db = pg_pconnect($dbconn); $sql="SELECT name FROM pkgs WHERE up_version='' AND watch!=''"; @@ -960,76 +1055,14 @@ } else if (is_int($msg) && $status == 1 /* default */) { die($msg); } else { - echo $msg; + fwrite(STDERR, $msg); die($status); } } -class events -{ - function newVersion($sourcePackage, $currentVersion, $newVersion, $distribution, $downloadLink = '') - { - global $base, $sendNotifications; - - if (!$sendNotifications) { - return null; - } - - $tpl = file_get_contents($base . '/tpls/new_version.mail.tpl'); - - $search = array(); - $replace = array(); - - $search [] = '#PACKAGE#'; - $replace[] = $sourcePackage; - $search [] = '#DVERSION#'; - $replace[] = $currentVersion; - $search [] = '#UVERSION#'; - $replace[] = $newVersion; - $search [] = '#DISTRO#'; - $replace[] = $distribution; - - $message = str_replace($search, $replace, $tpl); - $subject = events::_extractHeader('subject', $message); - $to = events::_extractHeader('to', $message); - - $extraHeaders = events::_getTefaultExtraHeaders(); - $extraHeaders[] = 'X-DEHS-UVERSION: ' . $newVersion; - - return mail($to, $subject, $message, implode("\r\n", $extraHeaders)); - } - - function _extractHeader($header, &$message) - { - $regex = '/^' . preg_quote($header, '/') . ':([ \t]*)(.*)$/i'; - - $matches = array(); - - if (preg_match($regex, $message, $matches)) { - - $spacer = $matches[1]; - $value = $matches[2]; - - $message = ltrim(preg_replace($regex, '', $message)); - - return $value; - - } - return false; - } - - function _getTefaultExtraHeaders() - { - global $notificationsMailFrom; - $headers = array(); - - $headers[] = 'From: ' . $notificationsMailFrom; - $headers[] = 'X-PTS-Approved: sure'; - $headers[] = 'X-BY-DEHS: sure'; - - return $headers; - } - +function sqldate($timestamp=null) { + if ($timestamp===null) $timestamp=time(); + return gmdate("Y-m-d H:i:s", $timestamp); } ?> \ No newline at end of file