/[dehs]/trunk/dehs_pg.php
ViewVC logotype

Diff of /trunk/dehs_pg.php

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 131 by atomo64-guest, Fri Apr 18 23:47:44 2008 UTC revision 187 by geissert, Fri Sep 24 15:33:49 2010 UTC
# Line 1  Line 1 
1  #!/usr/bin/php4 -q  #!/usr/bin/php -q
2  <?php  <?php
3  /*  /*
4    
# Line 25  Foundation, Inc., 59 Temple Place, Suite Line 25  Foundation, Inc., 59 Temple Place, Suite
25  #ini_set ( "error_reporting", "E_COMPILE_ERROR|E_ERROR|E_CORE_ERROR");  #ini_set ( "error_reporting", "E_COMPILE_ERROR|E_ERROR|E_CORE_ERROR");
26  ini_set ("html_errors", "off");  ini_set ("html_errors", "off");
27  set_time_limit(0);  set_time_limit(0);
28  ini_set ( "memory_limit", "40M");  ini_set ( "memory_limit", "80M");
29  #print_R($_SERVER);  #print_R($_SERVER);
30  require "dehs.config.inc";  require "dehs.config.inc";
31  require "events.class.php";  require "events.class.php";
32    # Use PEAR: Archive_Tar to uncompress 3.0 debian.tar.gz files
33    require "PEAR.php";
34    require "Archive/Tar.php";
35  dir_ctrl($dirs);  dir_ctrl($dirs);
36  if (!function_exists('file_get_contents')) {  if (!function_exists('file_get_contents')) {
37      function file_get_contents($filename) {      function file_get_contents($filename) {
# Line 63  if (in_array("-db_add",$argv))  db_add(( Line 66  if (in_array("-db_add",$argv))  db_add((
66      elseif (in_array("-db_query",$argv))  db_query($argv[2],(!empty($argv[3])?$argv[3]:'%') );      elseif (in_array("-db_query",$argv))  db_query($argv[2],(!empty($argv[3])?$argv[3]:'%') );
67      elseif (in_array("-dl_diffs",$argv) || in_array("-keep_diff",$argv))  dl_diffs($argv[2]);      elseif (in_array("-dl_diffs",$argv) || in_array("-keep_diff",$argv))  dl_diffs($argv[2]);
68      elseif (in_array("-keep_sources",$argv) || in_array("-download_sources",$argv))  download_sources();      elseif (in_array("-keep_sources",$argv) || in_array("-download_sources",$argv))  download_sources();
69      elseif (in_array("-db_popcon",$argv))  db_popcon($argv[2]);      elseif (in_array("-db_popcon",$argv))  db_popcon();
70      elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv))  dl_popcon();      elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv))  dl_popcon();
71      elseif (in_array("-db_upstream",$argv))  db_upstream((!empty($argv[2])?$argv[2]:'%'));      elseif (in_array("-db_upstream",$argv))  db_upstream((!empty($argv[2])?$argv[2]:'%'));
72        elseif (in_array("-dl_from_vcs",$argv))  dl_from_vcs((!empty($argv[2])?$argv[2]:'%'),(!empty($argv[3])?$argv[3]:'%'));
73      elseif (in_array("-update_all",$argv))  update_all();      elseif (in_array("-update_all",$argv))  update_all();
74      elseif (in_array("-update_new",$argv))  update_new();      elseif (in_array("-update_new",$argv))  update_new();
75      elseif (in_array("-update_bogus",$argv))  update_bogus();      elseif (in_array("-update_bogus",$argv))  update_bogus();
# Line 82  if (in_array("-db_add",$argv))  db_add(( Line 86  if (in_array("-db_add",$argv))  db_add((
86      else print "Command $argv[1] not found\n";      else print "Command $argv[1] not found\n";
87  #stat_up_error();  #stat_up_error();
88  function dir_ctrl($dirs) {  function dir_ctrl($dirs) {
89        if (!is_array($dirs)) {
90            $dirs = array($dirs);
91        }
92      foreach ($dirs as $dir) {      foreach ($dirs as $dir) {
93          if (!is_dir($dir)) exec("mkdir -m 0740 -p " .  escapeshellarg($dir), $output, $res);          if (!is_dir($dir)) exec("mkdir -m 0740 -p " .  escapeshellarg($dir), $output, $res);
94          if ($res != 0) die_status("Error creating basic dir: $dir\n");          if ($res != 0) die_status("Error creating basic dir: $dir\n");
# Line 89  function dir_ctrl($dirs) { Line 96  function dir_ctrl($dirs) {
96  }  }
97  function ext_watch ($file,$pkg,$version) {  function ext_watch ($file,$pkg,$version) {
98      if (!file_exists($file)) return "";      if (!file_exists($file)) return "";
99      $zp=gzopen($file,"r") or die_status("Fail to open $file");      if (substr($file,-8) == ".diff.gz") {
100      while (!gzeof($zp)) {          // Format: 1.0
101          $line = gzgets ($zp,4096);          $zp=gzopen($file,"r") or die_status("Fail to open $file");
102          if (preg_match('/\+\+\+\s.+\/debian\/watch$/',$line)) {          while (!gzeof($zp)) {
103              $line = gzgets ($zp,4096);              $line = gzgets ($zp,4096);
104              preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght);              if (preg_match('/\+\+\+\s.+\/debian\/watch$/',$line)) {
105              for ($i = 1; $i <= $watch_lenght[1]; $i++)      {                  $line = gzgets ($zp,4096);
106                  $line=gzgets ($zp,4096);                  preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght);
107                  $watch .=substr($line,1);                  for ($i = 1; $i <= $watch_lenght[1]; $i++)      {
108                        $line=gzgets ($zp,4096);
109                        $watch .=substr($line,1);
110                    }
111                    break;
112              }              }
             break;  
113          }          }
114      }      }
115        elseif (preg_match('/\.debian\.tar\.(gz|bz2)$/', $file, $matches)) {
116            // Format 3.0 (*)
117            $AT = new Archive_Tar($file,$matches[1]);
118            $watch = $AT->extractInString('debian/watch');
119        }
120      if ($watch!=null) return $watch;      if ($watch!=null) return $watch;
121      else return NULL;      else return NULL;
122  }  }
# Line 128  function dl_diffs($initial = '') { Line 143  function dl_diffs($initial = '') {
143      $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR die_status("Error in diff query\n");      $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR die_status("Error in diff query\n");
144      $total=pg_numrows($rsql);      $total=pg_numrows($rsql);
145      $btotal=byte_total($db,$initial);      $btotal=byte_total($db,$initial);
146      print "Total Diff to downlaod Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n";      print "Total Diff to download Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n";
147      while ($res_array=pg_fetch_array($rsql)) {      while ($res_array=pg_fetch_array($rsql)) {
148          #if ($res_array['dist']=='non-US') $conn_id = $conn_non_us;          #if ($res_array['dist']=='non-US') $conn_id = $conn_non_us;
149          #else          #else
# Line 153  function dl_diffs($initial = '') { Line 168  function dl_diffs($initial = '') {
168  }  }
169  function download_diff(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) {  function download_diff(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) {
170      global $dirs,$mirrors;      global $dirs,$mirrors;
171      $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz";      $diffTypes = array(".diff.gz",".debian.tar.gz",".debian.tar.bz2");
172      #if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz";      foreach($diffTypes as $diffType) {
173      #else          $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . $version . $diffType;
174      $remotefile="/debian/$directory/$pkg" . "_" . "$version.diff.gz";          $remotefile="/debian/$directory/$pkg" . "_" . "$version" . $diffType;
175      #print "$localfile - $remotefile\n";          if      (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
176      if      (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {              #    print "$pkg Diff.gz => Downloaded\n";
177          #    print "$pkg Diff.gz => Downloaded\n";              if(md5_of_file($localfile)==$md5_atsource) {
178          if(md5_of_file($localfile)==$md5_atsource) {                  #print "MD5 => OK\n";
179              #print "MD5 => OK\n";                  #unlink($localfile);
180              #unlink($localfile);                  return $localfile;
181              return $localfile;              }
182          }          }
183      }      }
184      print "Download Diff.gz of package $pkg => Fallito\n";      print "Download Diff.gz of package $pkg => Fallito\n";
# Line 181  function db_add($initial='') { Line 196  function db_add($initial='') {
196      pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die_status('Error creating index on temp table bin_atsrc');      pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die_status('Error creating index on temp table bin_atsrc');
197    
198      foreach ($dists as $dist=>$sections) {      foreach ($dists as $dist=>$sections) {
199            $pkgs = array();
200            $pkgsections = array();
201          foreach ($sections as $section) {          foreach ($sections as $section) {
202              $filename=$dirs['sources_dir'] . "/$dist/$section/Sources.gz";              $filename=$dirs['sources_dir'] . "/$dist/$section/Sources.gz";
203              print "\nI'm parsing $dist/$section => Sources.gz\n";              print "\nI'm parsing $dist/$section => Sources.gz\n";
204              # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5  
205              $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S{32})\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?(Dm-Upload-Allowed:\s(.+)[^a])?(Homepage:\s(.+)[^a])?(Vcs-Browser:\s(.+)[^a])?(Vcs-([A-Z][a-z]+):\s(.+)[^a])?(Checksums-\S+:\s(.+)[^a])?/Ssi";              // Big regexp to parse a package entry
206    
207                // [1] Source package name
208                $regexp  = "/Package:\s($initial.*)[^a]";
209                // [2]  : Binary package name
210                $regexp .= "Binary:\s(.+)[^a]";
211                // [3]  : Epoch
212                // [4]  : Version
213                $regexp .= "Version:\s(\d+:)?(.+)[^a]";
214                $regexp .= "Priority.+";
215                // [5]  : Maintainer email address
216                $regexp .= "Maintainer:.+<([^>]+)>[^a]";
217                $regexp .= ".+";
218                // [6]  : Source package format
219                $regexp .= "Format:\s(.+)[^a]";
220                // [7]  : Package directory
221                $regexp .= "Directory:\s(.+)[^a]";
222                // [8]  : md5sum of the .diff.gz (or .debian.tar.(gz|bz2))
223                // [9]  : Size of the file in bytes
224                // [10] : Extension (.diff.gz or .debian.tar.gz)
225                $regexp .= "Files:.+\s(\S{32})\s(\d+)\s" . '\1\S+' . "(\.diff\.gz|\.debian\.tar\.(?:gz|bz2))[^a]";
226                // [11] : Uploaders field
227                // [12] : Uploaders value
228                $regexp .= "(Uploaders:\s(.+)[^a])?";
229                // [13] : DM-Upload-Allowed field
230                // [14] : DM-Upload-Allowed value
231                $regexp .= "(Dm-Upload-Allowed:\s(.+)[^a])?";
232                // [15] : Homepage field
233                // [16] : Homepage value
234                $regexp .= "(Homepage:\s(.+)[^a])?";
235                // [17] : Vcs-Browser field
236                // [18] : Vcs-Browser value
237                $regexp .= "(Vcs-Browser:\s(.+)[^a])?";
238                // [19] : Vcs-* field
239                // [20] :     vcs type
240                // [21] : Vcs-* value
241                $regexp .= "(Vcs-([A-Z][a-z]+):\s(.+)[^a])?";
242                // [21] : Checksums-* field
243                // [22] : Checksums-* value
244                $regexp .= "(Checksums-\S+:\s(.+)[^a])?";
245                $regexp .= "/Ssi";
246              $zp = gzopen($filename, "r") or die_status("Could not open $filename");              $zp = gzopen($filename, "r") or die_status("Could not open $filename");
247              $extracted=tempnam("/tmp/", "$dist_$section_Sources");              $extracted=tempnam("/tmp/", "$dist_$section_Sources");
248              exec("gzip -c -d $filename > $extracted");              exec("gzip -c -d $filename > $extracted");
# Line 199  function db_add($initial='') { Line 256  function db_add($initial='') {
256                          $buff1 .= $line;                          $buff1 .= $line;
257                      }                      }
258                      if (preg_match($regexp,$buff1,$matches))  {                      if (preg_match($regexp,$buff1,$matches))  {
259                          if ($matches[5]) {                          $matches[4] = ((isset($matches[3]) && strlen($matches[3]))? $matches[3] : ''). $matches[4];
260                              $matches[5]=iconv("ISO-8859-1","UTF-8",$matches[5]);                          if (!isset($pkgs[$matches[1]]) || is_updated($pkgs[$matches[1]][4],$matches[4],true)) {
261                              $matches[5]=pg_escape_string($matches[5]);                              unset($matches[0]);
262                          }                              $pkgs[$matches[1]] = $matches;
263                          if ($matches[10]) {                              $pkgsections[$matches[1]] = $section;
264                              $matches[10]=iconv("ISO-8859-1","UTF-8",$matches[10]);                          }
                             $matches[10]=pg_escape_string($matches[10]);  
                         }  
                         @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR die_status("Temp table pkgs_atsrc query error");  
                         $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders,homepage,vcs_browser,vcs_type,vcs) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]','$matches[14]','$matches[16]','$matches[18]','$matches[19]')");  
                         if (!$rst) {  
                             $rsql=pg_exec($db, "SELECT pkgs.version FROM pkgs WHERE name='$matches[1]' AND dist='$dist';");  
                             $version = $matches[4]; // just for safety  
                             while ($res_array=pg_fetch_array($rsql)) {  
                                 $version = $res_array['version'];  
                             }  
                             $extra = '';  
                             if ($version != $matches[4]) {  
                                 // only update the dversionmangled field if the versions differ  
                                 $extra = ",dversionmangled='$matches[4]'";  
                                 // and reset wwiz_type  
                                 $extra = ",wwiz_type=NULL";  
                             }  
                             $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]'$extra,maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]',homepage='$matches[14]',vcs_browser='$matches[16]',vcs_type='$matches[18]',vcs='$matches[19]' WHERE name='$matches[1]' AND dist='$dist'") OR die_status("\nDb adding error =>" . pg_last_error() . "\n");  
                         }  
                         $bin_names=split(",", $matches[2]);  
                         foreach ($bin_names as $bin_name) {  
                             pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR die_status("Temp table pkgs_atsrc query error");  
                             $rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ;  
                         }  
265                      }                      }
266                      else {                      else {
267                          preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches);                          preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches);
# Line 242  function db_add($initial='') { Line 275  function db_add($initial='') {
275              }              }
276              gzclose($zp) ;              gzclose($zp) ;
277          }          }
278            foreach ($pkgs as $package=>$matches) {
279                $section = $pkgsections[$matches[1]];
280                // Maintainer email address
281                if ($matches[5]) {
282                    $matches[5]=iconv("ISO-8859-1","UTF-8",$matches[5]);
283                    $matches[5]=pg_escape_string($matches[5]);
284                }
285                // Uploaders
286                if ($matches[12]) {
287                    $matches[12]=iconv("ISO-8859-1","UTF-8",$matches[12]);
288                    $matches[12]=pg_escape_string($matches[12]);
289                }
290                @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR die_status("Temp table pkgs_atsrc query error");
291                $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders,homepage,vcs_browser,vcs_type,vcs,vcsoversource,lastpkgsourcesupdate) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[7]','$matches[8]','$matches[9]','$dist','$section','$matches[12]','$matches[16]','$matches[18]','$matches[20]','$matches[21]','0',now())");
292                if (!$rst) {
293                    $rsql=pg_exec($db, "SELECT pkgs.version FROM pkgs WHERE name='$matches[1]' AND dist='$dist';");
294                    $version = $matches[4]; // just for safety
295                    while ($res_array=pg_fetch_array($rsql)) {
296                        $version = $res_array['version'];
297                    }
298                    $extra = '';
299                    if ($version != $matches[4]) {
300                        // only update the dversionmangled field if the versions differ
301                        $extra = ",dversionmangled='$matches[4]'";
302                        // also reset wwiz_type
303                        $extra = ",wwiz_type=NULL";
304                        // and update the lastpkgsourcesupdate field
305                        $extra = ",lastpkgsourcesupdate=now()";
306                    }
307                    $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]'$extra,maint='$matches[5]',dir='$matches[7]',md5_atsource='$matches[8]',bytes='$matches[9]',dist='$dist',section='$section',uploaders='$matches[12]',homepage='$matches[16]',vcs_browser='$matches[18]',vcs_type='$matches[20]',vcs='$matches[21]' WHERE name='$matches[1]' AND dist='$dist'") OR die_status("\nDb adding error =>" . pg_last_error() . "\n");
308                }
309                $bin_names=split(",", $matches[2]);
310                foreach ($bin_names as $bin_name) {
311                    pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR die_status("Temp table pkgs_atsrc query error");
312                    $rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ;
313                }
314            }
315      }      }
316      clear_db($db);      clear_db($db);
317      pg_close($db);      pg_close($db);
# Line 315  function download_sources () { Line 385  function download_sources () {
385  }  }
386  function md5_source($section,$dist) {  function md5_source($section,$dist) {
387      global $mirrors;      global $mirrors;
388      $relase=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die_status("Failed to check MD5 of sources");      $release=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die_status("Failed to check MD5 of sources");
389      preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $relase,$matches);      preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $release,$matches);
390      return array("md5"=>$matches[1],"bytes"=>$matches[2]);      return array("md5"=>$matches[1],"bytes"=>$matches[2]);
391      # 183dea9ca3c2e35ffd6cf5eee9501c21          2941637 main/binary-i386/Packages.gz      # 183dea9ca3c2e35ffd6cf5eee9501c21          2941637 main/binary-i386/Packages.gz
392  }  }
# Line 345  function check_db() { Line 415  function check_db() {
415      global $dirs,$dbconn;      global $dirs,$dbconn;
416      $db = pg_pconnect($dbconn) or die_status(1/*pg_last_notice($db)*/);      $db = pg_pconnect($dbconn) or die_status(1/*pg_last_notice($db)*/);
417      if (!pg_table_exists($db, "pkgs")) {      if (!pg_table_exists($db, "pkgs")) {
418          pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text, lastcheck timestamp, homepage text, vcs text,  vcs_type text, vcs_browser text, lastupvsdebsync timestamp, pop_inst numeric,pop_vote numeric)") or die_status('Error creating table pkgs\n');          pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text, lastcheck timestamp, homepage text, vcs text,  vcs_type text, vcs_browser text,vcs_watch text, vcsoversource bool,vcs_up_version text,vcs_dversionmangled text,vcs_up_url text, vcs_watch_warn text, vcs_updated bool, lastvcsfetch timestamp, lastupvsdebsync timestamp, lastpkgsourcesupdate timestamp, failures_count numeric, pop_inst numeric,pop_vote numeric)") or die_status('Error creating table pkgs\n');
419          pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db));          pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db));
420          pg_exec($db, "CREATE INDEX checkx on pkgs (lastcheck)") or die_status('Errore creating index - ' . pg_last_error($db));          pg_exec($db, "CREATE INDEX checkx on pkgs (lastcheck)") or die_status('Errore creating index - ' . pg_last_error($db));
421            pg_exec($db, "CREATE INDEX updatex on pkgs (lastcheck,lastpkgsourcesupdate);") or die_status('Errore creating index - ' . pg_last_error($db));
422          pg_exec($db, "CREATE INDEX maintx on pkgs (maint)") or die_status('Errore creating index - ' . pg_last_error($db));          pg_exec($db, "CREATE INDEX maintx on pkgs (maint)") or die_status('Errore creating index - ' . pg_last_error($db));
423          $sw=true;          $sw=true;
424      }      }
# Line 416  function clear_db(&$db) { Line 487  function clear_db(&$db) {
487          print "Db clear complete\n";          print "Db clear complete\n";
488      }      }
489  }  }
490  function db_popcon($initial='') {  function db_popcon() {
491      global $dirs,$dbconn;      global $dirs,$dbconn;
492      $db = pg_pconnect($dbconn)  or  die_status(pg_last_error($db));      $db = pg_pconnect($dbconn)  or  die_status(pg_last_error($db));
493      $filename=$dirs[popcon_dir] . "/popcon.db";      $filename=$dirs['popcon_dir'] . "/popcon.db";
494      $popcon_db = dba_open("$filename", 'r-', 'db4');      $popcon_db = dba_open($filename, 'r-', 'db4');
495    
496      $rsql = pg_exec($db, "SELECT name FROM pkgs");      $rsql = pg_exec($db, "SELECT name FROM pkgs");
497    
# Line 434  function db_popcon($initial='') { Line 505  function db_popcon($initial='') {
505          $inst = dba_fetch("si:$package", $popcon_db);          $inst = dba_fetch("si:$package", $popcon_db);
506          $vote = dba_fetch("sv:$package", $popcon_db);          $vote = dba_fetch("sv:$package", $popcon_db);
507    
508            // when one of the values is 'missing' set to -1 so the other value
509            //  _is_ UPDATEd, but use -1 to catch those bugs
510            $inst = (is_numeric($inst))? $inst : -1;
511            $vote = (is_numeric($vote))? $vote : -1;
512    
513          @pg_exec($db, "UPDATE pkgs SET pop_inst='$inst',pop_vote='$vote' WHERE name='$package'");          @pg_exec($db, "UPDATE pkgs SET pop_inst='$inst',pop_vote='$vote' WHERE name='$package'");
514      }      }
515    
# Line 443  function db_popcon($initial='') { Line 519  function db_popcon($initial='') {
519  function dl_popcon() {  function dl_popcon() {
520      global $dirs,$dbconn;      global $dirs,$dbconn;
521    
522      $localtempdir=$dirs[popcon_dir] . "/new";      $localtempdir=$dirs['popcon_dir'] . "/new";
523      dir_ctrl($localtempdir);      dir_ctrl($localtempdir);
524    
525      $localfile=$dirs[popcon_dir] . "/popcon.db";      $localfile=$dirs['popcon_dir'] . "/popcon.db";
526      $remotefile='http://qa.debian.org/data/popcon/popcon.db';      $remotefile='http://qa.debian.org/data/popcon/popcon.db';
527    
528      $return = 0;      $return = 0;
529      $output = null;      $output = null;
530      exec('cd ' . escapeshellarg($localtempdir) . ' && ' .      exec('cd ' . escapeshellarg($localtempdir) . ' && ' .
531           '/usr/bin/wget -qNc ' . escapeshellarg($remotefile), $output, $return);           '/usr/bin/wget -qN ' . escapeshellarg($remotefile), $output, $return);
532    
533      if ($return != 0) {      if ($return != 0) {
534          echo __FUNCTION__ . "wget exited with status $return!\n";          echo __FUNCTION__ . "wget exited with status $return!\n";
# Line 472  function is_updated ($up_vers,$debian_ve Line 548  function is_updated ($up_vers,$debian_ve
548          $extendedConv = false;          $extendedConv = false;
549      }      }
550    
551      passthru("dpkg --compare-versions ".escapeshellarg($up_vers)." le " . escapeshellarg($dversionAsIs?$debian_vers:vers_conv($debian_vers, $extendedConv)),$great);      passthru("dpkg --compare-versions -- ".escapeshellarg($up_vers)." le " . escapeshellarg($dversionAsIs?$debian_vers:vers_conv($debian_vers, $extendedConv)),$great);
552    
553      return (int)!($great);      return (int)!($great);
554  }  }
# Line 485  function db_upstream($initial='%', $chec Line 561  function db_upstream($initial='%', $chec
561    
562      $postWHERE = '';      $postWHERE = '';
563      if ($checkStamps) {      if ($checkStamps) {
564          $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL)";          $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL OR (lastpkgsourcesupdate>lastcheck))";
565      }      }
566    
567      $postORDER = '';      $postORDER = '';
# Line 493  function db_upstream($initial='%', $chec Line 569  function db_upstream($initial='%', $chec
569          $postORDER = 'LIMIT ' . $limit;          $postORDER = 'LIMIT ' . $limit;
570      }      }
571    
572      $rsql=pg_exec($db, "SELECT id, name,dist,up_version,version,watch,lastupvsdebsync FROM pkgs WHERE watch is not null AND name LIKE '$initial' $postWHERE ORDER BY lastcheck DESC $postORDER");      $rsql=pg_exec($db, "SELECT id, name,dist,up_version,version,watch,lastupvsdebsync,failures_count FROM pkgs WHERE watch is not null AND name LIKE '$initial' $postWHERE ORDER BY lastcheck DESC $postORDER");
573      $tot=pg_numrows($rsql);      $tot=pg_numrows($rsql);
574    
575      while ($res_array=pg_fetch_array($rsql)) {      while ($res_array=pg_fetch_array($rsql)) {
# Line 506  function db_upstream($initial='%', $chec Line 582  function db_upstream($initial='%', $chec
582          $lastupvsdebsync = $res_array['lastupvsdebsync'];          $lastupvsdebsync = $res_array['lastupvsdebsync'];
583          $mangle = true;          $mangle = true;
584    
585            /* Do not increase the number of failures if uscan didn't throw any warning
586                because it might be that the watch file is "empty"
587            */
588            if (!$uscan_res['uversion'] && $uscan_res['warnings']) {
589                $res_array['failures_count']++;
590            } else {
591                $res_array['failures_count'] = 0;
592            }
593    
594          print "\rVersion=>" . $uscan_res['dversion'];          print "\rVersion=>" . $uscan_res['dversion'];
595          if ($uscan_res['dversion']!=null) $version=$uscan_res['dversion'];          if ($uscan_res['dversion']!=null) $version=$uscan_res['dversion'];
596          if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) {          if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) {
# Line 523  function db_upstream($initial='%', $chec Line 608  function db_upstream($initial='%', $chec
608          }          }
609    
610          if (!$updated) {          if (!$updated) {
611              events::newVersion($res_array['name'], $res_array['version'], $uscan_res['uversion'], $res_array['dist'], $up_url);              events::newVersion($res_array['name'], $res_array['version'], $up_version, $res_array['dist'], $up_url);
612          } else {          } else {
613              $lastupvsdebsync = sqldate();              $lastupvsdebsync = sqldate();
614          }          }
# Line 532  function db_upstream($initial='%', $chec Line 617  function db_upstream($initial='%', $chec
617              $lastupvsdebsync = sqldate(0);              $lastupvsdebsync = sqldate(0);
618          }          }
619    
620          if ($uscan_res['uversion']!=$res_array[up_version]) ($keep_changes=1);          $keep_changes = (int) ($up_version != $res_array['up_version']);
         else $keep_changes=0;  
621    
622          $watch_warn='';          $watch_warn='';
623          if($uscan_res['warnings'] != null) {          if($uscan_res['warnings'] != null) {
624              $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8", $uscan_res['warnings']));              $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8", $uscan_res['warnings']));
625          }          }
626    
627          pg_exec($db, "UPDATE pkgs SET up_version='" . pg_escape_string($up_version) . "', dversionmangled='".pg_escape_string($dversionmangled)."',up_url='" . pg_escape_string($up_url) . "', watch_warn='" . $watch_warn . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated', lastcheck=now(), lastupvsdebsync='$lastupvsdebsync'  WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream");          pg_exec($db, "UPDATE pkgs SET
628                                        up_version='" . pg_escape_string($up_version) . "',
629                                        dversionmangled='".pg_escape_string($dversionmangled)."',
630                                        up_url='" . pg_escape_string($up_url) . "',
631                                        watch_warn='" . $watch_warn . "',
632                                        keep_changes='$keep_changes',
633                                        wwiz_version=NULL,
634                                        wwiz_type=NULL,
635                                        wwiz=NULL,
636                                        failures_count=$res_array[failures_count],
637                                        updated='$updated',
638                                        lastcheck=now(),
639                                        lastupvsdebsync='$lastupvsdebsync'
640                                      WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream");
641    
642          #print "\rUpdated " . $res_array['name'] .  " => $updated Up Version => ". $uscan_res['uversion'];          #print "\rUpdated " . $res_array['name'] .  " => $updated Up Version => ". $uscan_res['uversion'];
643          print "\t Upstream left => " . $tot --;          print "\t Upstream left => " . $tot --;
644          #                               }          #                               }
# Line 661  function uscan_foo($name,$version,$watch Line 759  function uscan_foo($name,$version,$watch
759      xml_parse_into_struct($p, $uscan_res, $vals, $index);      xml_parse_into_struct($p, $uscan_res, $vals, $index);
760      xml_parser_free($p);      xml_parser_free($p);
761    
762      $upstream=$vals[$index['UPSTREAM-VERSION'][0]][value];      list($upstream)=explode("\n", $vals[$index['UPSTREAM-VERSION'][0]][value]);
763      $up_url=$vals[$index['UPSTREAM-URL'][0]][value];      list($up_url)=explode("\n", $vals[$index['UPSTREAM-URL'][0]][value]);
764      $deb_vers=$vals[$index['DEBIAN-UVERSION'][0]][value];      $deb_vers=$vals[$index['DEBIAN-UVERSION'][0]][value];
765      $deb_mangled_vers=$vals[$index['DEBIAN-MANGLED-UVERSION'][0]][value];      $deb_mangled_vers=$vals[$index['DEBIAN-MANGLED-UVERSION'][0]][value];
766      $warnings=$vals[$index['WARNINGS'][0]][value];      $warnings=$vals[$index['WARNINGS'][0]][value];
# Line 693  function db_up_error() { Line 791  function db_up_error() {
791      $db = pg_pconnect($dbconn) ;      $db = pg_pconnect($dbconn) ;
792      $bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done";      $bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done";
793      print "\nDownloading bugs list =>";      print "\nDownloading bugs list =>";
794      $ferrors=file_get_contents($bugurl) or die_status ("I can't keep error page from bugs.debian.org");      if (!($ferrors=file_get_contents($bugurl))) {
795            /*die_status ("I can't keep error page from bugs.debian.org");*/
796            fwrite(STDERR, "I can't keep error page from bugs.debian.org, but not aborting");
797            return false;
798        }
799      print "Done\n";      print "Done\n";
800      $ferrors=explode("<li>",$ferrors);      $ferrors=explode("<li>",$ferrors);
801      $totline=count($ferrors);      $totline=count($ferrors);
# Line 702  function db_up_error() { Line 804  function db_up_error() {
804      print "Total lines => $totline\n";      print "Total lines => $totline\n";
805      foreach ($ferrors as $line) {      foreach ($ferrors as $line) {
806          # print $line . "\n";          # print $line . "\n";
807          if (preg_match("/Package:.+?>([^<]+)<\/a>.+<strong>((\d+)\syear\sand\s|)(\d+)\sdays/sS",$line,$matches)) {          if (preg_match("/Package:.+?>([^<]+)<\/a>.+?<strong>((\d+)\syears?\sand\s|)(\d+)\sdays?/sS",$line,$matches)) {
808              $errors[$matches[1]][err_tot]++;              $errors[$matches[1]][err_tot]++;
809              $errors[$matches[1]][days]+=$matches[3]*365+$matches[4];              $errors[$matches[1]][days]+=$matches[3]*365+$matches[4];
810              #if ($matches[3]) print "Anni $matches[3]";              #if ($matches[3]) print "Anni $matches[3]";
# Line 942  function dehsqa_db () { Line 1044  function dehsqa_db () {
1044              if ($res_array['wwiz_type']=="watch") $wwiz=$res_array['wwiz_version'];              if ($res_array['wwiz_type']=="watch") $wwiz=$res_array['wwiz_version'];
1045              else $wwiz=$res_array[wwiz_type];              else $wwiz=$res_array[wwiz_type];
1046    
1047              $updated = $res_array['updated']? 'yes' : 'no';              // the updated flag is set to true even when up_version could not be determined
1048                $updated = ($res_array['updated'] && ($res_array[up_version] || $res_array[wwiz_version]))? 'yes' : 'no';
1049    
1050              $xml="<data><id>$res_array[id]</id>" .              $xml="<data><id>$res_array[id]</id>" .
1051              "<up_version>$version</up_version>" .              "<up_version>$version</up_version>" .
1052              "<dversion>$res_array[version]</dversion>" .              "<dversion>$res_array[version]</dversion>" .
1053              "<dversionmangled>$res_array[dversionmangled]</dversionmangled>" .              "<dversionmangled>$res_array[dversionmangled]</dversionmangled>" .
1054              "<updated>$updated</updated>" .              "<uptodate>$updated</uptodate>" .
1055              "<wwiz>$wwiz</wwiz>" .              "<wwiz>$wwiz</wwiz>" .
1056              "</data>";              "</data>";
1057              $value=addslashes($xml);              $value=addslashes($xml);
# Line 972  function update_all() { Line 1075  function update_all() {
1075      echo __FUNCTION__ . ' started the ' . $intime . "\n";      echo __FUNCTION__ . ' started the ' . $intime . "\n";
1076      db_add();      db_add();
1077      dl_diffs();      dl_diffs();
1078        dl_from_vcs('%','%',in_array('vcs',$timeStamp_basedCheck));
1079      db_up_error();      db_up_error();
1080      db_upstream('%', in_array('all', $timeStamp_basedCheck));      db_upstream('%', in_array('all', $timeStamp_basedCheck));
1081      watch_wizard('%', in_array('all', $timeStamp_basedCheck));      watch_wizard('%', in_array('all', $timeStamp_basedCheck));
# Line 980  function update_all() { Line 1084  function update_all() {
1084      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
1085      $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";      $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";
1086      mail($email, "DEHS report", $messaggio,      mail($email, "DEHS report", $messaggio,
1087      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .  
     "X-Mailer: PHP/" . phpversion());  
1088  }  }
1089  function update_new() {  function update_new() {
1090      global $email_all;      global $email_all, $timeStamp_basedCheck;
1091      $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);      $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
1092      echo __FUNCTION__ . ' started the ' . $intime . "\n";      echo __FUNCTION__ . ' started the ' . $intime . "\n";
1093      db_add();      db_add();
# Line 993  function update_new() { Line 1095  function update_new() {
1095      global $dbconn;      global $dbconn;
1096      check_db();      check_db();
1097      $db = pg_pconnect($dbconn);      $db = pg_pconnect($dbconn);
1098      $sql="SELECT name FROM pkgs WHERE (md5_diff!=md5_atsource or md5_diff is null) or (lastcheck is null and watch is not null)";      $sql = "SELECT name FROM pkgs WHERE"
1099              ." (md5_diff!=md5_atsource or md5_diff is null)"
1100              ." or (lastcheck is null and watch is not null)"
1101              ." or (lastpkgsourcesupdate>lastcheck)";
1102      $rsql=pg_exec($db, $sql);      $rsql=pg_exec($db, $sql);
1103      $res_array=pg_fetch_all($rsql);      $res_array=pg_fetch_all($rsql);
1104      pg_close($db);      pg_close($db);
# Line 1002  function update_new() { Line 1107  function update_new() {
1107      db_up_error();      db_up_error();
1108      if ($res_array !== false) {      if ($res_array !== false) {
1109          foreach ($res_array as $entry) {          foreach ($res_array as $entry) {
1110                dl_from_vcs($entry['name'],'%',in_array('vcs',$timeStamp_basedCheck));
1111              db_upstream($entry['name']);              db_upstream($entry['name']);
1112              up_changes($entry['name']);              up_changes($entry['name']);
1113              watch_wizard($entry['name']);              watch_wizard($entry['name']);
# Line 1011  function update_new() { Line 1117  function update_new() {
1117      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
1118      $messaggio="Dehs executed successful\nBegin: $intime\nPackages processed: ".((!is_array($res_array))?'none':count($res_array))."\nEnd: $endtime";      $messaggio="Dehs executed successful\nBegin: $intime\nPackages processed: ".((!is_array($res_array))?'none':count($res_array))."\nEnd: $endtime";
1119      mail($email_all, "DEHS update_new report", $messaggio,      mail($email_all, "DEHS update_new report", $messaggio,
1120      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .  
     "X-Mailer: PHP/" . phpversion());  
1121  }  }
1122  function update_bogus() {  function update_bogus() {
1123      global $email_all,$dbconn;      global $email_all,$dbconn;
# Line 1042  function update_bogus() { Line 1146  function update_bogus() {
1146      "Packages processed: $orig_count\n" .      "Packages processed: $orig_count\n" .
1147      "New bogus watches count: $new_count\nEnd: $endtime";      "New bogus watches count: $new_count\nEnd: $endtime";
1148      mail($email_all, "DEHS update_bogus report", $messaggio,      mail($email_all, "DEHS update_bogus report", $messaggio,
1149      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .  
     "X-Mailer: PHP/" . phpversion());  
1150  }  }
1151  function die_status($msg = '', $status = 1) {  function die_status($msg = '', $status = 1) {
1152      if (empty($msg)) {      if (empty($msg)) {
# Line 1062  function sqldate($timestamp=null) { Line 1164  function sqldate($timestamp=null) {
1164          return gmdate("Y-m-d H:i:s", $timestamp);          return gmdate("Y-m-d H:i:s", $timestamp);
1165  }  }
1166    
1167    function dl_from_vcs($package = '%', $dist = '%', $checkStamps = false, $db = null) {
1168        global $minimumVCSAge;
1169        $local_db = false;
1170    
1171        if (!$fetchWatchFromVCS) {
1172            return false;
1173        }
1174    
1175        if ($db === null) {
1176            $local_db = true;
1177            $db = pg_pconnect($dbconn)  or  die_status(pg_last_error($db));
1178        }
1179    
1180        $sql="SELECT name,dist,vcs_type,vcs FROM pkgs WHERE name LIKE '$package' AND "
1181        . "dist LIKE '$dist' AND vcs!='' AND vcs IS NOT NULL AND vcs_type!='' AND vcs_type IS NOT NULL "
1182        . ($checkStamps? "AND (lastvcsfetch IS NULL OR lastvcsfetch <='"
1183                        . (sqldate(time() - ($minimumVCSAge*24*3600)))."') "
1184                        : '');
1185        $rsql=pg_exec($db, $sql);
1186        while ($res = pg_fetch_array($rsql)) {
1187            // avoid miss-parsed entries:
1188            if (   count(explode('\n',$res['vcs_type'])) > 1
1189                || count(explode('\n',$res['vcs'])) > 1)
1190                continue;
1191    
1192            print "Fetching watch file of $res[name] (type: $res[vcs_type]) from $[vcs]\n";
1193    
1194            $tmp_dir = shell_exec('mktemp -d') or die_status("mktemp failed to create directory!");
1195            $vcs_watch = null;
1196            if (exec('debcheckout.pl -t ' . escapeshellarg($res['vcs_type'])
1197                                 . ' -f debian/watch '
1198                                 . escapeshellarg($res['vcs'])
1199                                 . ' ' . escapeshellarg($tmp_dir))) {
1200                $vcs_watch = implode('\n', file($tmp_dir . '/debian/watch'));
1201            }
1202    
1203            shell_exec('rm -rf ' . escapeshellarg($tmp_dir));
1204    
1205            $dehs_opts = array();
1206            $dehs_opts['vcs'] = 0;
1207    
1208            if (preg_match('/#\s*DEHS\s*:\s*([\w\s,]+)/i', $vcs_watch, $matches)) {
1209                $_dehs_opts = array_map('trim',explode(',', $matches[1]));
1210                switch ($_dehs_opts) {
1211                    case 'vcs':
1212                        $dehs_opts['vcs'] = 1;
1213                        break;
1214                    case 'novcs':
1215                        $dehs_opts['vcs'] = 0;
1216                        break;
1217                }
1218            }
1219    
1220            if ($vcs_watch != null) {
1221                $vcs_watch = iconv('ISO-8859-1', 'UTF-8', $vcs_watch);
1222                $vcs_watch = pg_escape_string($vcs_watch);
1223            }
1224    
1225            pg_exec($db, "UPDATE pkgs SET vcs_watch='" . $vcs_watch . "', lastvcsfetch = now(), vcsoversource='$dehs_opts[vcs]' WHERE name='$res[name]' AND dist='$res[dist]'") or die_status("Query error");
1226        }
1227    
1228        if ($local_db) pg_close($db);
1229        return true;
1230    }
1231    
1232  ?>  ?>
Legend:
Removed from v.131  
changed lines
  Added in v.187
  ViewVC Help
Powered by ViewVC 1.1.5