/[dehs]/trunk/dehs_pg.php
ViewVC logotype

Diff of /trunk/dehs_pg.php

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 135 by atomo64-guest, Thu May 29 21:35:30 2008 UTC revision 167 by atomo64-guest, Mon Jan 26 00:20:32 2009 UTC
# Line 66  if (in_array("-db_add",$argv))  db_add(( Line 66  if (in_array("-db_add",$argv))  db_add((
66      elseif (in_array("-db_popcon",$argv))  db_popcon();      elseif (in_array("-db_popcon",$argv))  db_popcon();
67      elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv))  dl_popcon();      elseif (in_array("-dl_popcon",$argv) || in_array("-keep_popcon",$argv))  dl_popcon();
68      elseif (in_array("-db_upstream",$argv))  db_upstream((!empty($argv[2])?$argv[2]:'%'));      elseif (in_array("-db_upstream",$argv))  db_upstream((!empty($argv[2])?$argv[2]:'%'));
69        elseif (in_array("-dl_from_vcs",$argv))  dl_from_vcs((!empty($argv[2])?$argv[2]:'%'),(!empty($argv[3])?$argv[3]:'%'));
70      elseif (in_array("-update_all",$argv))  update_all();      elseif (in_array("-update_all",$argv))  update_all();
71      elseif (in_array("-update_new",$argv))  update_new();      elseif (in_array("-update_new",$argv))  update_new();
72      elseif (in_array("-update_bogus",$argv))  update_bogus();      elseif (in_array("-update_bogus",$argv))  update_bogus();
# Line 211  function db_add($initial='') { Line 212  function db_add($initial='') {
212                              $matches[10]=pg_escape_string($matches[10]);                              $matches[10]=pg_escape_string($matches[10]);
213                          }                          }
214                          @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR die_status("Temp table pkgs_atsrc query error");                          @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR die_status("Temp table pkgs_atsrc query error");
215                          $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders,homepage,vcs_browser,vcs_type,vcs,lastpkgsourcesupdate) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]','$matches[14]','$matches[16]','$matches[18]','$matches[19]',now())");                          $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,dversionmangled,maint,dir,md5_atsource,bytes,dist,section,uploaders,homepage,vcs_browser,vcs_type,vcs,vcsoversource,lastpkgsourcesupdate) VALUES ('$matches[1]','$matches[4]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]','$matches[14]','$matches[16]','$matches[18]','$matches[19]','0',now())");
216                          if (!$rst) {                          if (!$rst) {
217                              $rsql=pg_exec($db, "SELECT pkgs.version FROM pkgs WHERE name='$matches[1]' AND dist='$dist';");                              $rsql=pg_exec($db, "SELECT pkgs.version FROM pkgs WHERE name='$matches[1]' AND dist='$dist';");
218                              $version = $matches[4]; // just for safety                              $version = $matches[4]; // just for safety
# Line 320  function download_sources () { Line 321  function download_sources () {
321  }  }
322  function md5_source($section,$dist) {  function md5_source($section,$dist) {
323      global $mirrors;      global $mirrors;
324      $relase=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die_status("Failed to check MD5 of sources");      $release=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die_status("Failed to check MD5 of sources");
325      preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $relase,$matches);      preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $release,$matches);
326      return array("md5"=>$matches[1],"bytes"=>$matches[2]);      return array("md5"=>$matches[1],"bytes"=>$matches[2]);
327      # 183dea9ca3c2e35ffd6cf5eee9501c21          2941637 main/binary-i386/Packages.gz      # 183dea9ca3c2e35ffd6cf5eee9501c21          2941637 main/binary-i386/Packages.gz
328  }  }
# Line 350  function check_db() { Line 351  function check_db() {
351      global $dirs,$dbconn;      global $dirs,$dbconn;
352      $db = pg_pconnect($dbconn) or die_status(1/*pg_last_notice($db)*/);      $db = pg_pconnect($dbconn) or die_status(1/*pg_last_notice($db)*/);
353      if (!pg_table_exists($db, "pkgs")) {      if (!pg_table_exists($db, "pkgs")) {
354          pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text, lastcheck timestamp, homepage text, vcs text,  vcs_type text, vcs_browser text, lastupvsdebsync timestamp, lastpkgsourcesupdate timestamp, pop_inst numeric,pop_vote numeric)") or die_status('Error creating table pkgs\n');          pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text, dversionmangled text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text, lastcheck timestamp, homepage text, vcs text,  vcs_type text, vcs_browser text,vcs_watch text, vcsoversource bool,vcs_up_version text,vcs_dversionmangled text,vcs_up_url text, vcs_watch_warn text, vcs_updated bool, lastvcsfetch timestamp, lastupvsdebsync timestamp, lastpkgsourcesupdate timestamp, failures_count numeric, pop_inst numeric,pop_vote numeric)") or die_status('Error creating table pkgs\n');
355          pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db));          pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die_status('Errore creating index - ' . pg_last_error($db));
356          pg_exec($db, "CREATE INDEX checkx on pkgs (lastcheck)") or die_status('Errore creating index - ' . pg_last_error($db));          pg_exec($db, "CREATE INDEX checkx on pkgs (lastcheck)") or die_status('Errore creating index - ' . pg_last_error($db));
357          pg_exec($db, "CREATE INDEX updatex on pkgs (lastcheck,lastpkgsourcesupdate);") or die_status('Errore creating index - ' . pg_last_error($db));          pg_exec($db, "CREATE INDEX updatex on pkgs (lastcheck,lastpkgsourcesupdate);") or die_status('Errore creating index - ' . pg_last_error($db));
# Line 442  function db_popcon() { Line 443  function db_popcon() {
443    
444          // when one of the values is 'missing' set to -1 so the other value          // when one of the values is 'missing' set to -1 so the other value
445          //  _is_ UPDATEd, but use -1 to catch those bugs          //  _is_ UPDATEd, but use -1 to catch those bugs
446          $inst = (is_int($inst))? $inst : -1;          $inst = (is_numeric($inst))? $inst : -1;
447          $vote = (is_int($vote))? $vote : -1;          $vote = (is_numeric($vote))? $vote : -1;
448    
449          @pg_exec($db, "UPDATE pkgs SET pop_inst='$inst',pop_vote='$vote' WHERE name='$package'");          @pg_exec($db, "UPDATE pkgs SET pop_inst='$inst',pop_vote='$vote' WHERE name='$package'");
450      }      }
# Line 496  function db_upstream($initial='%', $chec Line 497  function db_upstream($initial='%', $chec
497    
498      $postWHERE = '';      $postWHERE = '';
499      if ($checkStamps) {      if ($checkStamps) {
500          $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL)";          $postWHERE = "AND (lastcheck<='".sqldate(time() - ($minimumAge*24*3600))."' OR lastcheck IS NULL OR (lastpkgsourcesupdate>lastcheck))";
501      }      }
502    
503      $postORDER = '';      $postORDER = '';
# Line 504  function db_upstream($initial='%', $chec Line 505  function db_upstream($initial='%', $chec
505          $postORDER = 'LIMIT ' . $limit;          $postORDER = 'LIMIT ' . $limit;
506      }      }
507    
508      $rsql=pg_exec($db, "SELECT id, name,dist,up_version,version,watch,lastupvsdebsync FROM pkgs WHERE watch is not null AND name LIKE '$initial' $postWHERE ORDER BY lastcheck DESC $postORDER");      $rsql=pg_exec($db, "SELECT id, name,dist,up_version,version,watch,lastupvsdebsync,failures_count FROM pkgs WHERE watch is not null AND name LIKE '$initial' $postWHERE ORDER BY lastcheck DESC $postORDER");
509      $tot=pg_numrows($rsql);      $tot=pg_numrows($rsql);
510    
511      while ($res_array=pg_fetch_array($rsql)) {      while ($res_array=pg_fetch_array($rsql)) {
# Line 517  function db_upstream($initial='%', $chec Line 518  function db_upstream($initial='%', $chec
518          $lastupvsdebsync = $res_array['lastupvsdebsync'];          $lastupvsdebsync = $res_array['lastupvsdebsync'];
519          $mangle = true;          $mangle = true;
520    
521            /* Do not increase the number of failures if uscan didn't throw any warning
522                because it might be that the watch file is "empty"
523            */
524            if (!$uscan_res['uversion'] && $uscan_res['warnings']) {
525                $res_array['failures_count']++;
526            } else {
527                $res_array['failures_count'] = 0;
528            }
529    
530          print "\rVersion=>" . $uscan_res['dversion'];          print "\rVersion=>" . $uscan_res['dversion'];
531          if ($uscan_res['dversion']!=null) $version=$uscan_res['dversion'];          if ($uscan_res['dversion']!=null) $version=$uscan_res['dversion'];
532          if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) {          if ($uscan_res['dversionmangled']!=null && $uscan_res['dversionmangled'] != vers_conv($res_array['version'])) {
# Line 534  function db_upstream($initial='%', $chec Line 544  function db_upstream($initial='%', $chec
544          }          }
545    
546          if (!$updated) {          if (!$updated) {
547              events::newVersion($res_array['name'], $res_array['version'], $uscan_res['uversion'], $res_array['dist'], $up_url);              events::newVersion($res_array['name'], $res_array['version'], $up_version, $res_array['dist'], $up_url);
548          } else {          } else {
549              $lastupvsdebsync = sqldate();              $lastupvsdebsync = sqldate();
550          }          }
# Line 543  function db_upstream($initial='%', $chec Line 553  function db_upstream($initial='%', $chec
553              $lastupvsdebsync = sqldate(0);              $lastupvsdebsync = sqldate(0);
554          }          }
555    
556          if ($uscan_res['uversion']!=$res_array[up_version]) ($keep_changes=1);          $keep_changes = (int) ($up_version != $res_array['up_version']);
         else $keep_changes=0;  
557    
558          $watch_warn='';          $watch_warn='';
559          if($uscan_res['warnings'] != null) {          if($uscan_res['warnings'] != null) {
560              $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8", $uscan_res['warnings']));              $watch_warn=pg_escape_string(iconv("ISO-8859-1","UTF-8", $uscan_res['warnings']));
561          }          }
562    
563          pg_exec($db, "UPDATE pkgs SET up_version='" . pg_escape_string($up_version) . "', dversionmangled='".pg_escape_string($dversionmangled)."',up_url='" . pg_escape_string($up_url) . "', watch_warn='" . $watch_warn . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated', lastcheck=now(), lastupvsdebsync='$lastupvsdebsync'  WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream");          pg_exec($db, "UPDATE pkgs SET
564                                        up_version='" . pg_escape_string($up_version) . "',
565                                        dversionmangled='".pg_escape_string($dversionmangled)."',
566                                        up_url='" . pg_escape_string($up_url) . "',
567                                        watch_warn='" . $watch_warn . "',
568                                        keep_changes='$keep_changes',
569                                        wwiz_version=NULL,
570                                        wwiz_type=NULL,
571                                        wwiz=NULL,
572                                        failures_count=$res_array[failures_count],
573                                        updated='$updated',
574                                        lastcheck=now(),
575                                        lastupvsdebsync='$lastupvsdebsync'
576                                      WHERE id=$res_array[id]") or die_status("Error in update pkgs upstream");
577    
578          #print "\rUpdated " . $res_array['name'] .  " => $updated Up Version => ". $uscan_res['uversion'];          #print "\rUpdated " . $res_array['name'] .  " => $updated Up Version => ". $uscan_res['uversion'];
579          print "\t Upstream left => " . $tot --;          print "\t Upstream left => " . $tot --;
580          #                               }          #                               }
# Line 717  function db_up_error() { Line 740  function db_up_error() {
740      print "Total lines => $totline\n";      print "Total lines => $totline\n";
741      foreach ($ferrors as $line) {      foreach ($ferrors as $line) {
742          # print $line . "\n";          # print $line . "\n";
743          if (preg_match("/Package:.+?>([^<]+)<\/a>.+<strong>((\d+)\syear\sand\s|)(\d+)\sdays/sS",$line,$matches)) {          if (preg_match("/Package:.+?>([^<]+)<\/a>.+?<strong>((\d+)\syears?\sand\s|)(\d+)\sdays?/sS",$line,$matches)) {
744              $errors[$matches[1]][err_tot]++;              $errors[$matches[1]][err_tot]++;
745              $errors[$matches[1]][days]+=$matches[3]*365+$matches[4];              $errors[$matches[1]][days]+=$matches[3]*365+$matches[4];
746              #if ($matches[3]) print "Anni $matches[3]";              #if ($matches[3]) print "Anni $matches[3]";
# Line 957  function dehsqa_db () { Line 980  function dehsqa_db () {
980              if ($res_array['wwiz_type']=="watch") $wwiz=$res_array['wwiz_version'];              if ($res_array['wwiz_type']=="watch") $wwiz=$res_array['wwiz_version'];
981              else $wwiz=$res_array[wwiz_type];              else $wwiz=$res_array[wwiz_type];
982    
983              $updated = $res_array['updated']? 'yes' : 'no';              // the updated flag is set to true even when up_version could not be determined
984                $updated = ($res_array['updated'] && ($res_array[up_version] || $res_array[wwiz_version]))? 'yes' : 'no';
985    
986              $xml="<data><id>$res_array[id]</id>" .              $xml="<data><id>$res_array[id]</id>" .
987              "<up_version>$version</up_version>" .              "<up_version>$version</up_version>" .
988              "<dversion>$res_array[version]</dversion>" .              "<dversion>$res_array[version]</dversion>" .
989              "<dversionmangled>$res_array[dversionmangled]</dversionmangled>" .              "<dversionmangled>$res_array[dversionmangled]</dversionmangled>" .
990              "<updated>$updated</updated>" .              "<uptodate>$updated</uptodate>" .
991              "<wwiz>$wwiz</wwiz>" .              "<wwiz>$wwiz</wwiz>" .
992              "</data>";              "</data>";
993              $value=addslashes($xml);              $value=addslashes($xml);
# Line 987  function update_all() { Line 1011  function update_all() {
1011      echo __FUNCTION__ . ' started the ' . $intime . "\n";      echo __FUNCTION__ . ' started the ' . $intime . "\n";
1012      db_add();      db_add();
1013      dl_diffs();      dl_diffs();
1014        dl_from_vcs('%','%',in_array('vcs',$timeStamp_basedCheck));
1015      db_up_error();      db_up_error();
1016      db_upstream('%', in_array('all', $timeStamp_basedCheck));      db_upstream('%', in_array('all', $timeStamp_basedCheck));
1017      watch_wizard('%', in_array('all', $timeStamp_basedCheck));      watch_wizard('%', in_array('all', $timeStamp_basedCheck));
# Line 995  function update_all() { Line 1020  function update_all() {
1020      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
1021      $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";      $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";
1022      mail($email, "DEHS report", $messaggio,      mail($email, "DEHS report", $messaggio,
1023      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .  
     "X-Mailer: PHP/" . phpversion());  
1024  }  }
1025  function update_new() {  function update_new() {
1026      global $email_all;      global $email_all, $timeStamp_basedCheck;
1027      $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);      $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
1028      echo __FUNCTION__ . ' started the ' . $intime . "\n";      echo __FUNCTION__ . ' started the ' . $intime . "\n";
1029      db_add();      db_add();
# Line 1020  function update_new() { Line 1043  function update_new() {
1043      db_up_error();      db_up_error();
1044      if ($res_array !== false) {      if ($res_array !== false) {
1045          foreach ($res_array as $entry) {          foreach ($res_array as $entry) {
1046                dl_from_vcs($entry['name'],'%',in_array('vcs',$timeStamp_basedCheck));
1047              db_upstream($entry['name']);              db_upstream($entry['name']);
1048              up_changes($entry['name']);              up_changes($entry['name']);
1049              watch_wizard($entry['name']);              watch_wizard($entry['name']);
# Line 1029  function update_new() { Line 1053  function update_new() {
1053      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);      $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
1054      $messaggio="Dehs executed successful\nBegin: $intime\nPackages processed: ".((!is_array($res_array))?'none':count($res_array))."\nEnd: $endtime";      $messaggio="Dehs executed successful\nBegin: $intime\nPackages processed: ".((!is_array($res_array))?'none':count($res_array))."\nEnd: $endtime";
1055      mail($email_all, "DEHS update_new report", $messaggio,      mail($email_all, "DEHS update_new report", $messaggio,
1056      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .  
     "X-Mailer: PHP/" . phpversion());  
1057  }  }
1058  function update_bogus() {  function update_bogus() {
1059      global $email_all,$dbconn;      global $email_all,$dbconn;
# Line 1060  function update_bogus() { Line 1082  function update_bogus() {
1082      "Packages processed: $orig_count\n" .      "Packages processed: $orig_count\n" .
1083      "New bogus watches count: $new_count\nEnd: $endtime";      "New bogus watches count: $new_count\nEnd: $endtime";
1084      mail($email_all, "DEHS update_bogus report", $messaggio,      mail($email_all, "DEHS update_bogus report", $messaggio,
1085      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .      "From: dehs-noreply@{$_SERVER['SERVER_NAME']}");
     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .  
     "X-Mailer: PHP/" . phpversion());  
1086  }  }
1087  function die_status($msg = '', $status = 1) {  function die_status($msg = '', $status = 1) {
1088      if (empty($msg)) {      if (empty($msg)) {
# Line 1080  function sqldate($timestamp=null) { Line 1100  function sqldate($timestamp=null) {
1100          return gmdate("Y-m-d H:i:s", $timestamp);          return gmdate("Y-m-d H:i:s", $timestamp);
1101  }  }
1102    
1103    function dl_from_vcs($package = '%', $dist = '%', $checkStamps = false, $db = null) {
1104        global $minimumVCSAge;
1105        $local_db = false;
1106    
1107        if (!$fetchWatchFromVCS) {
1108            return false;
1109        }
1110    
1111        if ($db === null) {
1112            $local_db = true;
1113            $db = pg_pconnect($dbconn)  or  die_status(pg_last_error($db));
1114        }
1115    
1116        $sql="SELECT name,dist,vcs_type,vcs FROM pkgs WHERE name LIKE '$package' AND "
1117        . "dist LIKE '$dist' AND vcs!='' AND vcs IS NOT NULL AND vcs_type!='' AND vcs_type IS NOT NULL "
1118        . ($checkStamps? "AND (lastvcsfetch IS NULL OR lastvcsfetch <='"
1119                        . (sqldate(time() - ($minimumVCSAge*24*3600)))."') "
1120                        : '');
1121        $rsql=pg_exec($db, $sql);
1122        while ($res = pg_fetch_array($rsql)) {
1123            // avoid miss-parsed entries:
1124            if (   count(explode('\n',$res['vcs_type'])) > 1
1125                || count(explode('\n',$res['vcs'])) > 1)
1126                continue;
1127    
1128            print "Fetching watch file of $res[name] (type: $res[vcs_type]) from $[vcs]\n";
1129    
1130            $tmp_dir = shell_exec('mktemp -d') or die_status("mktemp failed to create directory!");
1131            $vcs_watch = null;
1132            if (exec('debcheckout.pl -t ' . escapeshellarg($res['vcs_type'])
1133                                 . ' -f debian/watch '
1134                                 . escapeshellarg($res['vcs'])
1135                                 . ' ' . escapeshellarg($tmp_dir))) {
1136                $vcs_watch = implode('\n', file($tmp_dir . '/debian/watch'));
1137            }
1138    
1139            shell_exec('rm -rf ' . escapeshellarg($tmp_dir));
1140    
1141            $dehs_opts = array();
1142            $dehs_opts['vcs'] = 0;
1143    
1144            if (preg_match('/#\s*DEHS\s*:\s*([\w\s,]+)/i', $vcs_watch, $matches)) {
1145                $_dehs_opts = array_map('trim',explode(',', $matches[1]));
1146                switch ($_dehs_opts) {
1147                    case 'vcs':
1148                        $dehs_opts['vcs'] = 1;
1149                        break;
1150                    case 'novcs':
1151                        $dehs_opts['vcs'] = 0;
1152                        break;
1153                }
1154            }
1155    
1156            if ($vcs_watch != null) {
1157                $vcs_watch = iconv('ISO-8859-1', 'UTF-8', $vcs_watch);
1158                $vcs_watch = pg_escape_string($vcs_watch);
1159            }
1160    
1161            pg_exec($db, "UPDATE pkgs SET vcs_watch='" . $vcs_watch . "', lastvcsfetch = now(), vcsoversource='$dehs_opts[vcs]' WHERE name='$res[name]' AND dist='$res[dist]'") or die_status("Query error");
1162        }
1163    
1164        if ($local_db) pg_close($db);
1165        return true;
1166    }
1167    
1168  ?>  ?>
Legend:
Removed from v.135  
changed lines
  Added in v.167
  ViewVC Help
Powered by ViewVC 1.1.5