/[dehs]/trunk/dehs_pg.php
ViewVC logotype

Contents of /trunk/dehs_pg.php

Parent Directory Parent Directory | Revision Log Revision Log


Revision 37 - (show annotations) (download)
Sat Nov 26 23:15:14 2005 UTC (7 years, 5 months ago) by evaso-guest
File size: 32729 byte(s)
Watch wizard generate version=3 watch file now
1 #!/usr/bin/php4 -q
2 <?
3 /*
4
5 Originally written by Stefano Fabri <bluefuture@nospam@email.it>
6 Copyright 2004, Stefano Fabri
7
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 */
22 // stabilire una connessione
23 #ini_set ( "error_reporting", "E_COMPILE_ERROR|E_ERROR|E_CORE_ERROR");
24 ini_set ("html_errors", "off");
25 set_time_limit(0);
26 ini_set ( "memory_limit", "40M");
27 #print_R($_SERVER);
28 require "dehs.config.inc";
29 dir_ctrl($dirs);
30 if (!function_exists('file_get_contents')) {
31 function file_get_contents($filename) {
32 if (!($file = fopen($filename,"r"))) return false;
33 else {
34 $contents = "";
35 do {
36 $data = fread($file, 4096);
37 if (strlen($data) == 0) {
38 break;
39 }
40 $contents .= $data;
41 } while (true);
42 fclose($file);
43 return $contents;
44 }
45 }
46 }
47 if (in_array("-db_add",$argv)) db_add($argv[2]);
48 elseif (in_array("-db_query",$argv)) db_query($argv[2],$argv[3] );
49 elseif (in_array("-keep_diff",$argv)) keep_diff($argv[2]);
50 elseif (in_array("-keep_sources",$argv)) keep_sources();
51 elseif (in_array("-db_popcon",$argv)) db_popcon($argv[2]);
52 elseif (in_array("-keep_popcon",$argv)) keep_popcon($argv[2]);
53 elseif (in_array("-db_upstream",$argv)) db_upstream($argv[2]);
54 elseif (in_array("-update_all",$argv)) update_all();
55 elseif (in_array("-check_db",$argv)) check_db();
56 elseif (in_array("-dehsqa_db",$argv)) dehsqa_db();
57 elseif (in_array("-clear_db",$argv)) clear_db();
58 elseif (in_array("-db_up_error",$argv)) db_up_error();
59 elseif (in_array("-up_changes",$argv)) up_changes($argv[2]);
60 elseif (in_array("-watchf",$argv)) watch_fill($argv[2]);
61 elseif (in_array("-kb_total",$argv)) print "Diff
62 #stat_up_error();
63 #stat_up_error();to download Kb => " . kb_total($argv[2]) . "\n" ;
64 elseif (in_array("-watch_stat",$argv)) watch_stat();
65 else print "Command $argv[1] not found\n";
66 #stat_up_error();
67 function dir_ctrl($dirs) {
68 foreach ($dirs as $dir) {
69 if (!is_dir($dir)) exec("mkdir -m 0740 -p $dir",$output,$res);
70 if ($res!=0) die("Error creating basic dir: $dir\n");
71 }
72 }
73 function ext_watch ($file,$pkg,$version){
74 if (!file_exists($file)) return "";
75 $zp=gzopen($file,"r") or die("Fail to open $file");
76 while (!gzeof($zp)) {
77 $line = gzgets ($zp,4096);
78 if (preg_match("/\+\+\+\s.+\/debian\/watch/",$line)) {
79 $line = gzgets ($zp,4096);
80 preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght);
81 for ($i = 1; $i <= $watch_lenght[1]; $i++) {
82 $line=gzgets ($zp,4096);
83 $watch .=substr($line,1);
84 }
85 break;
86 }
87 }
88 if ($watch!=null) return $watch;
89 else return NULL;
90 }
91 function ftp_cn($login,$password,$ftp_server) {
92 $conn_id = @ftp_connect($ftp_server) or DIE("Error: Cannot contact ftp server $ftp_server\n");
93 // login con user name e password
94 ftp_pasv($conn_id,1);
95 $login_result = ftp_login($conn_id, "anonymous","bo@bo.org");
96 // controllo della connessione
97 if ((!$conn_id) || (!$login_result)) {
98 echo "Ftp connection failed!\n";
99 echo "Failed connection to $ftp_server\n";
100 die();
101 } else {
102 return $conn_id;
103 }
104 }
105 function keep_diff($initial) {
106 global $dirs,$dbconn,$mirrors;
107 check_db();
108 $conn_normal=ftp_cn("anonymous","bo@bo.org",$mirrors['unstable']);
109 #$conn_non_us=ftp_cn("anonymous","bo@bo.org",$mirrors["non-US"]);
110 $db = pg_pconnect($dbconn) ;
111 $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR DIE("Error in diff query\n");
112 $total=pg_numrows($rsql);
113 $btotal=byte_total($db,$initial);
114 print "Total Diff to downlaod Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n";
115 while ($res_array=pg_fetch_array($rsql)) {
116 #if ($res_array['dist']=='non-US') $conn_id = $conn_non_us;
117 #else
118 $conn_id = $conn_normal;
119 if ($localfile=download_diffs($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) {
120 $watch=ext_watch($localfile,$res_array[name],$res_array[version]);
121 unlink($localfile);
122 if ($watch!=NULL) {
123 $watch=iconv("ISO-8859-1","UTF-8",$watch);
124 $watch= addslashes($watch);
125 pg_exec($db, "UPDATE pkgs SET watch='" . $watch . "', md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Query error");
126 }
127 else pg_exec($db, "UPDATE pkgs SET watch=NULL, md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Errore nella query");
128 }
129 $btotal=$btotal-$res_array['bytes'];
130 number_format ($btotal/1024, 2, ",",".");
131 print "\rDiffs left => " . $total-- . " Bytes left => " . number_format ($btotal/1024, 2, ",",".") . " - " . $res_array[name];
132 }
133 ftp_quit($conn_normal);
134 pg_close($db);
135 return "Fatto\n";
136 }
137 function download_diffs(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) {
138 global $dirs,$mirrors;
139 $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz";
140 #if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz";
141 #else
142 $remotefile="/debian/$directory/$pkg" . "_" . "$version.diff.gz";
143 #print "$localfile - $remotefile\n";
144 if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
145 # print "$pkg Diff.gz => Downloaded\n";
146 if(md5_of_file($localfile)==$md5_atsource) {
147 #print "MD5 => OK\n";
148 #unlink($localfile);
149 return $localfile;
150 }
151 }
152 print "Download Diff.gz of package $pkg => Fallito\n";
153 return false;
154 }
155 function db_add($initial=''){
156 global $dirs,$dists,$dbconn;
157 check_db();
158 keep_source();
159 $db = pg_connect($dbconn) or die(pg_errormessage($db));
160 pg_exec($db, "CREATE TEMP TABLE pkgs_atsrc (name text,dist text)") or die('Error creating temp table pkgs_atsrc');
161 pg_exec($db, "CREATE UNIQUE INDEX idxdis on pkgs_atsrc (name,dist)") or die('Error creating index on temp table');
162 pg_exec($db, "CREATE TEMP TABLE bin_atsrc (name text,bin_name text, dist text)") or die('Error creating temp table pkgs_atsrc');
163 pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die('Error creating index on temp table bin_atsrc');
164 foreach ($dists as $dist=>$sections) {
165 foreach ($sections as $section) {
166 $filename=$dirs[sources_dir] . "/$dist/$section/Sources.gz";
167 print "\nI'm parsing $dist/$section => Sources.gz\n";
168 # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5
169 $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S+)\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?/Ssi";
170 $zp = gzopen($filename, "r") or die("Could not open $filename");
171 $extracted="/tmp/$dist_$section_Sources.gz";
172 exec("gzip -c -d $filename > $extracted");
173 $sourcesize=filesize($extracted);
174 unlink($extracted);
175 if(filesize($filename)>20) {
176 while (!gzeof($zp)) {
177 $line='';
178 while ($line!="\n") {
179 $line = gzgets ($zp,4096) ;
180 $buff1 .= $line;
181 }
182 if (preg_match($regexp,$buff1,$matches)) {
183 if ($matches[10]) {
184 $matches[10]=iconv("ISO-8859-1","UTF-8",$matches[10]);
185 $matches[10]=addslashes($matches[10]);
186 }
187 @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR DIE("Temp table pkgs_atsrc query error");
188 $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,maint,dir,md5_atsource,bytes,dist,section,uploaders) VALUES ('$matches[1]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]')") ;
189 if (!$rst) {
190 $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]',maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]' WHERE name='$matches[1]' AND dist='$dist'") OR DIE ("\nDb adding error =>" . pg_errormessage() . "\n");
191 }
192 $bin_names=split(",", $matches[2]);
193 foreach ($bin_names as $bin_name) {
194 pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR DIE("Temp table pkgs_atsrc query error");
195 $rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ;
196 }
197 }
198 else {
199 preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches);
200 if(strncmp($initial,$matches[1],strlen($initial))<0) break;
201 }
202 $left=$sourcesize-gztell($zp);
203 //print "\rSource file Left => $left";
204 $buff1="";
205 unset($matches);
206 }
207 }
208 gzclose($zp) ;
209 }
210 }
211 clear_db($db);
212 pg_close($db);
213 }
214 function db_query($pkg,$dist) {
215 global $dirs,$dbconn;
216 $db = pg_pconnect($dbconn ) or die(pg_errormessage($db));
217 $rsql=pg_exec($db, "SELECT pkgs.* ,mpop_inst FROM (SELECT name,MAX(binpkgs.pop_inst) AS mpop_inst FROM binpkgs GROUP BY name)
218 AS binpkgs INNER JOIN pkgs ON pkgs.name=binpkgs.name
219 WHERE pkgs.name='$pkg' AND dist='$dist';");
220 if (pg_numrows($rsql)==1) {
221 $res_array=pg_fetch_array($rsql);
222 print "Package=> " . $res_array['name'] . "\n";
223 print "Version => " . $res_array['version'] . "\n";
224 print "Diff MD5 Downloaded => " . $res_array['md5_diff'] . "\n";
225 print "Diff MD5 at Sources => " . $res_array['md5_atsource'] . "\n";
226 print "Distribution => " . $res_array['dist'] . "\n";
227 print "Section => " . $res_array['section'] . "\n";
228 print "KBytes => " . number_format ( $res_array['bytes']/1024, 2, ",",".") . "\n";
229 print "Pop Inst=>" . $res_array['mpop_inst'] . "\n";
230 if ($res_array['watch']) {
231 $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['watch']);
232 print "Version Upstream => " . $uscan_res[0] . "\n";
233 print "Watch => " . $res_array['watch'];
234 print "Watch Warings => " . $uscan_res[1];
235 }
236 else print "Watch => Not present" . "\n";
237 }
238 else echo "Package=> " . $pkg . "\nNot found\n!!";
239 pg_close($db);
240 }
241 function byte_total(&$db,$initial){
242 global $dirs;
243 $rsql=pg_exec ($db,"Select sum(bytes) as totbytes FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)");
244 $res_array=pg_fetch_array($rsql);
245 return $res_array[totbytes] ;
246 }
247 function keep_source (){
248 global $dirs,$dists,$mirrors;
249 foreach ($dists as $dist=>$sections) {
250 $server=$mirrors[$dist];
251 $conn_id = ftp_cn("anonymous","bo@bo.org",$server);
252 foreach ($sections as $section) {
253 $localfile=$dirs[sources_dir] . "/$dist/$section/Sources.gz";
254 $remotefile=remote_dist_dir($dist) . "/$dist/$section/source/Sources.gz";
255 $sources_info=md5_source($section,$dist);
256 if (!file_exists ($dirs[sources_dir] . "/$dist/")) mkdir ( $dirs[sources_dir] . "/$dist",0740);
257 if (!file_exists ($dirs[sources_dir] . "/$dist/$section")) mkdir ($dirs[sources_dir] . "/$dist/$section",0740);
258 if (!file_exists($localfile) OR md5_of_file($localfile)!=$sources_info[md5]) {
259 print "Downloading $remotefile Bytes $sources_info[bytes]=> ";
260 if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
261 print "Done\n";
262 if(md5_of_file($localfile)==$sources_info[md5]) print "MD5 => OK\n";
263 }
264 else {
265 unlink("$localfile");
266 print "Fail\n";
267 die ("Error in source file");
268 }
269 }
270 }
271 ftp_quit($conn_id);
272 }
273 }
274 function md5_source($section,$dist) {
275 global $mirrors;
276 $relase=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die("Failed to check MD5 of sources");
277 preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $relase,$matches);
278 return array("md5"=>$matches[1],"bytes"=>$matches[2]);
279 # 183dea9ca3c2e35ffd6cf5eee9501c21 2941637 main/binary-i386/Packages.gz
280 }
281
282 function vers_conv($debvers) {
283 preg_match("/(.+:)?([^-]+)(ds|dsfg|debian)/",$debvers,$matches);
284 if (!$matches[3]) {
285 unset($matches);
286 preg_match("/(.+:)?([^-]+)/",$debvers,$matches);
287 }
288 if (substr($matches[2],-1)=='.') $matches[2]=substr($matches[2],0,-1);
289 return $matches[2];
290 }
291 function check_db() {
292 global $dirs,$dbconn;
293 $db = @pg_connect($dbconn) or die(pg_last_notice());
294 if (!pg_table_exists($db,"pkgs")){
295 pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text)") or die('Error creating table pkgs\n');
296 pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die('Errore creating index - ' . pg_last_error($db));
297 $sw=true;
298 }
299 if (!pg_table_exists($db,"binpkgs")){
300 pg_exec($db, "CREATE TABLE binpkgs (id serial PRIMARY KEY, name text,bin_name text,dist text, pop_inst numeric,pop_vote numeric, up_error numeric,avg_error_date numeric)") or die('Error creating table binpkgs\n');
301 pg_exec($db, "CREATE UNIQUE INDEX idxbin on binpkgs (name,bin_name,dist)") or die('Errore creating index - ' . pg_last_error($db));
302 $sw=true;
303 }
304 #if (!pg_table_exists($db,"upstream")){
305 # pg_exec($db, "CREATE TABLE upstream (name text,dist text,up_version text,watch_warn text,updated bool)") or die('Errore nel creare la tabella');
306 # pg_exec($db, "CREATE UNIQUE INDEX idxupstrm on upstream (name,dist)") or die('Errore nel creare gli indici ' . pg_last_error($db));
307 # $sw=true;
308 #}
309 pg_close($db);
310 if($sw) return $sw;
311 else return false;
312 }
313 function pg_table_exists (&$dblink, $table) {
314 $sql = "SELECT relname as tbl FROM pg_class WHERE relname = '$table'";
315 if ($res = pg_exec ($dblink, $sql)) {
316 return pg_numrows($res) > 0;
317 } else {
318 return false; // or throw exception
319 }
320 }
321
322
323 function watch_stat() {
324 global $dirs,$dbconn;
325 $db = pg_pconnect($dbconn) or die(pg_errormessage($db));
326 $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch!=''");
327 $tot_watch=pg_fetch_array($rsql);
328 print "Total packages with Watch =>$tot_watch[0]\n";
329 $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch IS NULL");
330 $tot_nowatch=pg_fetch_array($rsql);
331 pg_close($db);
332 print "Totale packages without Watch => $tot_nowatch[0]\n";
333 print "Watch presence rate => " . number_format ($tot_watch[0]/($tot_nowatch[0]+$tot_watch[0])*100, 2, ",",".") . "%\n";
334 }
335 function remote_dist_dir($dist) {
336 #if ($dist=="non-US") return "/debian-non-US/dists/unstable";
337 #else
338 return "/debian/dists";
339 }
340 function clear_db(&$db) {
341 #global $dbconn;
342 #if (check_db()) die("Il Database era vuoto\n");
343 #keep_source();
344 #$db = pg_pconnect($dbconn) or die(pg_result_error($db));
345 $sql1= "Delete from binpkgs where id NOT IN
346 (select id FROM binpkgs INNER JOIN bin_atsrc USING (name,bin_name,dist))" ;
347 $sql2= "Delete from pkgs where id NOT IN (select id FROM pkgs INNER JOIN pkgs_atsrc USING (name,dist))" ;
348 $sql3= "UPDATE pkgs SET wwiz=NULL, wwiz_type=NULL, wwiz_version=NULL WHERE watch IS NOT NULL or watch!=''" ;
349 $sql4= "UPDATE pkgs SET up_changes=NULL WHERE wwiz_version=up_version OR up_version=version" ;
350 if(pg_exec($db,$sql1) AND pg_exec($db,$sql2) AND pg_exec($db,$sql3) AND pg_exec($db,$sql4)) print "Db clear complete\n";
351 }
352 function db_popcon($initial='') {
353 global $dirs,$dbconn;
354 $db = pg_pconnect($dbconn) or die(pg_errormessage($db));
355 $filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
356 $regexp="/Package:\s($initial\S*)\s+(\d+)\s+(\d+)/iS";
357 $zp = gzopen($filename, "r") or die("Could not open $filename");
358 $tin=time();
359 if(filesize($filename)>20) {
360 #pg_exec($db, "BEGIN TRANSACTION") OR DIE("Impossibile iniziare la transazione");
361 while (!gzeof($zp)) {
362 $line = gzgets ($zp,4096) ;
363 if (preg_match($regexp,$line,$matches)) {
364 #print "\rName => $matches[1] Install => $matches[2] Vote => $matches[3]";
365 @pg_exec($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'");
366 $processed+=1;
367 $elapsed_time=time()-$tin;
368 if (($elapsed_time % 20)==0 AND $updtime!=$elapsed_time) {
369 $updtime=$elapsed_time;
370 $wprocessed=$processed-$wprocessed;
371 $avg_time=$wprocessed/20;
372 $wprocessed=$processed;
373 $countdown=(34701- $processed)/$avg_time;
374 $last_time=time();
375 }
376 //print "\rProcessing Popcon: " . date("i:s",$countdown=$countdown-(time()-$last_time)) . " =>" . (34701-$processed) . " Media $avg_time";
377 $last_time=time();
378 }
379 }
380 #pg_exec($db, "COMMIT") OR DIE("Impossibile commsionare la transazione");
381 }
382 @pg_close($db);
383 }
384 function db_popcon_test() {
385 global $dirs,$dbconn;
386 $db = pg_pconnect($dbconn) or die(pg_errormessage($db));
387 $filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
388 $regexp="/Package:\s(\S+)\s+(\d+)\s+(\d+)/";
389 $zp = gzfile($filename, "r") or die("Could not open $filename");
390 if(filesize($filename)>20) {
391 foreach ($zp as $line) {
392 if (preg_match($regexp,$line,$matches)) {
393 # print "Name => $matches[1] \n Install => $matches[2] \n Vote => $matches[3]\n";
394 print "\r" . $i++;
395 #pg_unbuffered_query($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'");
396 }
397 }
398 }
399 @pg_close($db);
400 }
401 function file_put_contents($filename, $content, $flags = 0) {
402 if (!($file = fopen($filename,$flags))) return false;
403 else {
404 $n = fwrite($file, $content);
405 fclose($file);
406 return true;
407 }
408 }
409 function keep_popcon() {
410 global $dirs,$dbconn;
411 $localfile=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
412 $remotefile="http://popcon.debian.org/all-popcon-results.txt.gz";
413 @file_put_contents($localfile,file_get_contents($remotefile),'w') or die("Errore nello scaricare il file popcon\n");
414 }
415 function is_updated ($up_vers,$debian_vers) {
416 if ($up_vers=='') return 0;
417 passthru("dpkg --compare-versions '$up_vers' eq '" . vers_conv($debian_vers) . "'",$great);
418 if ($great!=0) return 1;
419 else return 0;
420 }
421 function db_upstream($initial='%') {
422 global $dirs,$dbconn;
423 check_db();
424 $db = pg_connect($dbconn) ;
425 $rsql=pg_exec($db, "SELECT id, pkgs.name,pkgs.dist,up_version,version,watch FROM pkgs WHERE watch is not null AND name LIKE '$initial%' ORDER BY name DESC");
426 $tot=pg_numrows($rsql);
427 while ($res_array=pg_fetch_array($rsql)) {
428 $uscan_res=uscan_foo($res_array['name'],$res_array['version'],$res_array['watch']);
429 if($uscan_res[1] !=null) $uscan_res[1]=addslashes($uscan_res[1]);
430 #print $uscan_res[0] . "\n Name => $res_array[name]\n";
431 $version=$res_array[version];
432 print "Version=>" . $uscan_res[3] . "\n";
433 if ($uscan_res[3]!=null) $version=$uscan_res[3];
434 $updated=is_updated($uscan_res[0],$res_array[version]);
435 if ($uscan_res[0]!=$res_array[up_version]) ($keep_changes=1);
436 else $keep_changes=0;
437 pg_exec($db, "UPDATE pkgs SET version='". $version . "', up_version='" . $uscan_res[0] . "',up_url='" . $uscan_res[2] . "', watch_warn='" . $uscan_res[1] . "',keep_changes='$keep_changes', wwiz_version=NULL, wwiz_type=NULL, wwiz=NULL, updated='$updated' WHERE id=$res_array[id]") or Die("Error in update pkgs upstream");
438 #print "\rUpdated " . $res_array['name'] . " => $updated Up Version => ". $uscan_res[0];
439 print "\rUpstream left => " . $tot --;
440 # }
441 # else print "Added ". $res_array['name'] ."\n";
442 # if ($great==0) {
443
444 # print "Package=> " . $res_array['name'] . "\n";
445 # print "Version => " . $res_array['version'] . "\n";
446 # print "Upstream Ver=> " . $uscan_res[0] . "\n";
447 #print "Watch warings=> " . $uscan_res[1] . "\n";
448 # }
449 }
450 print "\n";
451 pg_close($db);
452 }
453 function check_url ($url) {
454 $url_p = parse_url ($url);
455 if (isset ($url_p["host"])) $host = $url_p["host"];
456 else return false;
457 preg_match("/(http|ftp)/",$host,$matches);
458 if ($matches[1]=='http') $fp = fsockopen ($host, 80, $errno, $errstr, 10);
459 elseif ($matches[1]=='ftp') $fp2 = fsockopen ($host, 21, $errno, $errstr, 10);
460 if (!$fp AND !$fp2) return false;
461 else return true;
462 }
463 function keep_remote ($localfile,$remotefile) {
464 #if (!check_url($remotefile)) return false;
465 $rfp = @fopen ($remotefile, "rb");
466 if (!$rfp) {
467 return false;
468 }
469 $lfp = fopen ($localfile, "w");
470 ini_set ( "memory_limit", "100M");
471 while ($data = fread($rfp, 1024*8)) {
472 $bytes+=1024*8;
473 #print "Downlaoding $remotefile: $bytes\r";
474 fwrite($lfp, $data);
475 }
476 ini_set ( "memory_limit", "15M");
477 fclose($lfp);
478 fclose($rfp);
479
480 return true;
481 }
482 function up_changes($src_name='%') {
483 global $dbconn;
484 $changes_files=array("CHANGES","NEWS","Changelog","ChangeLog","changelog");
485 $db = pg_connect($dbconn) ;
486 $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!='' AND keep_changes='t' AND name LIKE '$src_name%'");
487 $tot=pg_numrows($rsql);
488 while ($res_array=pg_fetch_array($rsql)) {
489 $localfile="/tmp/dehs-" . $res_array[id] . ".tar.gz";
490 #print "I'm downloading $res_array[up_url]\r";
491 $res=keep_remote($localfile,$res_array['up_url']);
492 #print "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]";
493 if (!$res) {
494 pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");
495 $notdown++;
496 $tot--;
497 print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
498 #print "\n Not Downlaoded: $res_array[up_url]\n";
499 continue;
500 }
501 foreach($changes_files as $filename) {
502 $command="tar -zxOf " . $localfile . " *$filename 2>/dev/null";
503 exec($command,$up_changes,$cmdres);
504 $up_changes=rtrim(implode("\n",$up_changes));
505 if ($cmdres==0 AND strlen($up_changes)>20) {
506 $sw=true;
507 $up_changes=addslashes($up_changes);
508 $conv_res=iconv("ISO-8859-1","UTF-8",$up_changes);
509 if ($conv_res) $up_changes=$conv_res;
510 $up_changes_sql="UPDATE pkgs SET up_changes='$up_changes', keep_changes='f' WHERE id=$res_array[id]";
511 pg_exec($db,$up_changes_sql);
512 #print "\nFounded $filename on $res_array[up_url]\n";
513 $found++;
514 break;
515 }
516 }
517 if (!$sw) {
518 pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");
519 #print "\n Not Founded any on $res_array[up_url]\n";
520 $notfound++;
521 }
522 unlink($localfile);
523 print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
524 }
525 pg_close($db);
526 }
527 function uscan_foo($name,$version,$watch){
528 $fw=fopen("/tmp/$name" . "_watch","w");
529 fwrite($fw,$watch);
530 fclose($fw);
531
532 $uscan_res=shell_exec($base . "uscan --dehs --package $name --upstream-version " . vers_conv($version) . " --watchfile /tmp/$name" . "_watch 2>/dev/null 2>&1");
533 unlink("/tmp/$name" . "_watch");
534 $p = xml_parser_create();
535 xml_parse_into_struct($p, $uscan_res, $vals, $index);
536 xml_parser_free($p);
537 $upstream=$vals[$index['UPSTREAM-VERSION'][0]][value];
538 $up_url=$vals[$index['UPSTREAM-URL'][0]][value];
539 $deb_vers=$vals[$index['DEBIAN-UVERSION'][0]][value];
540 $warnings=$vals[$index['WARNINGS'][0]][value];
541 if ($upstream=='') $upstream=null;
542 if ($up_url=='') $up_url=null;
543 if ($deb_vers=='') $deb_vers=null;
544 if ($warnings=='') $warnings=null;
545 return array($upstream,$warnings,$up_url,$deb_vers);
546 }
547 function db_up_error() {
548 global $dbconn;
549 check_db();
550 $db = pg_connect($dbconn) ;
551 $bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done";
552 print "Downloading bugs list =>";
553 $ferrors=file_get_contents($bugurl) or die ("I can't keep error page from bugs.debian.org");
554 print "Done\n";
555 $ferrors=explode("<li>",$ferrors);
556 $totline=count($ferrors);
557 #$errors[]=array();
558 #array_pop($errors);
559 print "Total line => $totline\n";
560 foreach ($ferrors as $line) {
561 # print $line . "\n";
562 $founded=preg_match("/Package:.+<strong>([^<]+)<\/strong><\/a>.+<strong>((\d+)\syear\sand\s|)(\d+)\sdays/sS",$line,$matches);
563 if ($founded) {
564 $errors[$matches[1]][err_tot]++;
565 $errors[$matches[1]][days]+=$matches[3]*365+$matches[4];
566 #if ($matches[3]) print "Anni $matches[3]";
567 #print "Giorni $matches[4]\n";
568 }
569 print "\rLine left => " . $totline--;
570 }
571 #arsort($errors);
572 $total=count($errors);
573 foreach ($errors as $error=>$value) {
574 pg_exec($db, "UPDATE binpkgs SET up_error=" . $value[err_tot] . ",avg_error_date=" . $value[days]/$value[err_tot] . " WHERE bin_name='$error'") or Die("Error in upstream error query");
575 #print "Pkg: $error Errori: $value[err_tot] Average Days: " . $value[days]/$value[err_tot] . "\n";
576 print "\rErrors left => " . $total--;
577 }
578 pg_close($db);
579 }
580 function watch_fill($src_name='%'){
581 global $dirs,$dbconn;
582 $db = pg_pconnect($dbconn) or DIE ("Db error");
583 $checked=$errors=$notfound=$verok=$notmatch=0;
584 $sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN
585 (Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist)
586 WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name%' ORDER BY mpop_inst DESC";
587 $rsql=pg_exec($db, $sqlstring);
588 while ($res_array=pg_fetch_array($rsql)) {
589 if (substr($res_array["name"],0,3)=='lib') $initial=substr($res_array["name"],0,4);
590 else $initial=substr($res_array["name"],0,1);
591 $url="http://packages.debian.org/changelogs/pool/" .
592 $res_array["section"] . "/" .
593 $initial . "/" .
594 $res_array["name"] ."/" .
595 $res_array["name"] . "_" .
596 $res_array["version"] . "/copyright";
597 $contents=@file_get_contents($url) or '';
598 $space='';
599 if (!$contents) {
600 $url="http://packages.debian.org/changelogs/pool/" .
601 $res_array["section"] . "/" .
602 $initial . "/" .
603 $res_array["name"] ."/" .
604 $res_array["name"] . "_" .
605 $res_array["version"] . "/" . $res_array["name"] . ".copyright";
606 $contents=@file_get_contents($url) or '';
607 }
608 if (!$contents) {
609 $notfound++;
610 pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz_type='no_cright' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
611 }
612 else {
613 $res=preg_match ('/(sf\.net|sourceforge\.net)/',$contents,$matches);
614 if ($res) {
615 preg_match("/([^\d]+)/",$res_array[name],$dmatches);
616 $depured_debian_name=$dmatches[1];
617 $matches[1]="ftp://download.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/";
618 }
619 else {
620 unset($matches);
621 $res=preg_match ('/((ftp:\/\/|http:\/\/)\S+(html|htm|\/(?(?=.+(tar|tgz))|[^>\s\)\a]+)))/',$contents,$matches);
622 $final=substr($matches[1],-1);
623 if ($final=='.') {
624 $matches[1]=substr($matches[1],0,-1);
625 $final=substr($matches[1],-1);
626 }
627 if ($matches[2]=="http://") $space=' ' . "(?:.*/)?";
628 elseif ($final!='/') $matches[1].='/';
629 $matches[1]=preg_replace("/((http|ftp):\/\/.+):(.*)/",'$1' . '$3', $matches[1],1);
630 }
631 if (!$res) {
632 $notmatch++;
633 $contents=iconv("ISO-8859-1","UTF-8",$contents);
634 pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
635 }
636 else {
637 if (!preg_match("/$res_array[name]/",$matches[1])) {
638 preg_match("/([^\d]+)/",$res_array[name],$dmatches);
639 $depured_debian_name=$dmatches[1];
640 }
641 else $depured_debian_name=$res_array[name];
642 $watch="version=3\n" .
643 $matches[1] . $space . $depured_debian_name . "-?_?([\d+\.]+|\d+)\.tar.* debian uupdate";
644 $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch);
645 if ($uscan_res[1] AND !$uscan_res[0]) {
646 pg_exec($db, "UPDATE pkgs SET wwiz_version=NULL, wwiz='" . addslashes($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
647 $errors++;
648 }
649 else {
650 $verok++;
651 if ($uscan_res[0]!=$res_array[wwiz_version]) ($keep_changes=1);
652 else $keep_changes=0;
653 pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($watch) ."', wwiz_type='watch', wwiz_version='" . $uscan_res[0] . "', up_url='" . $uscan_res[2] . "', keep_changes='$keep_changes' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
654 }
655 }
656 }
657 print "Package checked: " . $checked++ .
658 " OK: $verok Error: $errors Not Matched: $notmatch\r";
659 }
660 print "\nCopyright files not founded: $notfound\n";
661 print "Automatic genereted watch file failure: $errors\n";
662 pg_close($db);
663 }
664 function dbqa_conn ($dbname,$mode) {
665 $id = dba_open($dbname, $mode, "db4");
666 if (!$id) {
667 echo "dba_open failed\n";
668 exit;
669 }
670 return $id;
671 }
672 function dehsqa_db () {
673 #to complete!!!
674 #print phpinfo();
675 global $dirs,$dbconn;
676 $dir1='/org/alioth.debian.org/chroot/home/groups/dehs/htdocs/';
677 $dir2="/org/qa.debian.org/data/dehs/";
678 $dists=array("unstable","experimental");
679 foreach ($dists as $dist) {
680 $id=dbqa_conn($dir1 . "dehs_qa_" . $dist . ".db",'n');
681 $db = pg_pconnect($dbconn) or DIE ("Db error");
682 $sql="SELECT name, id, up_version, wwiz_version, watch, wwiz_type, watch_warn, updated FROM pkgs WHERE dist='$dist'";
683 $rsql=pg_exec($db, $sql);
684 while($res_array=pg_fetch_array($rsql)) {
685 if (!$res_array[watch]) $version="N/A";
686 elseif (!$res_array[up_version]) $version="Error";
687 else $version=$res_array[up_version];
688 if ($res_array[wwiz_type]=="watch") $wwiz=$res_array[wwiz_version];
689 else $wwiz=$res_array[wwiz_type];
690 $xml="<data><id>$res_array[id]</id>" .
691 "<up_version>$version</up_version>" .
692 "<wwiz>$wwiz</wwiz>" .
693 "</data>";
694 $value=addslashes($xml);
695 dba_replace($res_array[name],$xml,$id);
696 }
697 dba_optimize($id);
698 dba_close($id);
699 pg_close($db);
700 }
701 }
702 function md5_of_file($inFile) {
703 if (file_exists($inFile)) {
704 $fd = fopen($inFile, 'r');
705 $fileContents = fread($fd, filesize($inFile));
706 fclose ($fd);
707 return md5($fileContents);
708 } else {
709 return false;
710 }
711 }
712 function update_all() {
713 global $email;
714 $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
715 db_add();
716 keep_popcon();
717 db_popcon();
718 keep_diff();
719 db_up_error();
720 db_upstream();
721 watch_fill();
722 up_changes();
723 dehsqa_db();
724 $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
725 $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";
726 mail($email, "DEHS report", $messaggio,
727 "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .
728 "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .
729 "X-Mailer: PHP/" . phpversion());
730 }
731 ?>

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.5