/[dehs]/trunk/dehs_pg.php
ViewVC logotype

Contents of /trunk/dehs_pg.php

Parent Directory Parent Directory | Revision Log Revision Log


Revision 23 - (hide annotations) (download)
Mon Jul 19 16:39:33 2004 UTC (8 years, 10 months ago) by evaso-guest
File size: 30289 byte(s)
Some fixes over watch_fill function
1 evaso-guest 5 #!/usr/bin/php4 -q
2     <?
3     /*
4    
5     Originally written by Stefano Fabri <bluefuture@nospam@email.it>
6     Copyright 2004, Stefano Fabri
7    
8     This program is free software; you can redistribute it and/or modify
9     it under the terms of the GNU General Public License as published by
10     the Free Software Foundation; either version 2 of the License, or
11     (at your option) any later version.
12    
13     This program is distributed in the hope that it will be useful,
14     but WITHOUT ANY WARRANTY; without even the implied warranty of
15     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16     GNU General Public License for more details.
17    
18     You should have received a copy of the GNU General Public License
19     along with this program; if not, write to the Free Software
20     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21     */
22     // stabilire una connessione
23     #ini_set ( "error_reporting", "E_COMPILE_ERROR|E_ERROR|E_CORE_ERROR");
24     ini_set ("html_errors", "off");
25     set_time_limit(0);
26     ini_set ( "memory_limit", "15M");
27 evaso-guest 13 #print_R($_SERVER);
28     require "dehs.config.inc";
29 evaso-guest 5 dir_ctrl($dirs);
30 evaso-guest 13 if (!function_exists('file_get_contents')) {
31     function file_get_contents($filename) {
32 evaso-guest 17 if (!($file = fopen($filename,"r"))) return false;
33 evaso-guest 13 else {
34     $contents = "";
35     do {
36     $data = fread($file, 4096);
37     if (strlen($data) == 0) {
38     break;
39     }
40     $contents .= $data;
41     } while (true);
42     fclose($file);
43     return $contents;
44     }
45     }
46     }
47 evaso-guest 5 if (in_array("-db_add",$argv)) db_add($argv[2]);
48     elseif (in_array("-db_query",$argv)) db_query($argv[2],$argv[3] );
49     elseif (in_array("-keep_diff",$argv)) keep_diff($argv[2]);
50     elseif (in_array("-keep_sources",$argv)) keep_sources();
51     elseif (in_array("-db_popcon",$argv)) db_popcon($argv[2]);
52     elseif (in_array("-keep_popcon",$argv)) keep_popcon($argv[2]);
53     elseif (in_array("-db_upstream",$argv)) db_upstream($argv[2]);
54     elseif (in_array("-update_all",$argv)) update_all();
55     elseif (in_array("-check_db",$argv)) check_db();
56     elseif (in_array("-db_up_error",$argv)) db_up_error();
57 evaso-guest 17 elseif (in_array("-up_changes",$argv)) up_changes($argv[2]);
58     elseif (in_array("-watchf",$argv)) watch_fill($argv[2]);
59 evaso-guest 5 elseif (in_array("-kb_total",$argv)) print "Diff
60     #stat_up_error();
61     #stat_up_error();to download Kb => " . kb_total($argv[2]) . "\n" ;
62     elseif (in_array("-watch_stat",$argv)) watch_stat();
63     else print "Command $argv[1] not found\n";
64     #stat_up_error();
65     function dir_ctrl($dirs) {
66     foreach ($dirs as $dir) {
67 evaso-guest 20 if (!is_dir($dir)) exec("mkdir -m 0740 -p $dir",$output,$res);
68     if ($res!=0) die("Error creating basic dir: $dir\n");
69 evaso-guest 5 }
70     }
71     function ext_watch ($file,$pkg,$version){
72     if (!file_exists($file)) return "";
73     $zp=gzopen($file,"r") or die("Fail to open $file");
74     while (!gzeof($zp)) {
75     $line = gzgets ($zp,4096);
76     if (preg_match("/\+\+\+\s.+\/debian\/watch/",$line)) {
77     $line = gzgets ($zp,4096);
78     preg_match ("/.+\+\d+\,(\d+)\s/",$line,$watch_lenght);
79     for ($i = 1; $i <= $watch_lenght[1]; $i++) {
80     $line=gzgets ($zp,4096);
81     $watch .=substr($line,1);
82     }
83     break;
84     }
85     }
86     if ($watch!=null) return $watch;
87     else return NULL;
88     }
89     function ftp_cn($login,$password,$ftp_server) {
90 evaso-guest 17 $conn_id = @ftp_connect($ftp_server) or DIE("Error: Cannot contact ftp server $ftp_server\n");
91 evaso-guest 5 // login con user name e password
92     ftp_pasv($conn_id,1);
93     $login_result = ftp_login($conn_id, "anonymous","bo@bo.org");
94     // controllo della connessione
95     if ((!$conn_id) || (!$login_result)) {
96     echo "Ftp connection failed!\n";
97     echo "Failed connection to $ftp_server\n";
98     die();
99     } else {
100     return $conn_id;
101     }
102     }
103     function keep_diff($initial) {
104     global $dirs,$dbconn,$mirrors;
105     check_db();
106     $conn_normal=ftp_cn("anonymous","bo@bo.org",$mirrors['unstable']);
107     $conn_non_us=ftp_cn("anonymous","bo@bo.org",$mirrors["non-US"]);
108     $db = pg_pconnect($dbconn) ;
109     $rsql=pg_exec($db, "SELECT * FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)") OR DIE("Error in diff query\n");
110     $total=pg_numrows($rsql);
111     $btotal=byte_total($db,$initial);
112     print "Total Diff to downlaod Kb => " . number_format ($btotal/1024, 2, ",",".") . "\n";
113     while ($res_array=pg_fetch_array($rsql)) {
114     if ($res_array['dist']=='non-US') $conn_id = $conn_non_us;
115     else $conn_id = $conn_normal;
116     if ($localfile=download_diffs($conn_id,$res_array[name],$res_array[version],$res_array['dir'],$res_array[md5_diff],$res_array[md5_atsource],$res_array[bytes],$res_array[dist])) {
117     $watch=ext_watch($localfile,$res_array[name],$res_array[version]);
118     unlink($localfile);
119     if ($watch!=NULL) {
120     $watch= addslashes($watch);
121     pg_exec($db, "UPDATE pkgs SET watch='" . $watch . "', md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Query error");
122     }
123     else pg_exec($db, "UPDATE pkgs SET watch=NULL, md5_diff='" . $res_array[md5_atsource] . "' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'") or Die("Errore nella query");
124     }
125     $btotal=$btotal-$res_array['bytes'];
126     number_format ($btotal/1024, 2, ",",".");
127     print "\rDiffs left => " . $total-- . " Bytes left => " . number_format ($btotal/1024, 2, ",",".") . " - " . $res_array[name];
128     }
129     ftp_quit($conn_normal);
130     ftp_quit($conn_non_us);
131     pg_close($db);
132     return "Fatto\n";
133     }
134     function download_diffs(&$conn_id,$pkg,$version,$directory,$md5_diff,$md5_atsource,$diff_kb,$dist) {
135     global $dirs,$mirrors;
136     $localfile=$dirs[diffs_dir] . "/$pkg" . "_" . "$version.diff.gz";
137     if ($dist=="non-US") $remotefile="/debian-non-US/$directory/$pkg" . "_" . "$version.diff.gz";
138     else $remotefile="/debian/$directory/$pkg" . "_" . "$version.diff.gz";
139     #print "$localfile - $remotefile\n";
140     if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
141     # print "$pkg Diff.gz => Downloaded\n";
142     if(md5_of_file($localfile)==$md5_atsource) {
143     #print "MD5 => OK\n";
144     #unlink($localfile);
145     return $localfile;
146     }
147     }
148     print "Download Diff.gz of package $pkg => Fallito\n";
149     return false;
150     }
151     function db_add($initial=''){
152     global $dirs,$dists,$dbconn;
153     check_db();
154     keep_source();
155 evaso-guest 17 $db = pg_connect($dbconn) or die(pg_errormessage($db));
156 evaso-guest 5 pg_exec($db, "CREATE TEMP TABLE pkgs_atsrc (name text,dist text)") or die('Error creating temp table pkgs_atsrc');
157     pg_exec($db, "CREATE UNIQUE INDEX idxdis on pkgs_atsrc (name,dist)") or die('Error creating index on temp table');
158     pg_exec($db, "CREATE TEMP TABLE bin_atsrc (name text,bin_name text, dist text)") or die('Error creating temp table pkgs_atsrc');
159     pg_exec($db, "CREATE UNIQUE INDEX idxbis on bin_atsrc (name,bin_name,dist)") or die('Error creating index on temp table bin_atsrc');
160     foreach ($dists as $dist=>$sections) {
161     foreach ($sections as $section) {
162     $filename=$dirs[sources_dir] . "/$dist/$section/Sources.gz";
163     print "\nI'm parsing $dist/$section => Sources.gz\n";
164     # 1 Pacchetto - 2 versione - 3 np - 4 Directory - 5 md5
165 evaso-guest 20 $regexp="/Package:\s($initial.*)[^a]Binary:\s(.+)[^a]Version:\s(\d+:)?(.+)[^a]Priority.+Maintainer:.+<([^>]+)>[^a].+Directory:\s(.+)[^a]Files:.+\s(\S+)\s(\d+)\s" . '\1\S+' . "\.diff\.gz[^a](Uploaders:\s(.+)[^a])?/Ssi";
166 evaso-guest 5 $zp = gzopen($filename, "r") or die("Could not open $filename");
167     $extracted="/tmp/$dist_$section_Sources.gz";
168     exec("gzip -c -d $filename > $extracted");
169     $sourcesize=filesize($extracted);
170 evaso-guest 20 unlink($extracted);
171 evaso-guest 5 if(filesize($filename)>20) {
172     while (!gzeof($zp)) {
173     $line='';
174     while ($line!="\n") {
175     $line = gzgets ($zp,4096) ;
176     $buff1 .= $line;
177     }
178     if (preg_match($regexp,$buff1,$matches)) {
179 evaso-guest 20 if ($matches[10]) {
180     $matches[10]=iconv("ISO-8859-1","UTF-8",$matches[10]);
181     $matches[10]=addslashes($matches[10]);
182     }
183 evaso-guest 5 @pg_exec($db,"INSERT INTO pkgs_atsrc (name,dist) VALUES ('$matches[1]','$dist')") OR DIE("Temp table pkgs_atsrc query error");
184 evaso-guest 20 $rst=@pg_exec($db, "INSERT INTO pkgs (name,version,maint,dir,md5_atsource,bytes,dist,section,uploaders) VALUES ('$matches[1]','$matches[4]','$matches[5]','$matches[6]','$matches[7]','$matches[8]','$dist','$section','$matches[10]')") ;
185 evaso-guest 5 if (!$rst) {
186 evaso-guest 20 $rst=@pg_exec($db, "UPDATE pkgs SET name='$matches[1]',version='$matches[4]',maint='$matches[5]',dir='$matches[6]',md5_atsource='$matches[7]',bytes='$matches[8]',dist='$dist',section='$section',uploaders='$matches[10]' WHERE name='$matches[1]' AND dist='$dist'") OR DIE ("\nDb adding error =>" . pg_errormessage() . "\n");
187     }
188 evaso-guest 5 $bin_names=split(",", $matches[2]);
189     foreach ($bin_names as $bin_name) {
190     pg_exec($db,"INSERT INTO bin_atsrc (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") OR DIE("Temp table pkgs_atsrc query error");
191     $rst=@pg_exec($db, "INSERT INTO binpkgs (name,bin_name,dist) VALUES ('$matches[1]','" . trim($bin_name) . "','$dist')") ;
192     }
193     }
194     else {
195     preg_match("/Package:\s(.+)[^a]Binary/im",$buff1,$matches);
196     if(strncmp($initial,$matches[1],strlen($initial))<0) break;
197     }
198     $left=$sourcesize-gztell($zp);
199     print "\rSource file Left => $left";
200     $buff1="";
201     unset($matches);
202     }
203     }
204     gzclose($zp) ;
205     }
206     }
207     clear_db($db);
208     pg_close($db);
209     }
210     function db_query($pkg,$dist) {
211     global $dirs,$dbconn;
212 evaso-guest 17 $db = pg_pconnect($dbconn ) or die(pg_errormessage($db));
213 evaso-guest 5 $rsql=pg_exec($db, "SELECT pkgs.* ,mpop_inst FROM (SELECT name,MAX(binpkgs.pop_inst) AS mpop_inst FROM binpkgs GROUP BY name)
214     AS binpkgs INNER JOIN pkgs ON pkgs.name=binpkgs.name
215     WHERE pkgs.name='$pkg' AND dist='$dist';");
216     if (pg_numrows($rsql)==1) {
217     $res_array=pg_fetch_array($rsql);
218     print "Package=> " . $res_array['name'] . "\n";
219     print "Version => " . $res_array['version'] . "\n";
220     print "Diff MD5 Downloaded => " . $res_array['md5_diff'] . "\n";
221     print "Diff MD5 at Sources => " . $res_array['md5_atsource'] . "\n";
222     print "Distribution => " . $res_array['dist'] . "\n";
223     print "Section => " . $res_array['section'] . "\n";
224     print "KBytes => " . number_format ( $res_array['bytes']/1024, 2, ",",".") . "\n";
225     print "Pop Inst=>" . $res_array['mpop_inst'] . "\n";
226     if ($res_array['watch']) {
227     $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$res_array['watch']);
228     print "Version Upstream => " . $uscan_res[0] . "\n";
229     print "Watch => " . $res_array['watch'];
230     print "Watch Warings => " . $uscan_res[1];
231     }
232     else print "Watch => Not present" . "\n";
233     }
234     else echo "Package=> " . $pkg . "\nNot found\n!!";
235     pg_close($db);
236     }
237     function byte_total(&$db,$initial){
238     global $dirs;
239     $rsql=pg_exec ($db,"Select sum(bytes) as totbytes FROM pkgs WHERE name like '$initial%' AND (md5_diff!=md5_atsource or md5_diff is null)");
240     $res_array=pg_fetch_array($rsql);
241     return $res_array[totbytes] ;
242     }
243     function keep_source (){
244     global $dirs,$dists,$mirrors;
245     foreach ($dists as $dist=>$sections) {
246     $server=$mirrors[$dist];
247     $conn_id = ftp_cn("anonymous","bo@bo.org",$server);
248     foreach ($sections as $section) {
249     $localfile=$dirs[sources_dir] . "/$dist/$section/Sources.gz";
250     $remotefile=remote_dist_dir($dist) . "/$dist/$section/source/Sources.gz";
251     $sources_info=md5_source($section,$dist);
252 evaso-guest 20 if (!file_exists ($dirs[sources_dir] . "/$dist/")) mkdir ( $dirs[sources_dir] . "/$dist",0740);
253     if (!file_exists ($dirs[sources_dir] . "/$dist/$section")) mkdir ($dirs[sources_dir] . "/$dist/$section",0740);
254     if (!file_exists($localfile) OR md5_of_file($localfile)!=$sources_info[md5]) {
255     print "Downloading $remotefile Bytes $sources_info[bytes]=> ";
256     if (ftp_get($conn_id,$localfile,$remotefile,FTP_BINARY)) {
257     print "Done\n";
258     if(md5_of_file($localfile)==$sources_info[md5]) print "MD5 => OK\n";
259     }
260     else {
261     unlink("$localfile");
262     print "Fail\n";
263     die ("Error in source file");
264     }
265     }
266 evaso-guest 5 }
267 evaso-guest 20 ftp_quit($conn_id);
268 evaso-guest 5 }
269     }
270     function md5_source($section,$dist) {
271     global $mirrors;
272     $relase=file_get_contents ("ftp://$mirrors[$dist]" . remote_dist_dir($dist) . "/$dist/Release") or die("Failed to check MD5 of sources");
273     preg_match ("/\s([^\s]+)\s+(\d+)\s$section\/source\/Sources\.gz/i", $relase,$matches);
274     return array("md5"=>$matches[1],"bytes"=>$matches[2]);
275     # 183dea9ca3c2e35ffd6cf5eee9501c21 2941637 main/binary-i386/Packages.gz
276     }
277    
278     function vers_conv($debvers) {
279 evaso-guest 15 preg_match("/(.+)(ds-|-)/",$debvers,$matches);
280 evaso-guest 5 return $matches[1];
281     }
282     function check_db() {
283     global $dirs,$dbconn;
284 evaso-guest 17 $db = @pg_connect($dbconn) or die(pg_last_notice());
285 evaso-guest 5 if (!pg_table_exists($db,"pkgs")){
286 evaso-guest 20 pg_exec($db, "CREATE TABLE pkgs (id serial PRIMARY KEY,name text,version text,maint text, uploaders text, dir text,watch text,md5_diff text,md5_atsource text,bytes numeric,dist text,section text,up_version text,up_changes text, up_url text,watch_warn text,updated bool, keep_changes bool, wwiz text,wwiz_version text, wwiz_type text)") or die('Error creating table pkgs\n');
287 evaso-guest 5 pg_exec($db, "CREATE UNIQUE INDEX idxname on pkgs (name,dist)") or die('Errore creating index - ' . pg_last_error($db));
288     $sw=true;
289     }
290     if (!pg_table_exists($db,"binpkgs")){
291     pg_exec($db, "CREATE TABLE binpkgs (id serial PRIMARY KEY, name text,bin_name text,dist text, pop_inst numeric,pop_vote numeric, up_error numeric,avg_error_date numeric)") or die('Error creating table binpkgs\n');
292     pg_exec($db, "CREATE UNIQUE INDEX idxbin on binpkgs (name,bin_name,dist)") or die('Errore creating index - ' . pg_last_error($db));
293     $sw=true;
294     }
295     #if (!pg_table_exists($db,"upstream")){
296     # pg_exec($db, "CREATE TABLE upstream (name text,dist text,up_version text,watch_warn text,updated bool)") or die('Errore nel creare la tabella');
297     # pg_exec($db, "CREATE UNIQUE INDEX idxupstrm on upstream (name,dist)") or die('Errore nel creare gli indici ' . pg_last_error($db));
298     # $sw=true;
299     #}
300     pg_close($db);
301     if($sw) return $sw;
302     else return false;
303     }
304     function pg_table_exists (&$dblink, $table) {
305     $sql = "SELECT relname as tbl FROM pg_class WHERE relname = '$table'";
306     if ($res = pg_exec ($dblink, $sql)) {
307     return pg_numrows($res) > 0;
308     } else {
309     return false; // or throw exception
310     }
311     }
312    
313    
314     function watch_stat() {
315     global $dirs,$dbconn;
316 evaso-guest 17 $db = pg_pconnect($dbconn) or die(pg_errormessage($db));
317 evaso-guest 5 $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch!=''");
318     $tot_watch=pg_fetch_array($rsql);
319     print "Total packages with Watch =>$tot_watch[0]\n";
320     $rsql=pg_exec ($db,"Select count(name) as totwatch FROM pkgs WHERE watch IS NULL");
321     $tot_nowatch=pg_fetch_array($rsql);
322     pg_close($db);
323     print "Totale packages without Watch => $tot_nowatch[0]\n";
324     print "Watch presence rate => " . number_format ($tot_watch[0]/($tot_nowatch[0]+$tot_watch[0])*100, 2, ",",".") . "%\n";
325     }
326     function remote_dist_dir($dist) {
327 evaso-guest 20 if ($dist=="non-US") return "/debian-non-US/dists/unstable";
328 evaso-guest 5 else return "/debian/dists";
329     }
330     function clear_db(&$db) {
331     #global $dbconn;
332     #if (check_db()) die("Il Database era vuoto\n");
333     #keep_source();
334     #$db = pg_pconnect($dbconn) or die(pg_result_error($db));
335     $sql1= "Delete from binpkgs where id NOT IN
336     (select id FROM binpkgs INNER JOIN bin_atsrc USING (name,bin_name,dist))" ;
337     $sql2= "Delete from pkgs where id NOT IN (select id FROM pkgs INNER JOIN pkgs_atsrc USING (name,dist))" ;
338     #$sql3= "Delete from upstream where name NOT IN
339     # (select name FROM pkgs_atsrc) AND dist NOT IN (select dist FROM pkgs_atsrc)" ;
340     if(pg_exec($db,$sql1) AND pg_exec($db,$sql2)) print "Db clear complete\n";
341     }
342     function db_popcon($initial='') {
343     global $dirs,$dbconn;
344 evaso-guest 17 $db = pg_pconnect($dbconn) or die(pg_errormessage($db));
345 evaso-guest 5 $filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
346     $regexp="/Package:\s($initial\S*)\s+(\d+)\s+(\d+)/iS";
347     $zp = gzopen($filename, "r") or die("Could not open $filename");
348     $tin=time();
349     if(filesize($filename)>20) {
350     #pg_exec($db, "BEGIN TRANSACTION") OR DIE("Impossibile iniziare la transazione");
351     while (!gzeof($zp)) {
352     $line = gzgets ($zp,4096) ;
353     if (preg_match($regexp,$line,$matches)) {
354     #print "\rName => $matches[1] Install => $matches[2] Vote => $matches[3]";
355     @pg_exec($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'");
356     $processed+=1;
357     $elapsed_time=time()-$tin;
358     if (($elapsed_time % 20)==0 AND $updtime!=$elapsed_time) {
359     $updtime=$elapsed_time;
360     $wprocessed=$processed-$wprocessed;
361     $avg_time=$wprocessed/20;
362     $wprocessed=$processed;
363     $countdown=(34701- $processed)/$avg_time;
364     $last_time=time();
365     }
366     print "\rProcessing Popcon: " . date("i:s",$countdown=$countdown-(time()-$last_time)) . " =>" . (34701-$processed) . " Media $avg_time";
367     $last_time=time();
368     }
369     }
370     #pg_exec($db, "COMMIT") OR DIE("Impossibile commsionare la transazione");
371     }
372     @pg_close($db);
373     }
374     function db_popcon_test() {
375     global $dirs,$dbconn;
376 evaso-guest 17 $db = pg_pconnect($dbconn) or die(pg_errormessage($db));
377 evaso-guest 5 $filename=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
378     $regexp="/Package:\s(\S+)\s+(\d+)\s+(\d+)/";
379     $zp = gzfile($filename, "r") or die("Could not open $filename");
380     if(filesize($filename)>20) {
381     foreach ($zp as $line) {
382     if (preg_match($regexp,$line,$matches)) {
383     # print "Name => $matches[1] \n Install => $matches[2] \n Vote => $matches[3]\n";
384     print "\r" . $i++;
385     #pg_unbuffered_query($db, "UPDATE binpkgs SET pop_inst='$matches[2]',pop_vote='$matches[3]' WHERE bin_name='$matches[1]'");
386     }
387     }
388     }
389     @pg_close($db);
390     }
391     function file_put_contents($filename, $content, $flags = 0) {
392     if (!($file = fopen($filename,$flags))) return false;
393     else {
394     $n = fwrite($file, $content);
395     fclose($file);
396     return true;
397     }
398     }
399     function keep_popcon() {
400     global $dirs,$dbconn;
401     $localfile=$dirs[popcon_dir] . "/all-popcon-results.txt.gz";
402     $remotefile="http://popcon.debian.org/all-popcon-results.txt.gz";
403     @file_put_contents($localfile,file_get_contents($remotefile),'w') or die("Errore nello scaricare il file popcon\n");
404     }
405 evaso-guest 17 function is_updated ($up_vers,$debian_vers) {
406     if ($up_vers=='') return 0;
407     passthru("dpkg --compare-versions '$up_vers' gt '" . vers_conv($debian_vers) . "'",$great);
408     if ($great==0) return 1;
409     else return 0;
410     }
411     function db_upstream($initial='%') {
412 evaso-guest 5 global $dirs,$dbconn;
413     check_db();
414     $db = pg_connect($dbconn) ;
415 evaso-guest 20 $rsql=pg_exec($db, "SELECT pkgs.name,pkgs.dist,up_version,version,watch FROM pkgs WHERE watch is not null AND name LIKE '$initial%' ORDER BY name DESC");
416 evaso-guest 5 $tot=pg_numrows($rsql);
417     while ($res_array=pg_fetch_array($rsql)) {
418     $uscan_res=uscan_foo($res_array['name'],$res_array['version'],$res_array['watch']);
419     if($uscan_res[1] !=null) $uscan_res[1]=addslashes($uscan_res[1]);
420     #print $uscan_res[0] . "\n Name => $res_array[name]\n";
421 evaso-guest 17 $updated=is_updated($uscan_res[0],$res_array[version]);
422     if ($uscan_res[0]!=$res_array[up_version]) ($keep_changes=1);
423     else $keep_changes=0;
424     pg_exec($db, "UPDATE pkgs SET up_version='" . $uscan_res[0] . "',up_url='" . $uscan_res[2] . "', watch_warn='" . $uscan_res[1] . "',keep_changes='$keep_changes', updated='$updated' WHERE name='$res_array[name]' AND dist='$res_array[dist]'") or Die("Error in update pkgs upstream");
425     #print "\rUpdated " . $res_array['name'] . " => $updated Up Version => ". $uscan_res[0];
426     print "\rUpstream left => " . $tot --;
427     # }
428 evaso-guest 5 # else print "Added ". $res_array['name'] ."\n";
429     # if ($great==0) {
430    
431     # print "Package=> " . $res_array['name'] . "\n";
432     # print "Version => " . $res_array['version'] . "\n";
433     # print "Upstream Ver=> " . $uscan_res[0] . "\n";
434     #print "Watch warings=> " . $uscan_res[1] . "\n";
435     # }
436     }
437     print "\n";
438     pg_close($db);
439     }
440 evaso-guest 20 function check_url ($url) {
441     $url_p = parse_url ($url);
442     if (isset ($url_p["host"])) $host = $url_p["host"];
443     else return false;
444     preg_match("/(http|ftp)/",$host,$matches);
445     if ($matches[1]=='http') $fp = fsockopen ($host, 80, $errno, $errstr, 10);
446     elseif ($matches[1]=='ftp') $fp2 = fsockopen ($host, 21, $errno, $errstr, 10);
447     if (!$fp AND !$fp2) return false;
448     else return true;
449     }
450 evaso-guest 17 function keep_remote ($localfile,$remotefile) {
451 evaso-guest 20 #if (!check_url($remotefile)) return false;
452 evaso-guest 17 $rfp = @fopen ($remotefile, "rb");
453     if (!$rfp) {
454     return false;
455     }
456     $lfp = fopen ($localfile, "w");
457     ini_set ( "memory_limit", "100M");
458     while ($data = fread($rfp, 1024*8)) {
459     $bytes+=1024*8;
460 evaso-guest 20 #print "Downlaoding $remotefile: $bytes\r";
461 evaso-guest 17 fwrite($lfp, $data);
462     }
463     ini_set ( "memory_limit", "15M");
464     fclose($lfp);
465     fclose($rfp);
466    
467     return true;
468     }
469     function up_changes($src_name='%') {
470     global $dbconn;
471     $changes_files=array("CHANGES","NEWS","Changelog","ChangeLog","changelog");
472     $db = pg_connect($dbconn) ;
473     $rsql=pg_exec($db, "SELECT id,up_url FROM pkgs WHERE up_url!='' AND keep_changes='t' AND name LIKE '$src_name%'");
474     $tot=pg_numrows($rsql);
475     while ($res_array=pg_fetch_array($rsql)) {
476     $localfile="/tmp/dehs-" . $res_array[id] . ".tar.gz";
477     #print "I'm downloading $res_array[up_url]\r";
478     $res=keep_remote($localfile,$res_array['up_url']);
479     #print "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]";
480     if (!$res) {
481     pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");
482     $notdown++;
483     $tot--;
484     print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
485     #print "\n Not Downlaoded: $res_array[up_url]\n";
486     continue;
487     }
488     foreach($changes_files as $filename) {
489     $command="tar -zxOf " . $localfile . " *$filename 2>/dev/null";
490     exec($command,$up_changes,$cmdres);
491     $up_changes=rtrim(implode("\n",$up_changes));
492     if ($cmdres==0 AND strlen($up_changes)>20) {
493     $sw=true;
494     $up_changes=addslashes($up_changes);
495     $conv_res=iconv("ISO-8859-1","UTF-8",$up_changes);
496     if ($conv_res) $up_changes=$conv_res;
497     $up_changes_sql="UPDATE pkgs SET up_changes='$up_changes', keep_changes='f' WHERE id=$res_array[id]";
498     pg_exec($db,$up_changes_sql);
499     #print "\nFounded $filename on $res_array[up_url]\n";
500     $found++;
501     break;
502     }
503     }
504     if (!$sw) {
505     pg_exec($db, "UPDATE pkgs SET up_changes='', keep_changes='f' WHERE id=$res_array[id]");
506     #print "\n Not Founded any on $res_array[up_url]\n";
507     $notfound++;
508     }
509     unlink($localfile);
510     print "Packages Left: " . $tot-- . " Found: $found Not found: $notfound Not Down: $notdown\r";
511     }
512     pg_close($db);
513     }
514 evaso-guest 5 function uscan_foo($name,$version,$watch){
515     $fw=fopen("/tmp/$name" . "_watch","w");
516     fwrite($fw,$watch);
517     fclose($fw);
518 evaso-guest 17 $uscan_res=shell_exec("perl uscan3.pl $name " . vers_conv($version) . " /tmp/$name" . "_watch 2>/dev/null 2>&1");
519 evaso-guest 5 unlink("/tmp/$name" . "_watch");
520 evaso-guest 17 preg_match("/\+\+(.+)\+\+\s\+-(.+)-\+/",$uscan_res, $matches);
521 evaso-guest 5 if ($matches[1]!='') $upstream=$matches[1];
522     else $upstream=null;
523 evaso-guest 17 if ($matches[2]!='') $up_url=$matches[2];
524     else $up_url=null;
525     $uscan_res=preg_replace("/(\+\+.+-\+)/", "", $uscan_res);
526 evaso-guest 5 if ($uscan_res=='') $uscan_res=null;
527 evaso-guest 17 return array($upstream,$uscan_res,$up_url);
528 evaso-guest 5 }
529     function db_up_error() {
530     global $dbconn;
531     check_db();
532     $db = pg_connect($dbconn) ;
533     $bugurl="http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=tag&data=upstream&archive=no&show_list_header=no&pend-exc=done";
534     print "Downloading bugs list =>";
535     $ferrors=file_get_contents($bugurl) or die ("I can't keep error page from bugs.debian.org");
536     print "Done\n";
537     $ferrors=explode("<li>",$ferrors);
538     $totline=count($ferrors);
539     #$errors[]=array();
540     #array_pop($errors);
541     print "Total line => $totline\n";
542     foreach ($ferrors as $line) {
543     # print $line . "\n";
544     $founded=preg_match("/Package:.+<strong>([^<]+)<\/strong><\/a>.+<strong>((\d+)\syear\sand\s|)(\d+)\sdays/sS",$line,$matches);
545     if ($founded) {
546     $errors[$matches[1]][err_tot]++;
547     $errors[$matches[1]][days]+=$matches[3]*365+$matches[4];
548     #if ($matches[3]) print "Anni $matches[3]";
549     #print "Giorni $matches[4]\n";
550     }
551     print "\rLine left => " . $totline--;
552     }
553     #arsort($errors);
554     $total=count($errors);
555     foreach ($errors as $error=>$value) {
556     pg_exec($db, "UPDATE binpkgs SET up_error=" . $value[err_tot] . ",avg_error_date=" . $value[days]/$value[err_tot] . " WHERE bin_name='$error'") or Die("Error in upstream error query");
557     #print "Pkg: $error Errori: $value[err_tot] Average Days: " . $value[days]/$value[err_tot] . "\n";
558     print "\rErrors left => " . $total--;
559     }
560     pg_close($db);
561     }
562 evaso-guest 17 function watch_fill($src_name='%'){
563 evaso-guest 13 global $dirs,$dbconn;
564 evaso-guest 17 $db = pg_pconnect($dbconn) or DIE ("Db error");
565 evaso-guest 13 $checked=$errors=$notfound=$verok=$notmatch=0;
566 evaso-guest 17 $sqlstring="SELECT mpop_inst,name,pkgs.version,pkgs.section,pkgs.dist,tot_up_error, tot_avg_error_date, wwiz_version FROM pkgs INNER JOIN
567 evaso-guest 13 (Select max(pop_inst) as mpop_inst,SUM(up_error) as tot_up_error,AVG(avg_error_date) as tot_avg_error_date,name,dist FROM binpkgs GROUP BY name,dist HAVING max(pop_inst) IS NOT NULL) as binpkgs using(name,dist)
568 evaso-guest 17 WHERE (watch IS NULL OR watch='') AND name LIKE '$src_name%' ORDER BY mpop_inst DESC";
569 evaso-guest 13 $rsql=pg_exec($db, $sqlstring);
570     while ($res_array=pg_fetch_array($rsql)) {
571     if (substr($res_array["name"],0,3)=='lib') $initial=substr($res_array["name"],0,4);
572     else $initial=substr($res_array["name"],0,1);
573     $url="http://packages.debian.org/changelogs/pool/" .
574     $res_array["section"] . "/" .
575     $initial . "/" .
576     $res_array["name"] ."/" .
577     $res_array["name"] . "_" .
578     $res_array["version"] . "/copyright";
579     $contents=@file_get_contents($url) or '';
580 evaso-guest 17 $space='';
581 evaso-guest 13 if (!$contents) {
582     $notfound++;
583     pg_exec($db, "UPDATE pkgs SET wwiz_type='no_cright' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
584     }
585     else {
586 evaso-guest 17 $res=preg_match ('/(sf\.net|sourceforge\.net)/',$contents,$matches);
587 evaso-guest 23 if ($res) {
588     preg_match("/([^\d]+)/",$res_array[name],$dmatches);
589     $depured_debian_name=$dmatches[1];
590     $matches[1]="ftp://ftp2.sf.net/pub/sourceforge/" . substr($depured_debian_name,0,1) . "/" . substr($depured_debian_name,0,2) . "/" . $depured_debian_name . "/";
591     }
592 evaso-guest 17 else {
593     unset($matches);
594 evaso-guest 21 $res=preg_match ('/((ftp:\/\/|http:\/\/)\S+(html|htm|\/(?(?=.+(tar|tgz))|[^>\s\)\a]+)))/',$contents,$matches);
595 evaso-guest 17 $final=substr($matches[1],-1);
596 evaso-guest 23 if ($final=='.') {
597     $matches[1]=substr($matches[1],0,-1);
598     $final=substr($matches[1],-1);
599     }
600     if ($matches[2]=="http://") $space=' ' . "(?:.*/)?";
601     elseif ($final!='/') $matches[1].='/';
602     $matches[1]=preg_replace("/((http|ftp):\/\/.+):(.*)/",'$1' . '$3', $matches[1],1);
603 evaso-guest 17 }
604 evaso-guest 13 if (!$res) {
605     $notmatch++;
606 evaso-guest 23 $contents=iconv("ISO-8859-1","UTF-8",$contents);
607 evaso-guest 20 pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($contents) . "', wwiz_type='notmatch', up_url='', up_changes='',keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
608 evaso-guest 13 }
609     else {
610 evaso-guest 23 if (!preg_match("/$res_array[name]/",$matches[1])) {
611     preg_match("/([^\d]+)/",$res_array[name],$dmatches);
612     $depured_debian_name=$dmatches[1];
613     }
614     else $depured_debian_name=$res_array[name];
615 evaso-guest 13 $watch="version=2\n" .
616 evaso-guest 23 $matches[1] . $space . $depured_debian_name . "-([\d+\.]+|\d+)(\.tar|\.tgz)(\.gz|\.bz2|) debian uupdate";
617 evaso-guest 13 $uscan_res=uscan_foo($res_array['name'] ,$res_array['version'],$watch);
618 evaso-guest 15 if ($uscan_res[1] AND !$uscan_res[0]) {
619 evaso-guest 20 pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($uscan_res[1]) . "', wwiz_type='error', up_url='', up_changes='', keep_changes='f' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
620 evaso-guest 13 $errors++;
621     }
622     else {
623     $verok++;
624 evaso-guest 17 if ($uscan_res[0]!=$res_array[wwiz_version]) ($keep_changes=1);
625     else $keep_changes=0;
626     pg_exec($db, "UPDATE pkgs SET wwiz='" . addslashes($watch) ."', wwiz_type='watch', wwiz_version='" . $uscan_res[0] . "', up_url='" . $uscan_res[2] . "', keep_changes='$keep_changes' where name='" . $res_array[name] . "' AND dist='" . $res_array[dist] ."'");
627 evaso-guest 13 }
628     }
629     }
630     print "Package checked: " . $checked++ .
631     " OK: $verok Error: $errors Not Matched: $notmatch\r";
632     }
633 evaso-guest 17 print "\nCopyright files not founded: $notfound\n";
634 evaso-guest 13 print "Automatic genereted watch file failure: $errors\n";
635     pg_close($db);
636     }
637 evaso-guest 5 function update_all() {
638     global $email;
639     $intime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
640     db_add();
641     keep_popcon();
642     db_popcon();
643     keep_diff();
644     db_up_error();
645     db_upstream();
646 evaso-guest 13 watch_fill();
647 evaso-guest 17 up_changes();
648 evaso-guest 5 $endtime=gmdate('D, d M Y H:i:s \C\E\S\T',time()+3600*2);
649     $messaggio="Dehs executed successful\nBegin: $intime\nEnd: $endtime";
650     mail($email, "DEHS report", $messaggio,
651     "From: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .
652     "Reply-To: dehs-noreply@{$_SERVER['SERVER_NAME']}\r\n" .
653     "X-Mailer: PHP/" . phpversion());
654     }
655     function md5_of_file($inFile) {
656     if (file_exists($inFile)) {
657     $fd = fopen($inFile, 'r');
658     $fileContents = fread($fd, filesize($inFile));
659     fclose ($fd);
660     return md5($fileContents);
661     } else {
662     return false;
663     }
664     }
665     ?>

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.5