/[qa]/trunk/pts/www/bin/update_incoming.sh
ViewVC logotype

Diff of /trunk/pts/www/bin/update_incoming.sh

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1927 by zack, Fri Aug 8 07:17:49 2008 UTC revision 2291 by hertzog, Mon Oct 12 09:58:25 2009 UTC
# Line 1  Line 1 
1  #!/bin/sh  #!/bin/sh
2    
3  # Copyright 2002 Raphaël Hertzog  # Copyright: © 2002-2008 Raphaël Hertzog
4    # Copyright: © 2007-2009 Stefano Zacchiroli <zack@debian.org>
5    
6  # This file is distributed under the terms of the General Public License  # This file is distributed under the terms of the General Public License
7  # version 2 or (at your option) any later version.  # version 2 or (at your option) any later version.
8    
9  #mirror="http://gluck.debian.org/debian"  #mirror="http://gluck.debian.org/debian"
10  mirror="http://ftp.us.debian.org/debian"  mirror="http://ftp.us.debian.org/debian"
11  #mirror="http://ftp.debian.org/debian"  security="http://security.debian.org"
12    volatile="http://volatile.debian.org"
13    piuparts="http://piuparts.debian.org"
14    
15    official_distros="oldstable stable testing unstable experimental"
16    pu_distros="stable-proposed-updates testing-proposed-updates"
17    security_distros="oldstable stable"
18    
19  umask 002  umask 002
20  set -e  set -e
# Line 28  nice_wget() { Line 36  nice_wget() {
36      # the link)      # the link)
37      cp -a $2 $2.new || true      cp -a $2 $2.new || true
38      # Beware that -N conflicts with -O (#88176, #202911)      # Beware that -N conflicts with -O (#88176, #202911)
39      if wget -U pts -q -O $2.new $1 ; then      if wget -U pts -q -O $2.new "$1" ; then
40          mv $2.new $2          mv $2.new $2
41      else      else
42          echo "Downloading $1 failed, $2 is stale now"          echo "Downloading $1 failed, $2 is stale now"
# Line 36  nice_wget() { Line 44  nice_wget() {
44      fi      fi
45  }  }
46    
47    http_head() {
48        # $1: url
49        # return 0 if the URL exists (HEAD was OK), non-0 otherwise
50        curl --head --fail --silent "$1" > /dev/null
51        return $?
52    }
53    
54    nice_redirect_to() {
55      # $1 : target file
56      # $2-... : shell command
57      dest="$1"
58      shift
59      if [ -f "$dest" ] ; then
60        mv "$dest" "$dest.bak"
61      fi
62      $* > "$dest" || (echo "Failure while executing $* . Continuing ..." ; cp "$dest.bak" "$dest")
63    }
64    
65  # Download all Sources.gz  # Download all Sources.gz
66  for comp in main contrib non-free  for comp in main contrib non-free
67  do  do
68      for dist in oldstable stable testing unstable experimental      for dist in $official_distros ; do
69      do        nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
70          nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \          Sources-${dist}_$comp.gz
71              Sources_${dist}_$comp.gz      done
72        for dist in $pu_distros ; do
73          nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
74            Sources-${dist}_$comp.gz
75      done      done
76      nice_wget $mirror/dists/stable-proposed-updates/$comp/source/Sources.gz \      for dist in $security_distros ; do
77          Sources-spu_$comp.gz        nice_wget $security/dists/$dist/updates/$comp/source/Sources.gz \
78      nice_wget $mirror/dists/testing-proposed-updates/$comp/source/Sources.gz \          Sources-security-${dist}_$comp.gz
79          Sources-tpu_$comp.gz      done
80      nice_wget http://security.debian.org/dists/oldstable/updates/$comp/source/Sources.gz \      nice_wget $volatile/debian-volatile/dists/stable/volatile/$comp/source/Sources.gz \
         Sources-security-oldstable_$comp.gz  
     nice_wget http://security.debian.org/dists/stable/updates/$comp/source/Sources.gz \  
         Sources-security-stable_$comp.gz  
     #nice_wget http://security.debian.org/dists/testing/updates/$comp/source/Sources.gz \  
         #Sources-security-testing_$comp.gz  
     nice_wget http://volatile.debian.org/debian-volatile/dists/stable/volatile/$comp/source/Sources.gz \  
81          Sources-volatile_$comp.gz          Sources-volatile_$comp.gz
82  done  done
83    
84    # Download mentors.d.n Sources
85    for comp in main contrib non-free
86    do
87      nice_wget http://mentors.debian.net/debian/dists/unstable/$comp/source/Sources.gz \
88        Sources-mentors_$comp.gz
89    done
90    
91  # Download update_excuses.html  # Download update_excuses.html
92  nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \  nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \
93      update_excuses.html.gz      update_excuses.html.gz
94    
95  # Download PTS subscription count  # Download PTS subscription count
96  nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt count.txt  nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt \
97      count.txt
98    
99  # Download override disparities  # Download override disparities
100  nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \  nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \
# Line 76  nice_wget http://qa.debian.org/data/ftp/ Line 107  nice_wget http://qa.debian.org/data/ftp/
107  nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt  nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt
108    
109  # Download bugs summary  # Download bugs summary
110  nice_wget http://merkel.debian.org/~hertzog/pts/bugs.txt bugs.txt  nice_wget http://udd.debian.org/cgi-bin/bugs-binpkgs-pts.cgi bugs.txt
111  nice_wget http://qa.debian.org/data/ddpo/results/bugs.txt bugs-src.txt  nice_wget http://udd.debian.org/cgi-bin/ddpo-bugs.cgi bugs-src.txt
112  nice_wget http://qa.debian.org/data/bts/sources sources  nice_wget http://qa.debian.org/data/bts/sources sources
 # Temporary measure because merkel is fucked  
 #nice_wget http://www.wolffelaar.nl/~jeroen/pts.bugs.txt bugs.txt  
 #cp /org/bugs.debian.org/etc/indices/sources ./  
113  cut -f1,3 sources >sources.map.new  cut -f1,3 sources >sources.map.new
114  touch -r sources sources.map.new  touch -r sources sources.map.new
115  mv sources.map.new sources.map  mv sources.map.new sources.map
116    nice_redirect_to bugs.help.txt $root/bin/tagged_bugs.py "help"
117    nice_redirect_to bugs.gift.txt \
118      $root/bin/tagged_bugs.py "gift" "debian-qa@lists.debian.org"
119    piuparts_distros="sid"
120    for distro in $piuparts_distros ; do
121      nice_wget $piuparts/$distro/sources.txt piuparts-$distro.txt
122    done
123    
124  # Download debcheck lists  # Download debcheck lists
125  DCROOT=http://qa.debian.org/data/debcheck/result  DCROOT=http://qa.debian.org/data/debcheck/result
126    nice_wget $DCROOT/oldstable/lists/ALL-pkglist debcheck-oldstable
127  nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable  nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable
128  nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing  nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing
129  nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable  nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable
130    
131  # Download the list of packages with debconf templates  # Download the translation status of packages
132  # DISABLED until ddtp.debian.org is back up  nice_wget http://i18n.debian.net/l10n-pkg-status/pkglist l10n-status.txt
 #nice_wget http://ddtp.debian.org/cgi-bin/debconf_list.cgi debconf-list  
133    
134  # Get wnpp information  # Get wnpp information
135  nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm  nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm
136    
137  # get patches from ubuntu  # get patches from ubuntu
138  nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu  nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu
139  # get packages version in Ubuntu (preprocessed by Lucas on merkel)  # get packages version in Ubuntu
140  nice_wget http://qa.debian.org/~lucas/ubuntu/sources-versions.txt versions.ubuntu  nice_wget http://udd.debian.org/cgi-bin/ubuntupackages.cgi versions.ubuntu
141  # get bugs in Ubuntu (preprocessed by Lucas on merkel)  # get bugs in Ubuntu
142  nice_wget http://qa.debian.org/~lucas/ubuntu/ubuntu-bugs.txt bugs.ubuntu  nice_wget http://udd.debian.org/cgi-bin/ubuntubugs.cgi bugs.ubuntu
143    
144  # download LowThresholdNmu list  # download LowThresholdNmu list
145  nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \  nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \
# Line 115  nice_wget http://ftp-master.debian.org/t Line 150  nice_wget http://ftp-master.debian.org/t
150      transitions.yaml      transitions.yaml
151    
152  # download the list of packages indexed by svnbuildstat  # download the list of packages indexed by svnbuildstat
153  nice_wget http://svnbuildstat.debian.net/packages/flatlist \  #nice_wget http://svnbuildstat.debian.net/packages/flatlist \
154      svnbuildstat_list.txt  #    svnbuildstat_list.txt
155    
156  # download DEHS (http://dehs.alioth.debian.org)  # download DEHS (http://dehs.alioth.debian.org)
157  nice_wget http://dehs.alioth.debian.org/no_updated.txt \  nice_wget http://dehs.alioth.debian.org/no_updated.txt \
# Line 124  nice_wget http://dehs.alioth.debian.org/ Line 159  nice_wget http://dehs.alioth.debian.org/
159  nice_wget http://dehs.alioth.debian.org/no_upstream.txt \  nice_wget http://dehs.alioth.debian.org/no_upstream.txt \
160      dehs_error.txt      dehs_error.txt
161    
162    # download list of security issues
163    nice_wget http://alioth.debian.org/~geissert/security-packages.txt \
164        security_issues.txt
165    
166    # download NEW queue info
167    nice_wget http://ftp-master.debian.org/new.822 new.822
168    
169    # retrieve package descriptions from UDD
170    #nice_redirect_to shortdesc.txt $root/bin/retrieve_shortdesc.sh
171    nice_wget http://master.debian.org/~zack/pts/shortdesc.txt shortdesc.txt
172    
173  # What more ?  # What more ?
174    
175  # Decompress all files  # Decompress all files
Legend:
Removed from v.1927  
changed lines
  Added in v.2291
  ViewVC Help
Powered by ViewVC 1.1.5