/[qa]/trunk/pts/www/bin/update_incoming.sh
ViewVC logotype

Diff of /trunk/pts/www/bin/update_incoming.sh

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 479 by hertzog, Sat Jan 11 22:56:05 2003 UTC revision 2648 by bartm, Wed Nov 9 20:48:12 2011 UTC
# Line 1  Line 1 
1  #!/bin/sh  #!/bin/sh
2    
3  # Copyright 2002 Raphaël Hertzog  # Copyright: © 2002-2011 Raphaël Hertzog
4    # Copyright: © 2007-2009 Stefano Zacchiroli <zack@debian.org>
5    
6  # This file is distributed under the terms of the General Public License  # This file is distributed under the terms of the General Public License
7  # version 2 or (at your option) any later version.  # version 2 or (at your option) any later version.
8    
9  mirror="http://gluck.debian.org/debian"  mirror="http://cdn.debian.net/debian"
10  nonus="http://non-us.debian.org/debian-non-US"  security="http://security.debian.org"
11  wgetopt="-t 5 -q -N"  backports="http://backports.debian.org/debian-backports/"
12    piuparts="http://piuparts.debian.org"
13    
14    official_distros="oldstable stable testing unstable experimental"
15    pu_distros="stable-updates stable-proposed-updates testing-proposed-updates"
16    security_distros="oldstable stable testing"
17    backports_distros="oldstable-backports stable-backports"
18    
19    umask 002
20    set -e
21    
22  if [ -d "../incoming" ]; then  if [ -d "../incoming" ]; then
23      root=$PWD/..      root=$PWD/..
# Line 21  cd $root/incoming Line 32  cd $root/incoming
32  nice_wget() {  nice_wget() {
33      # $1: url      # $1: url
34      # $2: filename      # $2: filename
35      file=`basename $1`      # For timestamping to work, copy to .new (not link, wget will not break
36      if [ -e "$2" ]; then      # the link)
37          cp -a $2 $file      test -e $2 && cp -a $2 $2.new || true
38        # Beware that -N conflicts with -O (#88176, #202911)
39        if wget -U pts -q -O $2.new "$1" ; then
40            mv -f $2.new $2
41        else
42            echo "Downloading $1 failed, $2 is stale now"
43            rm -f $2.new
44      fi      fi
45      wget $wgetopt $1  }
46      mv $file $2  
47    http_head() {
48        # $1: url
49        # return 0 if the URL exists (HEAD was OK), non-0 otherwise
50        curl --head --fail --silent "$1" > /dev/null
51        return $?
52    }
53    
54    nice_redirect_to() {
55      # $1 : target file
56      # $2-... : shell command
57      dest="$1"
58      shift
59      if [ -f "$dest" ] ; then
60        mv -f "$dest" "$dest.bak"
61      fi
62      $* > "$dest" || (echo "Failure while executing $* . Continuing ..." ; cp "$dest.bak" "$dest")
63  }  }
64    
65  # Download all Sources.gz  # Download all Sources.gz
66  for comp in main contrib non-free  for comp in main contrib non-free
67  do  do
68      for dist in stable testing unstable      for dist in $official_distros $pu_distros; do
69      do        nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
70          nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \          Sources-${dist}_$comp.gz
             Sources_${dist}_$comp.gz  
         nice_wget $nonus/dists/$dist/non-US/$comp/source/Sources.gz \  
             Sources-nonus_${dist}_$comp.gz  
71      done      done
72      nice_wget $mirror/project/experimental/$comp/source/Sources.gz \      for dist in $security_distros ; do
73          Sources-experimental_$comp.gz        nice_wget $security/dists/$dist/updates/$comp/source/Sources.gz \
74      nice_wget $mirror/dists/stable-proposed-updates/$comp/source/Sources.gz \          Sources-security-${dist}_$comp.gz
75          Sources-spu_$comp.gz      done
76      nice_wget $mirror/dists/testing-proposed-updates/$comp/source/Sources.gz \      for dist in $backports_distros ; do
77          Sources-tpu_$comp.gz        nice_wget $backports/dists/$dist/$comp/source/Sources.gz \
78      nice_wget $nonus/dists/stable-proposed-updates/non-US/$comp/source/Sources.gz \          Sources-${dist}_$comp.gz
79          Sources-nonus-spu_$comp.gz      done
80      nice_wget $nonus/dists/testing-proposed-updates/non-US/$comp/source/Sources.gz \      nice_wget http://mentors.debian.net/debian/dists/unstable/$comp/source/Sources.gz \
81          Sources-nonus-tpu_$comp.gz          Sources-mentors_$comp.gz
     nice_wget http://security.debian.org/dists/stable/updates/$comp/source/Sources.gz \  
         Sources-security-stable_$comp.gz  
     nice_wget http://security.debian.org/dists/testing/updates/$comp/source/Sources.gz \  
         Sources-security-testing_$comp.gz  
82  done  done
83    
84  # Download update_excuses.html  # Download update_excuses.html
85  wget $wgetopt http://ftp-master.debian.org/testing/update_excuses.html.gz  nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \
86        update_excuses.html.gz
87    
88  # Download PTS subscription count  # Download PTS subscription count
89  wget $wgetopt http://master.debian.org/~hertzog/pts/count.txt  nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt \
90      count.txt
91    
92    # Download override disparities
93    nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \
94        override-disparities.unstable
95    nice_wget http://qa.debian.org/data/ftp/override-disparities.experimental \
96        override-disparities.experimental
97    
98    # Download lintian.d.o run info
99    #nice_wget http://lintian.debian.org/lintian.log lintian.log
100    nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt
101    
102  # Download bugs summary  # Download bugs summary
103  wget $wgetopt http://master.debian.org/~hertzog/pts/bugs.txt  nice_wget http://udd.debian.org/cgi-bin/bugs-binpkgs-pts.cgi bugs.txt
104  wget $wgetopt http://master.debian.org/~hertzog/pts/sources  nice_wget http://udd.debian.org/cgi-bin/ddpo-bugs.cgi bugs-src.txt
105  cut -f1,3 sources >sources.new  nice_redirect_to bugs.help.txt $root/bin/tagged_bugs.py "help"
106  touch -r sources sources.new  nice_redirect_to bugs.gift.txt \
107  mv sources.new sources    $root/bin/tagged_bugs.py "gift" "debian-qa@lists.debian.org"
108    piuparts_distros="sid lenny2squeeze"
109    for distro in $piuparts_distros ; do
110      nice_wget $piuparts/$distro/sources.txt piuparts-$distro.txt
111    done
112    
113  # Download debcheck lists  # Download debcheck lists
114  nice_wget http://qa.debian.org/~weasel/debcheck-result/stable/lists/ALL-pkglist \  DCROOT=http://qa.debian.org/data/debcheck/result
115      debcheck-stable  #nice_wget $DCROOT/oldstable/lists/ALL-pkglist debcheck-oldstable
116  nice_wget http://qa.debian.org/~weasel/debcheck-result/testing/lists/ALL-pkglist \  nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable
117      debcheck-testing  nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing
118  nice_wget http://qa.debian.org/~weasel/debcheck-result/unstable/lists/ALL-pkglist \  nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable
119      debcheck-unstable  
120    # Download the translation status of packages
121    nice_wget http://i18n.debian.net/l10n-pkg-status/pkglist l10n-status.txt
122    
123    # Get wnpp information
124    nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm
125    
126    # get patches from ubuntu
127    nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu
128    # get packages version in Ubuntu
129    nice_wget http://udd.debian.org/cgi-bin/ubuntupackages.cgi versions.ubuntu
130    # get bugs in Ubuntu
131    nice_wget http://udd.debian.org/cgi-bin/ubuntubugs.cgi bugs.ubuntu
132    
133    # download LowThresholdNmu list
134    nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \
135        low_threshold_nmu.txt
136    
137    # ongoing transitions
138    nice_wget http://ftp-master.debian.org/transitions.yaml \
139        transitions.yaml
140    
141    # download the list of packages indexed by svnbuildstat
142    #nice_wget http://svnbuildstat.debian.net/packages/flatlist \
143    #    svnbuildstat_list.txt
144    
145    # download watch file information
146    nice_wget http://qa.debian.org/cgi-bin/udd-dehs dehs.yaml
147    
148    # download list of security issues
149    nice_wget http://security-tracker.debian.org/tracker/data/pts/1 \
150        security_issues.txt
151    
152  # Download the list of packages with debconf templates  # download NEW queue info
153  nice_wget http://ddtp.debian.org/cgi-bin/debconf_list.cgi debconf-list  nice_wget http://ftp-master.debian.org/new.822 new.822
154    
155    # retrieve package descriptions from UDD
156    nice_wget http://qa.debian.org/data/pts/shortdesc.txt shortdesc.txt
157    
158    # download the Debian fonts review
159    nice_wget http://pkg-fonts.alioth.debian.org/review/debian-font-review.yaml debian-font-review.yaml
160    
161  # What more ?  # What more ?
162    
163  # Decompress all files  # Decompress all files
164  for file in *.gz; do  for file in *.gz; do
165      gzip -d -c $file > ${file%%.gz}      gzip -d -c $file > ${file%%.gz}.new
166        touch -r $file ${file%%.gz}.new
167        mv -f ${file%%.gz}.new ${file%%.gz}
168  done  done
169    
170    # filter out some packages from Sources-mentors_*
171    ../bin/filter_mentors.pl
172    mv Sources-mentors_main_new Sources-mentors_main
173    mv Sources-mentors_contrib_new Sources-mentors_contrib
174    mv Sources-mentors_non-free_new Sources-mentors_non-free
175    
176    # vim: ts=8
Legend:
Removed from v.479  
changed lines
  Added in v.2648
  ViewVC Help
Powered by ViewVC 1.1.5