| 1 |
#!/bin/sh
|
| 2 |
|
| 3 |
# Copyright 2002-2008 Raphaël Hertzog
|
| 4 |
# Copyright 2007-2008 Stefano Zacchiroli <zack@debian.org>
|
| 5 |
|
| 6 |
# This file is distributed under the terms of the General Public License
|
| 7 |
# version 2 or (at your option) any later version.
|
| 8 |
|
| 9 |
#mirror="http://gluck.debian.org/debian"
|
| 10 |
mirror="http://ftp.us.debian.org/debian"
|
| 11 |
security="http://security.debian.org"
|
| 12 |
volatile="http://volatile.debian.org"
|
| 13 |
|
| 14 |
official_distros="oldstable stable testing unstable experimental"
|
| 15 |
pu_distros="stable-proposed-updates testing-proposed-updates"
|
| 16 |
security_distros="oldstable stable"
|
| 17 |
|
| 18 |
umask 002
|
| 19 |
set -e
|
| 20 |
|
| 21 |
if [ -d "../incoming" ]; then
|
| 22 |
root=$PWD/..
|
| 23 |
elif [ -d "incoming" ]; then
|
| 24 |
root=$PWD
|
| 25 |
else
|
| 26 |
root=/org/packages.qa.debian.org/www
|
| 27 |
fi
|
| 28 |
|
| 29 |
cd $root/incoming
|
| 30 |
|
| 31 |
nice_wget() {
|
| 32 |
# $1: url
|
| 33 |
# $2: filename
|
| 34 |
# For timestamping to work, copy to .new (not link, wget will not break
|
| 35 |
# the link)
|
| 36 |
cp -a $2 $2.new || true
|
| 37 |
# Beware that -N conflicts with -O (#88176, #202911)
|
| 38 |
if wget -U pts -q -O $2.new "$1" ; then
|
| 39 |
mv $2.new $2
|
| 40 |
else
|
| 41 |
echo "Downloading $1 failed, $2 is stale now"
|
| 42 |
rm -f $2.new
|
| 43 |
fi
|
| 44 |
}
|
| 45 |
|
| 46 |
http_head() {
|
| 47 |
# $1: url
|
| 48 |
# return 0 if the URL exists (HEAD was OK), non-0 otherwise
|
| 49 |
curl --head --fail --silent "$1" > /dev/null
|
| 50 |
return $?
|
| 51 |
}
|
| 52 |
|
| 53 |
nice_redirect_to() {
|
| 54 |
# $1 : target file
|
| 55 |
# $2-... : shell command
|
| 56 |
dest="$1"
|
| 57 |
shift
|
| 58 |
if [ -f "$dest" ] ; then
|
| 59 |
mv "$dest" "$dest.bak"
|
| 60 |
fi
|
| 61 |
$* > "$dest" || (echo "Failure while executing $* . Continuing ..." ; cp "$dest.bak" "$dest")
|
| 62 |
}
|
| 63 |
|
| 64 |
# Download all Sources.gz
|
| 65 |
for comp in main contrib non-free
|
| 66 |
do
|
| 67 |
for dist in $official_distros ; do
|
| 68 |
nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
|
| 69 |
Sources-${dist}_$comp.gz
|
| 70 |
done
|
| 71 |
for dist in $pu_distros ; do
|
| 72 |
nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
|
| 73 |
Sources-${dist}_$comp.gz
|
| 74 |
done
|
| 75 |
for dist in $security_distros ; do
|
| 76 |
nice_wget $security/dists/$dist/updates/$comp/source/Sources.gz \
|
| 77 |
Sources-security-${dist}_$comp.gz
|
| 78 |
done
|
| 79 |
nice_wget $volatile/debian-volatile/dists/stable/volatile/$comp/source/Sources.gz \
|
| 80 |
Sources-volatile_$comp.gz
|
| 81 |
done
|
| 82 |
|
| 83 |
# Download mentors.d.n Sources
|
| 84 |
for comp in main contrib non-free
|
| 85 |
do
|
| 86 |
nice_wget http://mentors.debian.net/debian/dists/unstable/$comp/source/Sources.gz \
|
| 87 |
Sources-mentors_$comp.gz
|
| 88 |
done
|
| 89 |
|
| 90 |
# Download update_excuses.html
|
| 91 |
nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \
|
| 92 |
update_excuses.html.gz
|
| 93 |
|
| 94 |
# Download PTS subscription count
|
| 95 |
nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt \
|
| 96 |
count.txt
|
| 97 |
|
| 98 |
# Download override disparities
|
| 99 |
nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \
|
| 100 |
override-disparities.unstable
|
| 101 |
nice_wget http://qa.debian.org/data/ftp/override-disparities.experimental \
|
| 102 |
override-disparities.experimental
|
| 103 |
|
| 104 |
# Download lintian.d.o run info
|
| 105 |
#nice_wget http://lintian.debian.org/lintian.log lintian.log
|
| 106 |
nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt
|
| 107 |
|
| 108 |
# Download bugs summary
|
| 109 |
nice_wget http://merkel.debian.org/~hertzog/pts/bugs.txt bugs.txt
|
| 110 |
nice_wget http://qa.debian.org/data/ddpo/results/bugs.txt bugs-src.txt
|
| 111 |
nice_wget http://qa.debian.org/data/bts/sources sources
|
| 112 |
# Temporary measure because merkel is fucked
|
| 113 |
#nice_wget http://www.wolffelaar.nl/~jeroen/pts.bugs.txt bugs.txt
|
| 114 |
#cp /org/bugs.debian.org/etc/indices/sources ./
|
| 115 |
cut -f1,3 sources >sources.map.new
|
| 116 |
touch -r sources sources.map.new
|
| 117 |
mv sources.map.new sources.map
|
| 118 |
nice_redirect_to bugs.help.txt $root/bin/tagged_bugs.py "help"
|
| 119 |
nice_redirect_to bugs.gift.txt \
|
| 120 |
$root/bin/tagged_bugs.py "gift" "debian-qa@lists.debian.org"
|
| 121 |
piuparts_url="http://piuparts.debian.org/results-per-source.txt"
|
| 122 |
# extra care because $piuparts_url doesn't work yet
|
| 123 |
if http_head $piuparts_url ; then
|
| 124 |
nice_wget $piuparts_url piuparts.txt
|
| 125 |
else
|
| 126 |
test -f piuparts.txt || touch piuparts.txt
|
| 127 |
fi
|
| 128 |
|
| 129 |
# Download debcheck lists
|
| 130 |
DCROOT=http://qa.debian.org/data/debcheck/result
|
| 131 |
nice_wget $DCROOT/oldstable/lists/ALL-pkglist debcheck-oldstable
|
| 132 |
nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable
|
| 133 |
nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing
|
| 134 |
nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable
|
| 135 |
|
| 136 |
# Download the list of packages with debconf templates
|
| 137 |
# DISABLED until ddtp.debian.org is back up
|
| 138 |
#nice_wget http://ddtp.debian.org/cgi-bin/debconf_list.cgi debconf-list
|
| 139 |
|
| 140 |
# Get wnpp information
|
| 141 |
nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm
|
| 142 |
|
| 143 |
# get patches from ubuntu
|
| 144 |
nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu
|
| 145 |
# get packages version in Ubuntu (preprocessed by Lucas on merkel)
|
| 146 |
nice_wget http://qa.debian.org/~lucas/ubuntu/sources-versions.txt \
|
| 147 |
versions.ubuntu
|
| 148 |
# get bugs in Ubuntu (preprocessed by Lucas on merkel)
|
| 149 |
nice_wget http://qa.debian.org/~lucas/ubuntu/ubuntu-bugs.txt bugs.ubuntu
|
| 150 |
|
| 151 |
# download LowThresholdNmu list
|
| 152 |
nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \
|
| 153 |
low_threshold_nmu.txt
|
| 154 |
|
| 155 |
# ongoing transitions
|
| 156 |
nice_wget http://ftp-master.debian.org/transitions.yaml \
|
| 157 |
transitions.yaml
|
| 158 |
|
| 159 |
# download the list of packages indexed by svnbuildstat
|
| 160 |
nice_wget http://svnbuildstat.debian.net/packages/flatlist \
|
| 161 |
svnbuildstat_list.txt
|
| 162 |
|
| 163 |
# download DEHS (http://dehs.alioth.debian.org)
|
| 164 |
nice_wget http://dehs.alioth.debian.org/no_updated.txt \
|
| 165 |
dehs_out_of_date.txt
|
| 166 |
nice_wget http://dehs.alioth.debian.org/no_upstream.txt \
|
| 167 |
dehs_error.txt
|
| 168 |
|
| 169 |
# retrieve package descriptions from UDD
|
| 170 |
#nice_redirect_to shortdesc.txt $root/bin/retrieve_shortdesc.sh
|
| 171 |
nice_wget http://master.debian.org/~zack/pts/shortdesc.txt shortdesc.txt
|
| 172 |
|
| 173 |
# What more ?
|
| 174 |
|
| 175 |
# Decompress all files
|
| 176 |
for file in *.gz; do
|
| 177 |
gzip -d -c $file > ${file%%.gz}.new
|
| 178 |
touch -r $file ${file%%.gz}.new
|
| 179 |
mv ${file%%.gz}.new ${file%%.gz}
|
| 180 |
done
|
| 181 |
|
| 182 |
# vim: ts=8
|
Parent Directory
| 
Revision Log