www/bin/update_incoming.sh

#!/bin/sh

# Copyright: © 2002-2008 Raphaël Hertzog
# Copyright: © 2007-2009 Stefano Zacchiroli <zack@debian.org>

# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

#mirror="http://gluck.debian.org/debian"
mirror="http://cdn.debian.net/debian"
security="http://security.debian.org"
piuparts="http://piuparts.debian.org"

official_distros="oldstable stable testing unstable experimental"
pu_distros="stable-updates stable-proposed-updates testing-proposed-updates"
security_distros="oldstable stable testing"

umask 002
set -e

if [ -d "../incoming" ]; then
    root=$PWD/..
elif [ -d "incoming" ]; then
    root=$PWD
else
    root=/org/packages.qa.debian.org/www
fi

cd $root/incoming

nice_wget() {
    # $1: url
    # $2: filename
    # For timestamping to work, copy to .new (not link, wget will not break
    # the link)
    cp -a $2 $2.new || true
    # Beware that -N conflicts with -O (#88176, #202911)
    if wget -U pts -q -O $2.new "$1" ; then
        mv -f $2.new $2
    else
        echo "Downloading $1 failed, $2 is stale now"
        rm -f $2.new
    fi
}

http_head() {
    # $1: url
    # return 0 if the URL exists (HEAD was OK), non-0 otherwise
    curl --head --fail --silent "$1" > /dev/null
    return $?
}

nice_redirect_to() {
  # $1 : target file
  # $2-... : shell command
  dest="$1"
  shift
  if [ -f "$dest" ] ; then
    mv -f "$dest" "$dest.bak"
  fi
  $* > "$dest" || (echo "Failure while executing $* . Continuing ..." ; cp "$dest.bak" "$dest")
}

# Download all Sources.gz
for comp in main contrib non-free
do
    for dist in $official_distros ; do
      nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
        Sources-${dist}_$comp.gz
    done
    for dist in $pu_distros ; do
      nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
        Sources-${dist}_$comp.gz
    done
    for dist in $security_distros ; do
      nice_wget $security/dists/$dist/updates/$comp/source/Sources.gz \
        Sources-security-${dist}_$comp.gz
    done
done

# Download mentors.d.n Sources
for comp in main contrib non-free
do
  nice_wget http://mentors.debian.net/debian/dists/unstable/$comp/source/Sources.gz \
    Sources-mentors_$comp.gz
done

# Download update_excuses.html
nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \
    update_excuses.html.gz

# Download PTS subscription count
nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt \
  count.txt

# Download override disparities
nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \
    override-disparities.unstable
nice_wget http://qa.debian.org/data/ftp/override-disparities.experimental \
    override-disparities.experimental

# Download lintian.d.o run info
#nice_wget http://lintian.debian.org/lintian.log lintian.log
nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt

# Download bugs summary
nice_wget http://udd.debian.org/cgi-bin/bugs-binpkgs-pts.cgi bugs.txt
nice_wget http://udd.debian.org/cgi-bin/ddpo-bugs.cgi bugs-src.txt
rsync rsync://bugs-mirror.debian.org/bts-spool-index/sources ./
cut -f1,3 sources >sources.map.new
touch -r sources sources.map.new
mv -f sources.map.new sources.map
nice_redirect_to bugs.help.txt $root/bin/tagged_bugs.py "help"
nice_redirect_to bugs.gift.txt \
  $root/bin/tagged_bugs.py "gift" "debian-qa@lists.debian.org"
piuparts_distros="sid lenny2squeeze"
for distro in $piuparts_distros ; do
  nice_wget $piuparts/$distro/sources.txt piuparts-$distro.txt
done

# Download debcheck lists
DCROOT=http://qa.debian.org/data/debcheck/result
#nice_wget $DCROOT/oldstable/lists/ALL-pkglist debcheck-oldstable
nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable
nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing
nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable

# Download the translation status of packages
nice_wget http://i18n.debian.net/l10n-pkg-status/pkglist l10n-status.txt

# Get wnpp information
nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm

# get patches from ubuntu
nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu
# get packages version in Ubuntu
nice_wget http://udd.debian.org/cgi-bin/ubuntupackages.cgi versions.ubuntu
# get bugs in Ubuntu
nice_wget http://udd.debian.org/cgi-bin/ubuntubugs.cgi bugs.ubuntu

# download LowThresholdNmu list
nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \
    low_threshold_nmu.txt

# ongoing transitions
nice_wget http://ftp-master.debian.org/transitions.yaml \
    transitions.yaml

# download the list of packages indexed by svnbuildstat
#nice_wget http://svnbuildstat.debian.net/packages/flatlist \
#    svnbuildstat_list.txt

# download DEHS (http://dehs.alioth.debian.org)
nice_wget http://dehs.alioth.debian.org/no_updated.txt \
    dehs_out_of_date.txt
nice_wget http://dehs.alioth.debian.org/no_upstream.txt \
    dehs_error.txt

# download list of security issues
nice_wget http://security-tracker.debian.org/tracker/data/pts/1 \
    security_issues.txt

# download NEW queue info
nice_wget http://ftp-master.debian.org/new.822 new.822

# retrieve package descriptions from UDD
nice_wget http://qa.debian.org/data/pts/shortdesc.txt shortdesc.txt

# download the Debian fonts review
nice_wget http://pkg-fonts.alioth.debian.org/review/debian-font-review.yaml debian-font-review.yaml

# What more ?

# Decompress all files
for file in *.gz; do
    gzip -d -c $file > ${file%%.gz}.new
    touch -r $file ${file%%.gz}.new
    mv -f ${file%%.gz}.new ${file%%.gz}
done

# vim: ts=8
1	hertzog	344	#!/bin/sh
2
3	zack	2223	# Copyright: © 2002-2008 Raphaël Hertzog
4			# Copyright: © 2007-2009 Stefano Zacchiroli <zack@debian.org>
5	zack	1953
6	hertzog	351	# This file is distributed under the terms of the General Public License
7			# version 2 or (at your option) any later version.
8
9	hertzog	692	#mirror="http://gluck.debian.org/debian"
10	geissert	2480	mirror="http://cdn.debian.net/debian"
11	zack	1953	security="http://security.debian.org"
12	zack	2223	piuparts="http://piuparts.debian.org"
13	hertzog	344
14	hertzog	2476	official_distros="oldstable stable testing unstable experimental"
15	myon	2500	pu_distros="stable-updates stable-proposed-updates testing-proposed-updates"
16	hertzog	2476	security_distros="oldstable stable testing"
17	zack	1953
18	hertzog	692	umask 002
19	jeroen	910	set -e
20	hertzog	692
21	hertzog	344	if [ -d "../incoming" ]; then
22			root=$PWD/..
23			elif [ -d "incoming" ]; then
24			root=$PWD
25			else
26	hertzog	345	root=/org/packages.qa.debian.org/www
27	hertzog	344	fi
28
29			cd $root/incoming
30
31	hertzog	372	nice_wget() {
32			# $1: url
33			# $2: filename
34	jeroen	910	# For timestamping to work, copy to .new (not link, wget will not break
35			# the link)
36	jeroen	911	cp -a $2 $2.new \|\| true
37	jeroen	913	# Beware that -N conflicts with -O (#88176, #202911)
38	zack	2047	if wget -U pts -q -O $2.new "$1" ; then
39	hertzog	2383	mv -f $2.new $2
40	hertzog	1900	else
41	filippo	1898	echo "Downloading $1 failed, $2 is stale now"
42			rm -f $2.new
43			fi
44	hertzog	372	}
45
46	zack	2047	http_head() {
47			# $1: url
48			# return 0 if the URL exists (HEAD was OK), non-0 otherwise
49			curl --head --fail --silent "$1" > /dev/null
50			return $?
51			}
52
53	zack	1949	nice_redirect_to() {
54			# $1 : target file
55			# $2-... : shell command
56			dest="$1"
57			shift
58			if [ -f "$dest" ] ; then
59	hertzog	2383	mv -f "$dest" "$dest.bak"
60	zack	1949	fi
61			$* > "$dest" \|\| (echo "Failure while executing $* . Continuing ..." ; cp "$dest.bak" "$dest")
62			}
63
64	hertzog	344	# Download all Sources.gz
65			for comp in main contrib non-free
66			do
67	zack	1953	for dist in $official_distros ; do
68			nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
69			Sources-${dist}_$comp.gz
70	hertzog	344	done
71	zack	1953	for dist in $pu_distros ; do
72			nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
73			Sources-${dist}_$comp.gz
74			done
75			for dist in $security_distros ; do
76			nice_wget $security/dists/$dist/updates/$comp/source/Sources.gz \
77			Sources-security-${dist}_$comp.gz
78			done
79	hertzog	344	done
80
81	zack	1933	# Download mentors.d.n Sources
82			for comp in main contrib non-free
83			do
84			nice_wget http://mentors.debian.net/debian/dists/unstable/$comp/source/Sources.gz \
85			Sources-mentors_$comp.gz
86			done
87
88	hertzog	344	# Download update_excuses.html
89	jeroen	910	nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \
90			update_excuses.html.gz
91	hertzog	344
92			# Download PTS subscription count
93	zack	1953	nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt \
94			count.txt
95	hertzog	344
96	jeroen	947	# Download override disparities
97			nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \
98			override-disparities.unstable
99			nice_wget http://qa.debian.org/data/ftp/override-disparities.experimental \
100			override-disparities.experimental
101	jeroen	944
102	zack	1927	# Download lintian.d.o run info
103			#nice_wget http://lintian.debian.org/lintian.log lintian.log
104			nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt
105
106	hertzog	344	# Download bugs summary
107	hertzog	2274	nice_wget http://udd.debian.org/cgi-bin/bugs-binpkgs-pts.cgi bugs.txt
108	hertzog	2272	nice_wget http://udd.debian.org/cgi-bin/ddpo-bugs.cgi bugs-src.txt
109	hertzog	2423	rsync rsync://bugs-mirror.debian.org/bts-spool-index/sources ./
110	jeroen	914	cut -f1,3 sources >sources.map.new
111			touch -r sources sources.map.new
112	hertzog	2383	mv -f sources.map.new sources.map
113	zack	1949	nice_redirect_to bugs.help.txt $root/bin/tagged_bugs.py "help"
114			nice_redirect_to bugs.gift.txt \
115			$root/bin/tagged_bugs.py "gift" "debian-qa@lists.debian.org"
116	zack	2353	piuparts_distros="sid lenny2squeeze"
117	zack	2223	for distro in $piuparts_distros ; do
118			nice_wget $piuparts/$distro/sources.txt piuparts-$distro.txt
119			done
120	hertzog	344
121	hertzog	372	# Download debcheck lists
122	jeroen	914	DCROOT=http://qa.debian.org/data/debcheck/result
123	hertzog	2383	#nice_wget $DCROOT/oldstable/lists/ALL-pkglist debcheck-oldstable
124	jeroen	914	nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable
125			nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing
126			nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable
127	hertzog	344
128	zack	2230	# Download the translation status of packages
129			nice_wget http://i18n.debian.net/l10n-pkg-status/pkglist l10n-status.txt
130	hertzog	479
131	jeroen	912	# Get wnpp information
132			nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm
133	hertzog	560
134	hertzog	1049	# get patches from ubuntu
135	hertzog	1355	nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu
136	hertzog	2291	# get packages version in Ubuntu
137			nice_wget http://udd.debian.org/cgi-bin/ubuntupackages.cgi versions.ubuntu
138			# get bugs in Ubuntu
139			nice_wget http://udd.debian.org/cgi-bin/ubuntubugs.cgi bugs.ubuntu
140	hertzog	1049
141	zack	1750	# download LowThresholdNmu list
142			nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \
143			low_threshold_nmu.txt
144
145	zack	1866	# ongoing transitions
146			nice_wget http://ftp-master.debian.org/transitions.yaml \
147			transitions.yaml
148
149	zack	1800	# download the list of packages indexed by svnbuildstat
150	hertzog	2286	#nice_wget http://svnbuildstat.debian.net/packages/flatlist \
151			# svnbuildstat_list.txt
152	zack	1800
153	zack	1823	# download DEHS (http://dehs.alioth.debian.org)
154			nice_wget http://dehs.alioth.debian.org/no_updated.txt \
155			dehs_out_of_date.txt
156			nice_wget http://dehs.alioth.debian.org/no_upstream.txt \
157			dehs_error.txt
158
159	geissert	2277	# download list of security issues
160	geissert	2292	nice_wget http://security-tracker.debian.org/tracker/data/pts/1 \
161	geissert	2277	security_issues.txt
162
163	zack	2224	# download NEW queue info
164			nice_wget http://ftp-master.debian.org/new.822 new.822
165
166	zack	1950	# retrieve package descriptions from UDD
167	zack	2323	nice_wget http://qa.debian.org/data/pts/shortdesc.txt shortdesc.txt
168	zack	1950
169	geissert	2422	# download the Debian fonts review
170			nice_wget http://pkg-fonts.alioth.debian.org/review/debian-font-review.yaml debian-font-review.yaml
171
172	hertzog	344	# What more ?
173
174			# Decompress all files
175			for file in *.gz; do
176	jeroen	910	gzip -d -c $file > ${file%%.gz}.new
177			touch -r $file ${file%%.gz}.new
178	hertzog	2383	mv -f ${file%%.gz}.new ${file%%.gz}
179	hertzog	344	done
180
181	jeroen	910	# vim: ts=8
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision