www/bin/update_incoming.sh

#!/bin/sh

# Copyright 2002 Raphaël Hertzog
# This file is distributed under the terms of the General Public License
# version 2 or (at your option) any later version.

#mirror="http://gluck.debian.org/debian"
mirror="http://ftp.us.debian.org/debian"
#mirror="http://ftp.debian.org/debian"

umask 002
set -e

if [ -d "../incoming" ]; then
    root=$PWD/..
elif [ -d "incoming" ]; then
    root=$PWD
else
    root=/org/packages.qa.debian.org/www
fi

cd $root/incoming

nice_wget() {
    # $1: url
    # $2: filename
    # For timestamping to work, copy to .new (not link, wget will not break
    # the link)
    cp -a $2 $2.new || true
    # Beware that -N conflicts with -O (#88176, #202911)
    if wget -U pts -q -O $2.new $1 ; then
        mv $2.new $2
    else
        echo "Downloading $1 failed, $2 is stale now"
        rm -f $2.new
    fi
}

nice_redirect_to() {
  # $1 : target file
  # $2-... : shell command
  dest="$1"
  shift
  if [ -f "$dest" ] ; then
    mv "$dest" "$dest.bak"
  fi
  $* > "$dest" || (echo "Failure while executing $* . Continuing ..." ; cp "$dest.bak" "$dest")
}

# Download all Sources.gz
for comp in main contrib non-free
do
    for dist in oldstable stable testing unstable experimental
    do
        nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
            Sources_${dist}_$comp.gz
    done
    nice_wget $mirror/dists/stable-proposed-updates/$comp/source/Sources.gz \
        Sources-spu_$comp.gz
    nice_wget $mirror/dists/testing-proposed-updates/$comp/source/Sources.gz \
        Sources-tpu_$comp.gz
    nice_wget http://security.debian.org/dists/oldstable/updates/$comp/source/Sources.gz \
        Sources-security-oldstable_$comp.gz
    nice_wget http://security.debian.org/dists/stable/updates/$comp/source/Sources.gz \
        Sources-security-stable_$comp.gz
    #nice_wget http://security.debian.org/dists/testing/updates/$comp/source/Sources.gz \
        #Sources-security-testing_$comp.gz
    nice_wget http://volatile.debian.org/debian-volatile/dists/stable/volatile/$comp/source/Sources.gz \
        Sources-volatile_$comp.gz
done

# Download mentors.d.n Sources
for comp in main contrib non-free
do
  nice_wget http://mentors.debian.net/debian/dists/unstable/$comp/source/Sources.gz \
    Sources-mentors_$comp.gz
done


# Download update_excuses.html
nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \
    update_excuses.html.gz

# Download PTS subscription count
nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt count.txt

# Download override disparities
nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \
    override-disparities.unstable
nice_wget http://qa.debian.org/data/ftp/override-disparities.experimental \
    override-disparities.experimental

# Download lintian.d.o run info
#nice_wget http://lintian.debian.org/lintian.log lintian.log
nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt

# Download bugs summary
nice_wget http://merkel.debian.org/~hertzog/pts/bugs.txt bugs.txt
nice_wget http://qa.debian.org/data/ddpo/results/bugs.txt bugs-src.txt
nice_wget http://qa.debian.org/data/bts/sources sources
# Temporary measure because merkel is fucked
#nice_wget http://www.wolffelaar.nl/~jeroen/pts.bugs.txt bugs.txt
#cp /org/bugs.debian.org/etc/indices/sources ./
cut -f1,3 sources >sources.map.new
touch -r sources sources.map.new
mv sources.map.new sources.map
nice_redirect_to bugs.help.txt $root/bin/tagged_bugs.py "help"
nice_redirect_to bugs.gift.txt \
  $root/bin/tagged_bugs.py "gift" "debian-qa@lists.debian.org"

# Download debcheck lists
DCROOT=http://qa.debian.org/data/debcheck/result
nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable
nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing
nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable

# Download the list of packages with debconf templates
# DISABLED until ddtp.debian.org is back up
#nice_wget http://ddtp.debian.org/cgi-bin/debconf_list.cgi debconf-list

# Get wnpp information
nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm

# get patches from ubuntu
nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu
# get packages version in Ubuntu (preprocessed by Lucas on merkel)
nice_wget http://qa.debian.org/~lucas/ubuntu/sources-versions.txt versions.ubuntu
# get bugs in Ubuntu (preprocessed by Lucas on merkel)
nice_wget http://qa.debian.org/~lucas/ubuntu/ubuntu-bugs.txt bugs.ubuntu

# download LowThresholdNmu list
nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \
    low_threshold_nmu.txt

# ongoing transitions
nice_wget http://ftp-master.debian.org/transitions.yaml \
    transitions.yaml

# download the list of packages indexed by svnbuildstat
nice_wget http://svnbuildstat.debian.net/packages/flatlist \
    svnbuildstat_list.txt

# download DEHS (http://dehs.alioth.debian.org)
nice_wget http://dehs.alioth.debian.org/no_updated.txt \
    dehs_out_of_date.txt
nice_wget http://dehs.alioth.debian.org/no_upstream.txt \
    dehs_error.txt

# What more ?

# Decompress all files
for file in *.gz; do
    gzip -d -c $file > ${file%%.gz}.new
    touch -r $file ${file%%.gz}.new
    mv ${file%%.gz}.new ${file%%.gz}
done

# vim: ts=8
1	hertzog	344	#!/bin/sh
2
3	hertzog	1899	# Copyright 2002 Raphaël Hertzog
4	hertzog	351	# This file is distributed under the terms of the General Public License
5			# version 2 or (at your option) any later version.
6
7	hertzog	692	#mirror="http://gluck.debian.org/debian"
8	hertzog	1602	mirror="http://ftp.us.debian.org/debian"
9			#mirror="http://ftp.debian.org/debian"
10	hertzog	344
11	hertzog	692	umask 002
12	jeroen	910	set -e
13	hertzog	692
14	hertzog	344	if [ -d "../incoming" ]; then
15			root=$PWD/..
16			elif [ -d "incoming" ]; then
17			root=$PWD
18			else
19	hertzog	345	root=/org/packages.qa.debian.org/www
20	hertzog	344	fi
21
22			cd $root/incoming
23
24	hertzog	372	nice_wget() {
25			# $1: url
26			# $2: filename
27	jeroen	910	# For timestamping to work, copy to .new (not link, wget will not break
28			# the link)
29	jeroen	911	cp -a $2 $2.new \|\| true
30	jeroen	913	# Beware that -N conflicts with -O (#88176, #202911)
31	hertzog	1900	if wget -U pts -q -O $2.new $1 ; then
32			mv $2.new $2
33			else
34	filippo	1898	echo "Downloading $1 failed, $2 is stale now"
35			rm -f $2.new
36			fi
37	hertzog	372	}
38
39	zack	1949	nice_redirect_to() {
40			# $1 : target file
41			# $2-... : shell command
42			dest="$1"
43			shift
44			if [ -f "$dest" ] ; then
45			mv "$dest" "$dest.bak"
46			fi
47			$* > "$dest" \|\| (echo "Failure while executing $* . Continuing ..." ; cp "$dest.bak" "$dest")
48			}
49
50	hertzog	344	# Download all Sources.gz
51			for comp in main contrib non-free
52			do
53	jeroen	1223	for dist in oldstable stable testing unstable experimental
54	hertzog	344	do
55	hertzog	372	nice_wget $mirror/dists/$dist/$comp/source/Sources.gz \
56			Sources_${dist}_$comp.gz
57	hertzog	344	done
58	hertzog	372	nice_wget $mirror/dists/stable-proposed-updates/$comp/source/Sources.gz \
59			Sources-spu_$comp.gz
60			nice_wget $mirror/dists/testing-proposed-updates/$comp/source/Sources.gz \
61			Sources-tpu_$comp.gz
62	jeroen	1020	nice_wget http://security.debian.org/dists/oldstable/updates/$comp/source/Sources.gz \
63			Sources-security-oldstable_$comp.gz
64	hertzog	372	nice_wget http://security.debian.org/dists/stable/updates/$comp/source/Sources.gz \
65			Sources-security-stable_$comp.gz
66	jeroen	1023	#nice_wget http://security.debian.org/dists/testing/updates/$comp/source/Sources.gz \
67			#Sources-security-testing_$comp.gz
68	hertzog	1602	nice_wget http://volatile.debian.org/debian-volatile/dists/stable/volatile/$comp/source/Sources.gz \
69	hertzog	1092	Sources-volatile_$comp.gz
70	hertzog	344	done
71
72	zack	1933	# Download mentors.d.n Sources
73			for comp in main contrib non-free
74			do
75			nice_wget http://mentors.debian.net/debian/dists/unstable/$comp/source/Sources.gz \
76			Sources-mentors_$comp.gz
77			done
78
79
80	hertzog	344	# Download update_excuses.html
81	jeroen	910	nice_wget http://ftp-master.debian.org/testing/update_excuses.html.gz \
82			update_excuses.html.gz
83	hertzog	344
84			# Download PTS subscription count
85	jeroen	936	nice_wget http://packages.qa.debian.org/data/pts-subscription-count.txt count.txt
86	hertzog	344
87	jeroen	947	# Download override disparities
88			nice_wget http://qa.debian.org/data/ftp/override-disparities.unstable \
89			override-disparities.unstable
90			nice_wget http://qa.debian.org/data/ftp/override-disparities.experimental \
91			override-disparities.experimental
92	jeroen	944
93	zack	1927	# Download lintian.d.o run info
94			#nice_wget http://lintian.debian.org/lintian.log lintian.log
95			nice_wget http://lintian.debian.org/qa-list.txt lintian.qa-list.txt
96
97	hertzog	344	# Download bugs summary
98	jeroen	910	nice_wget http://merkel.debian.org/~hertzog/pts/bugs.txt bugs.txt
99	hertzog	1911	nice_wget http://qa.debian.org/data/ddpo/results/bugs.txt bugs-src.txt
100	jeroen	916	nice_wget http://qa.debian.org/data/bts/sources sources
101	hertzog	812	# Temporary measure because merkel is fucked
102	jeroen	913	#nice_wget http://www.wolffelaar.nl/~jeroen/pts.bugs.txt bugs.txt
103	jeroen	905	#cp /org/bugs.debian.org/etc/indices/sources ./
104	jeroen	914	cut -f1,3 sources >sources.map.new
105			touch -r sources sources.map.new
106			mv sources.map.new sources.map
107	zack	1949	nice_redirect_to bugs.help.txt $root/bin/tagged_bugs.py "help"
108			nice_redirect_to bugs.gift.txt \
109			$root/bin/tagged_bugs.py "gift" "debian-qa@lists.debian.org"
110	hertzog	344
111	hertzog	372	# Download debcheck lists
112	jeroen	914	DCROOT=http://qa.debian.org/data/debcheck/result
113			nice_wget $DCROOT/stable/lists/ALL-pkglist debcheck-stable
114			nice_wget $DCROOT/testing/lists/ALL-pkglist debcheck-testing
115			nice_wget $DCROOT/unstable/lists/ALL-pkglist debcheck-unstable
116	hertzog	344
117	hertzog	479	# Download the list of packages with debconf templates
118	hertzog	692	# DISABLED until ddtp.debian.org is back up
119			#nice_wget http://ddtp.debian.org/cgi-bin/debconf_list.cgi debconf-list
120	hertzog	479
121	jeroen	912	# Get wnpp information
122			nice_wget http://qa.debian.org/data/bts/wnpp_rm wnpp_rm
123	hertzog	560
124	hertzog	1049	# get patches from ubuntu
125	hertzog	1355	nice_wget http://patches.ubuntu.com/PATCHES patches.ubuntu
126	hertzog	1887	# get packages version in Ubuntu (preprocessed by Lucas on merkel)
127			nice_wget http://qa.debian.org/~lucas/ubuntu/sources-versions.txt versions.ubuntu
128			# get bugs in Ubuntu (preprocessed by Lucas on merkel)
129			nice_wget http://qa.debian.org/~lucas/ubuntu/ubuntu-bugs.txt bugs.ubuntu
130	hertzog	1049
131	zack	1750	# download LowThresholdNmu list
132			nice_wget 'http://wiki.debian.org/LowThresholdNmu?action=raw' \
133			low_threshold_nmu.txt
134
135	zack	1866	# ongoing transitions
136			nice_wget http://ftp-master.debian.org/transitions.yaml \
137			transitions.yaml
138
139	zack	1800	# download the list of packages indexed by svnbuildstat
140			nice_wget http://svnbuildstat.debian.net/packages/flatlist \
141			svnbuildstat_list.txt
142
143	zack	1823	# download DEHS (http://dehs.alioth.debian.org)
144			nice_wget http://dehs.alioth.debian.org/no_updated.txt \
145			dehs_out_of_date.txt
146			nice_wget http://dehs.alioth.debian.org/no_upstream.txt \
147			dehs_error.txt
148
149	hertzog	344	# What more ?
150
151			# Decompress all files
152			for file in *.gz; do
153	jeroen	910	gzip -d -c $file > ${file%%.gz}.new
154			touch -r $file ${file%%.gz}.new
155			mv ${file%%.gz}.new ${file%%.gz}
156	hertzog	344	done
157
158	jeroen	910	# vim: ts=8
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision