Contents of /trunk/scripts/fai-mirror

#! /bin/bash

# $Id$
#*********************************************************************
#
# fai-mirror -- create and manage a partitial mirror for FAI
#
# This script is part of FAI (Fully Automatic Installation)
# (c) 2004-2005, Thomas Lange, lange@informatik.uni-koeln.de
#
#*********************************************************************
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; see the file COPYING. If not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
# MA 02111-1307, USA.
#*********************************************************************

version="Version 1.2, 16-jan-2005"

# additional files: install_packages.conf
# variables: NFSROOT, FAI_CONFIGDIR

# TODO:
# - add support for package that will be removed like: lilo-
#   currently you must remove those package from you package_config files

set -a 
. /etc/fai/fai.conf
FAI_ROOT=/ # do not execute in chroot
[ -z "$MAXPACKAGES" ] && MAXPACKAGES=9999
set +a

trap "umount_dirs" EXIT ERR
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
usage() {

    echo "fai-mirror -- create and manage a partitial mirror for FAI."
    echo "$version"
    echo "Please the the manual page fai-mirror(1)."
    exit
}
# - - - - - - - - - - - - - - - - - - - - - - - - - -
die() {

    local e=$1   # first parameter is the exit code
    shift

    echo "$@"    # print error message
    exit $e
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
excludeclass() {

    # removes/excludes all classes in $* from $classes
    local insert eclasses newclass c e

    eclasses=$(echo $* | sed "s/,/ /g")
   
    for c in $classes; do
        insert=1
        for e in $eclasses; do
          [ $c = $e ] && insert=0
        done
        [ $insert = 1 ] && newclass="$newclass $c"
    done
    classes="$newclass"
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
get_addpackages() {

    # add packages (mostly kernels) which are defined in the variable $addpackages
    # TODO: maybe better to create a tmp file with all these package
    # names, then to call install_packages for the tmp file

    local f p pkg pkglist

    echo -n "Adding packages from variable \$addpackages:"
    for f in $(echo $FAI_CONFIGDIR/class/*.var); do
        pkg=$(egrep ^addpackages= $f | sed 's/addpackages=//'|sed 's/"//g'| perl -pe 's/\$[\w_-]+//g')
        pkglist="$pkglist $pkg"
    done

    # loop over the list, because maybe some packages doesn't exist in the
    # partitial mirror, but only in the local repository. These can't be
    # downloaded via the normal method
    for p in $pkglist; do
        echo -n " $p"
        # test if .deb file is available
        if [ ! -f $FAI_CONFIG/files/packages/${p}_*.deb ]; then
            apt-get -qq -d $aptoptions -y --force-yes --fix-missing install $p
        fi
    done
    echo ""
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
umount_dirs() {

    [ "$FAI_DEBMIRROR" ] && umount $FAI_ROOT/$MNTPOINT 2>/dev/null 1>&2
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
cleandirs() {

    return # currently nothing to do
#    rm $statefile
#    rm -rf $mirrordir/.apt-move  $statefile
#    [ $debug -eq 1 ] || rm -rf $aptcache $archivedir
#    rm -rf $aptcache $archivedir
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
initialize() {

    # TODO: root is only needed when FAI_DEBMIRROR is defined. Then we
    # must mount a directory

    if [ ! -f /etc/fai/install_packages.conf ]; then
        echo "/etc/fai/install_packages.conf not found. Aborting."
        exit 3
    fi

    # store all packages temporary in the mirror partition so we need only a move,
    # not a copy later during apt-move
    aptcache=$mirrordir/aptcache   # holds the package cache data
    archivedir=$aptcache/var/cache/apt/archives
    aptmovefile=$aptcache/etc/apt-move.conf # stores apt-move.conf
    statefile=$aptcache/statefile   # also used in install_packages.conf

    # also used in install_pacakges.conf
    export aptoptions=" \
      -o Dir::State::status=$statefile \
      -o Dir::Cache=$aptcache/var/cache/apt \
      -o Dir::State=$aptcache/var/cache/apt \
      -o Dir::Cache::Archives=$archivedir \
      -o Dir::Etc::sourcelist=$aptcache/etc/apt/sources.list \
      -o Dir::State::Lists=$aptcache/var/lib/apt/lists/"

    # not needed
    # -o APT::Get::ReInstall
    # ."-o APT::Get::Download-Only=true -o Aptitude::Cmd-Line::Download-Only=true "

    # we only need some empty dirs
    mkdir -p $archivedir/partial $aptcache/etc/apt
    mkdir -p $aptcache/var/lib/apt/lists/partial 
    > $statefile
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
delete_base_packages() {

    # now delete all packages that are already included in base.tgz
    local p

    if [ ! -f $NFSROOT/var/tmp/base-pkgs.lis ]; then
        echo "$NFSROOT/var/tmp/base-pkgs.lis not available."
        echo "Can't remove wasteful packages that are already in base.tgz."
        return
    fi
    echo "Removing packages that are already included in base.tgz"
    for p in $(cat $NFSROOT/var/tmp/base-pkgs.lis); do
        if [ -f $archivedir/${p}_*.deb ]; then
            [ $verbose -eq 1 ] && echo "deleting package $p"
            rm $archivedir/${p}_*.deb
        # else commands only for debugging
        #    else
        #       echo "package $p not found"
        #       ls $archivedir/${p}_*.deb
        fi
    done
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
set-classes() {

    # if -c is given, ignore -x
    if [ -n "cclasses" ]; then
        export classes=${cclasses//,/ }
        return
    fi

    # all available file names are classes
    classes=$(cd $FAI_CONFIGDIR/package_config; ls -1 | egrep "^[A-Z0-9_]+$")
    addclasses=$(grep -h PACKAGES $FAI_CONFIGDIR/package_config/* | awk '{printf $3"\n"$4"\n"$5"\n"$6"\n"}')
    export classes=$(echo -e "$classes\n$addclasses\n" | sort | uniq)
    [ -n "$exclasses" ] && excludeclass $exclasses
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

preserve=0
verbose=0
while getopts "vhx:pc:" opt ; do
    case "$opt" in
        h) usage ;;
        x) exclasses="$OPTARG";;
        c) cclasses="$OPTARG";;
        p) preserve=1;;
        v) verbose=1; vflag=-v;;
        ?) echo "Unknown option"; exit 1;;
    esac
done
shift $(($OPTIND - 1))

[ -n "$exclasses" -a -n "$cclasses" ] && die 3 "Options -x and -c not allowed at the same time."
debug=0
[ $debug -eq 0 ] && quiet=-q

mirrordir=$1
if [ -z "$mirrordir" ]; then
    echo "Please give the directory for the mirror."
    exit 2
fi
if [ -d $mirrordir/pool -o -d $mirrordir/dists ]; then
    echo "Please first remove $mirrordir/pool and $mirrordir/dists"
    exit 3
fi

initialize

# if we are using nfs mounts for Debian mirror, this may fail here, since inside a chroot environment different dir are used

# if sources.list includes file AND FAI_DEBMIRROR is defined we have to mount
# otherwise mounting is not needed. call task_mirror

[ "$FAI_DEBMIRROR" ] && mount -r $FAI_DEBMIRROR $FAI_ROOT/$MNTPOINT || exit 9
# TODO: use -p to preserve sources.list
perl -p -e 's/file:/copy:/' /etc/fai/sources.list > $aptcache/etc/apt/sources.list

echo "Getting package information"
apt-get $quiet $aptoptions update >/dev/null

set-classes
echo "Downloading packages for classes:" $classes
# flag -i for install_packages seems to be silly. Remove it ASAP
FAI=$FAI_CONFIGDIR install_packages $vflag -f /etc/fai
get_addpackages
umount_dirs
trap "" EXIT ERR
delete_base_packages

# create mirror directory structure
echo "Calling apt-move"
cat > $aptmovefile <<EOF   # generate apt-move.conf
APTSITES=*
LOCALDIR=$mirrordir
DIST=sarge
FILECACHE=$archivedir
LISTSTATE=$aptcache/var/lib/apt/lists
DELETE=no
CONTENTS=no
EOF
apt-move $quiet -c $aptmovefile update
echo "$0 finished."
echo -n "Mirror size and location: ";du -sh $mirrordir
cleandirs
1	lange	2566	#! /bin/bash
2
3			# $Id$
4			#*********************************************************************
5			#
6			# fai-mirror -- create and manage a partitial mirror for FAI
7			#
8			# This script is part of FAI (Fully Automatic Installation)
9			# (c) 2004-2005, Thomas Lange, lange@informatik.uni-koeln.de
10			#
11			#*********************************************************************
12			# This program is free software; you can redistribute it and/or modify
13			# it under the terms of the GNU General Public License as published by
14			# the Free Software Foundation; either version 2 of the License, or
15			# (at your option) any later version.
16			#
17			# This program is distributed in the hope that it will be useful, but
18			# WITHOUT ANY WARRANTY; without even the implied warranty of
19			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20			# General Public License for more details.
21			#
22			# You should have received a copy of the GNU General Public License
23			# along with this program; see the file COPYING. If not, write to the
24			# Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
25			# MA 02111-1307, USA.
26			#*********************************************************************
27
28	lange	2590	version="Version 1.2, 16-jan-2005"
29	lange	2566
30			# additional files: install_packages.conf
31			# variables: NFSROOT, FAI_CONFIGDIR
32
33			# TODO:
34			# - add support for package that will be removed like: lilo-
35			# currently you must remove those package from you package_config files
36
37			set -a
38			. /etc/fai/fai.conf
39			FAI_ROOT=/ # do not execute in chroot
40			[ -z "$MAXPACKAGES" ] && MAXPACKAGES=9999
41			set +a
42
43			trap "umount_dirs" EXIT ERR
44			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
45			usage() {
46
47	lange	2588	echo "fai-mirror -- create and manage a partitial mirror for FAI."
48			echo "$version"
49	lange	2581	echo "Please the the manual page fai-mirror(1)."
50	lange	2566	exit
51			}
52	lange	2590	# - - - - - - - - - - - - - - - - - - - - - - - - - -
53			die() {
54
55			local e=$1 # first parameter is the exit code
56			shift
57
58			echo "$@" # print error message
59			exit $e
60			}
61	lange	2566	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
62			excludeclass() {
63
64			# removes/excludes all classes in $* from $classes
65			local insert eclasses newclass c e
66
67			eclasses=$(echo $* \| sed "s/,/ /g")
68
69			for c in $classes; do
70			insert=1
71			for e in $eclasses; do
72			[ $c = $e ] && insert=0
73			done
74			[ $insert = 1 ] && newclass="$newclass $c"
75			done
76			classes="$newclass"
77			}
78			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
79			get_addpackages() {
80
81			# add packages (mostly kernels) which are defined in the variable $addpackages
82	lange	2581	# TODO: maybe better to create a tmp file with all these package
83	lange	2566	# names, then to call install_packages for the tmp file
84
85			local f p pkg pkglist
86
87			echo -n "Adding packages from variable \$addpackages:"
88			for f in $(echo $FAI_CONFIGDIR/class/*.var); do
89			pkg=$(egrep ^addpackages= $f \| sed 's/addpackages=//'\|sed 's/"//g'\| perl -pe 's/\$[\w_-]+//g')
90			pkglist="$pkglist $pkg"
91			done
92
93			# loop over the list, because maybe some packages doesn't exist in the
94			# partitial mirror, but only in the local repository. These can't be
95			# downloaded via the normal method
96			for p in $pkglist; do
97			echo -n " $p"
98			# test if .deb file is available
99			if [ ! -f $FAI_CONFIG/files/packages/${p}_*.deb ]; then
100			apt-get -qq -d $aptoptions -y --force-yes --fix-missing install $p
101			fi
102			done
103			echo ""
104			}
105			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
106			umount_dirs() {
107
108			[ "$FAI_DEBMIRROR" ] && umount $FAI_ROOT/$MNTPOINT 2>/dev/null 1>&2
109			}
110			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
111			cleandirs() {
112
113	lange	2576	return # currently nothing to do
114	lange	2566	# rm $statefile
115			# rm -rf $mirrordir/.apt-move $statefile
116			# [ $debug -eq 1 ] \|\| rm -rf $aptcache $archivedir
117			# rm -rf $aptcache $archivedir
118			}
119			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
120			initialize() {
121
122			# TODO: root is only needed when FAI_DEBMIRROR is defined. Then we
123			# must mount a directory
124
125			if [ ! -f /etc/fai/install_packages.conf ]; then
126			echo "/etc/fai/install_packages.conf not found. Aborting."
127			exit 3
128			fi
129
130			# store all packages temporary in the mirror partition so we need only a move,
131			# not a copy later during apt-move
132			aptcache=$mirrordir/aptcache # holds the package cache data
133			archivedir=$aptcache/var/cache/apt/archives
134			aptmovefile=$aptcache/etc/apt-move.conf # stores apt-move.conf
135			statefile=$aptcache/statefile # also used in install_packages.conf
136
137			# also used in install_pacakges.conf
138			export aptoptions=" \
139			-o Dir::State::status=$statefile \
140			-o Dir::Cache=$aptcache/var/cache/apt \
141			-o Dir::State=$aptcache/var/cache/apt \
142			-o Dir::Cache::Archives=$archivedir \
143			-o Dir::Etc::sourcelist=$aptcache/etc/apt/sources.list \
144			-o Dir::State::Lists=$aptcache/var/lib/apt/lists/"
145
146			# not needed
147			# -o APT::Get::ReInstall
148			# ."-o APT::Get::Download-Only=true -o Aptitude::Cmd-Line::Download-Only=true "
149
150			# we only need some empty dirs
151			mkdir -p $archivedir/partial $aptcache/etc/apt
152			mkdir -p $aptcache/var/lib/apt/lists/partial
153			> $statefile
154			}
155			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
156			delete_base_packages() {
157
158			# now delete all packages that are already included in base.tgz
159			local p
160
161	lange	2590	if [ ! -f $NFSROOT/var/tmp/base-pkgs.lis ]; then
162			echo "$NFSROOT/var/tmp/base-pkgs.lis not available."
163			echo "Can't remove wasteful packages that are already in base.tgz."
164			return
165			fi
166	lange	2566	echo "Removing packages that are already included in base.tgz"
167			for p in $(cat $NFSROOT/var/tmp/base-pkgs.lis); do
168			if [ -f $archivedir/${p}_*.deb ]; then
169			[ $verbose -eq 1 ] && echo "deleting package $p"
170			rm $archivedir/${p}_*.deb
171			# else commands only for debugging
172			# else
173			# echo "package $p not found"
174			# ls $archivedir/${p}_*.deb
175			fi
176			done
177			}
178			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
179	lange	2590	set-classes() {
180	lange	2566
181	lange	2590	# if -c is given, ignore -x
182			if [ -n "cclasses" ]; then
183			export classes=${cclasses//,/ }
184			return
185			fi
186
187			# all available file names are classes
188			classes=$(cd $FAI_CONFIGDIR/package_config; ls -1 \| egrep "^[A-Z0-9_]+$")
189			addclasses=$(grep -h PACKAGES $FAI_CONFIGDIR/package_config/* \| awk '{printf $3"\n"$4"\n"$5"\n"$6"\n"}')
190			export classes=$(echo -e "$classes\n$addclasses\n" \| sort \| uniq)
191			[ -n "$exclasses" ] && excludeclass $exclasses
192			}
193			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
194
195	lange	2566	preserve=0
196			verbose=0
197	lange	2590	while getopts "vhx:pc:" opt ; do
198	lange	2566	case "$opt" in
199			h) usage ;;
200			x) exclasses="$OPTARG";;
201	lange	2590	c) cclasses="$OPTARG";;
202	lange	2566	p) preserve=1;;
203			v) verbose=1; vflag=-v;;
204			?) echo "Unknown option"; exit 1;;
205			esac
206			done
207			shift $(($OPTIND - 1))
208
209	lange	2590	[ -n "$exclasses" -a -n "$cclasses" ] && die 3 "Options -x and -c not allowed at the same time."
210	lange	2566	debug=0
211			[ $debug -eq 0 ] && quiet=-q
212
213			mirrordir=$1
214	lange	2576	if [ -z "$mirrordir" ]; then
215			echo "Please give the directory for the mirror."
216			exit 2
217			fi
218	lange	2566	if [ -d $mirrordir/pool -o -d $mirrordir/dists ]; then
219			echo "Please first remove $mirrordir/pool and $mirrordir/dists"
220			exit 3
221			fi
222
223			initialize
224
225			# if we are using nfs mounts for Debian mirror, this may fail here, since inside a chroot environment different dir are used
226
227			# if sources.list includes file AND FAI_DEBMIRROR is defined we have to mount
228			# otherwise mounting is not needed. call task_mirror
229
230	lange	2588	[ "$FAI_DEBMIRROR" ] && mount -r $FAI_DEBMIRROR $FAI_ROOT/$MNTPOINT \|\| exit 9
231	lange	2566	# TODO: use -p to preserve sources.list
232			perl -p -e 's/file:/copy:/' /etc/fai/sources.list > $aptcache/etc/apt/sources.list
233
234			echo "Getting package information"
235			apt-get $quiet $aptoptions update >/dev/null
236
237	lange	2590	set-classes
238	lange	2566	echo "Downloading packages for classes:" $classes
239	lange	2590	# flag -i for install_packages seems to be silly. Remove it ASAP
240			FAI=$FAI_CONFIGDIR install_packages $vflag -f /etc/fai
241	lange	2566	get_addpackages
242			umount_dirs
243	lange	2576	trap "" EXIT ERR
244	lange	2566	delete_base_packages
245
246			# create mirror directory structure
247			echo "Calling apt-move"
248			cat > $aptmovefile <<EOF # generate apt-move.conf
249			APTSITES=*
250			LOCALDIR=$mirrordir
251			DIST=sarge
252			FILECACHE=$archivedir
253			LISTSTATE=$aptcache/var/lib/apt/lists
254			DELETE=no
255			CONTENTS=no
256			EOF
257			apt-move $quiet -c $aptmovefile update
258			echo "$0 finished."
259			echo -n "Mirror size and location: ";du -sh $mirrordir
260			cleandirs
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision