Contents of /trunk/scripts/fai-mirror

#! /bin/bash

# $Id$
#*********************************************************************
#
# fai-mirror -- create and manage a partitial mirror for FAI
#
# This script is part of FAI (Fully Automatic Installation)
# (c) 2004-2005, Thomas Lange, lange@informatik.uni-koeln.de
#
#*********************************************************************
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; see the file COPYING. If not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
# MA 02111-1307, USA.
#*********************************************************************

version="Version 1.2, 16-jan-2005"

# additional files: install_packages.conf
# variables: NFSROOT, FAI_CONFIGDIR

# TODO:
# - add support for package that will be removed like: lilo-
#   currently you must remove those package from you package_config files

set -a 
. /etc/fai/fai.conf
FAI_ROOT=/ # do not execute in chroot
[ -z "$MAXPACKAGES" ] && MAXPACKAGES=9999
set +a

trap "umount_dirs" EXIT ERR
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
usage() {

    echo "fai-mirror -- create and manage a partitial mirror for FAI."
    echo "$version"
    echo "Please the the manual page fai-mirror(1)."
    exit
}
# - - - - - - - - - - - - - - - - - - - - - - - - - -
die() {

    local e=$1   # first parameter is the exit code
    shift

    echo "$@"    # print error message
    exit $e
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
excludeclass() {

    # removes/excludes all classes in $* from $classes
    local insert eclasses newclass c e

    eclasses=$(echo $* | sed "s/,/ /g")
   
    for c in $classes; do
        insert=1
        for e in $eclasses; do
          [ $c = $e ] && insert=0
        done
        [ $insert = 1 ] && newclass="$newclass $c"
    done
    classes="$newclass"
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
get_addpackages() {

    # add packages (mostly kernels) which are defined in the variable $addpackages
    # TODO: maybe better to create a tmp file with all these package
    # names, then to call install_packages for the tmp file

    local f p pkg pkglist

    echo -n "Adding packages from variable \$addpackages:"
    for f in $(echo $FAI_CONFIGDIR/class/*.var); do
        pkg=$(egrep ^addpackages= $f | sed 's/addpackages=//'|sed 's/"//g'| perl -pe 's/\$[\w_-]+//g')
        pkglist="$pkglist $pkg"
    done

    # loop over the list, because maybe some packages doesn't exist in the
    # partitial mirror, but only in the local repository. These can't be
    # downloaded via the normal method
    for p in $pkglist; do
        echo -n " $p"
        # test if .deb file is available
        if [ ! -f $FAI_CONFIG/files/packages/${p}_*.deb ]; then
            apt-get -qq -d $aptoptions -y --force-yes --fix-missing install $p
        fi
    done
    echo ""
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
umount_dirs() {

    [ "$FAI_DEBMIRROR" ] && umount $FAI_ROOT/$MNTPOINT 2>/dev/null 1>&2
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
cleandirs() {

    return # currently nothing to do
#    rm $statefile
#    rm -rf $mirrordir/.apt-move  $statefile
#    [ $debug -eq 1 ] || rm -rf $aptcache $archivedir
#    rm -rf $aptcache $archivedir
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
initialize() {

    # TODO: root is only needed when FAI_DEBMIRROR is defined. Then we
    # must mount a directory

    if [ ! -f /etc/fai/install_packages.conf ]; then
        echo "/etc/fai/install_packages.conf not found. Aborting."
        exit 3
    fi

    # store all packages temporary in the mirror partition so we need only a move,
    # not a copy later during apt-move
    aptcache=$mirrordir/aptcache   # holds the package cache data
    archivedir=$aptcache/var/cache/apt/archives
    aptmovefile=$aptcache/etc/apt-move.conf # stores apt-move.conf
    statefile=$aptcache/statefile   # also used in install_packages.conf

    # also used in install_pacakges.conf
    export aptoptions=" \
      -o Dir::State::status=$statefile \
      -o Dir::Cache=$aptcache/var/cache/apt \
      -o Dir::State=$aptcache/var/cache/apt \
      -o Dir::Cache::Archives=$archivedir \
      -o Dir::Etc::sourcelist=$aptcache/etc/apt/sources.list \
      -o Dir::State::Lists=$aptcache/var/lib/apt/lists/"

    # not needed
    # -o APT::Get::ReInstall
    # ."-o APT::Get::Download-Only=true -o Aptitude::Cmd-Line::Download-Only=true "

    # we only need some empty dirs
    mkdir -p $archivedir/partial $aptcache/etc/apt
    mkdir -p $aptcache/var/lib/apt/lists/partial 
    > $statefile
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
delete_base_packages() {

    # now delete all packages that are already included in base.tgz
    local p

    if [ ! -f $NFSROOT/var/tmp/base-pkgs.lis ]; then
        echo "$NFSROOT/var/tmp/base-pkgs.lis not available."
        echo "Can't remove wasteful packages that are already in base.tgz."
        return
    fi
    echo "Removing packages that are already included in base.tgz"
    for p in $(cat $NFSROOT/var/tmp/base-pkgs.lis); do
        if [ -f $archivedir/${p}_*.deb ]; then
            [ $verbose -eq 1 ] && echo "deleting package $p"
            rm $archivedir/${p}_*.deb
        # else commands only for debugging
        #    else
        #       echo "package $p not found"
        #       ls $archivedir/${p}_*.deb
        fi
    done
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
set-classes() {

    # if -c is given, ignore -x
    if [ -n "$cclasses" ]; then
        export classes=${cclasses//,/ }
        return
    fi

    set +e
    # all available file names are classes
    classes=$(cd $FAI_CONFIGDIR/package_config; ls -1 | egrep "^[A-Z0-9_]+$")
    addclasses=$(grep -h PACKAGES $FAI_CONFIGDIR/package_config/* | awk '{printf $3"\n"$4"\n"$5"\n"$6"\n"}')
    export classes=$(echo -e "$classes\n$addclasses\n" | sort | uniq)
    [ -n "$exclasses" ] && excludeclass $exclasses
    set -e
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

preserve=0
verbose=0
while getopts "vhx:pc:" opt ; do
    case "$opt" in
        h) usage ;;
        x) exclasses="$OPTARG";;
        c) cclasses="$OPTARG";;
        p) preserve=1;;
        v) verbose=1; vflag=-v;;
        ?) echo "Unknown option"; exit 1;;
    esac
done
shift $(($OPTIND - 1))

[ -n "$exclasses" -a -n "$cclasses" ] && die 3 "Options -x and -c not allowed at the same time."
debug=0
[ $debug -eq 0 ] && quiet=-q

mirrordir=$1
if [ -z "$mirrordir" ]; then
    echo "Please give the directory for the mirror."
    exit 2
fi
if [ -d $mirrordir/pool -o -d $mirrordir/dists ]; then
    echo "Please first remove $mirrordir/pool and $mirrordir/dists"
    exit 3
fi

initialize

# if we are using nfs mounts for Debian mirror, this may fail here, since inside a chroot environment different dir are used

# if sources.list includes file AND FAI_DEBMIRROR is defined we have to mount
# otherwise mounting is not needed. call task_mirror

[ "$FAI_DEBMIRROR" ] && mount -r $FAI_DEBMIRROR $FAI_ROOT/$MNTPOINT || exit 9
# TODO: use -p to preserve sources.list
perl -p -e 's/file:/copy:/' /etc/fai/sources.list > $aptcache/etc/apt/sources.list

echo "Getting package information"
apt-get $quiet $aptoptions update >/dev/null

set-classes
echo "Downloading packages for classes:" $classes
# flag -i for install_packages seems to be silly. Remove it ASAP
FAI=$FAI_CONFIGDIR install_packages $vflag -f /etc/fai
get_addpackages
umount_dirs
trap "" EXIT ERR
delete_base_packages

# create mirror directory structure
echo "Calling apt-move"
cat > $aptmovefile <<EOF   # generate apt-move.conf
APTSITES=*
LOCALDIR=$mirrordir
DIST=sarge
FILECACHE=$archivedir
LISTSTATE=$aptcache/var/lib/apt/lists
DELETE=no
CONTENTS=no
EOF
apt-move $quiet -c $aptmovefile update
echo "$0 finished."
echo -n "Mirror size and location: ";du -sh $mirrordir
cleandirs
1	lange	2566	#! /bin/bash
2
3			# $Id$
4			#*********************************************************************
5			#
6			# fai-mirror -- create and manage a partitial mirror for FAI
7			#
8			# This script is part of FAI (Fully Automatic Installation)
9			# (c) 2004-2005, Thomas Lange, lange@informatik.uni-koeln.de
10			#
11			#*********************************************************************
12			# This program is free software; you can redistribute it and/or modify
13			# it under the terms of the GNU General Public License as published by
14			# the Free Software Foundation; either version 2 of the License, or
15			# (at your option) any later version.
16			#
17			# This program is distributed in the hope that it will be useful, but
18			# WITHOUT ANY WARRANTY; without even the implied warranty of
19			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20			# General Public License for more details.
21			#
22			# You should have received a copy of the GNU General Public License
23			# along with this program; see the file COPYING. If not, write to the
24			# Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
25			# MA 02111-1307, USA.
26			#*********************************************************************
27
28	lange	2590	version="Version 1.2, 16-jan-2005"
29	lange	2566
30			# additional files: install_packages.conf
31			# variables: NFSROOT, FAI_CONFIGDIR
32
33			# TODO:
34			# - add support for package that will be removed like: lilo-
35			# currently you must remove those package from you package_config files
36
37			set -a
38			. /etc/fai/fai.conf
39			FAI_ROOT=/ # do not execute in chroot
40			[ -z "$MAXPACKAGES" ] && MAXPACKAGES=9999
41			set +a
42
43			trap "umount_dirs" EXIT ERR
44			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
45			usage() {
46
47	lange	2588	echo "fai-mirror -- create and manage a partitial mirror for FAI."
48			echo "$version"
49	lange	2581	echo "Please the the manual page fai-mirror(1)."
50	lange	2566	exit
51			}
52	lange	2590	# - - - - - - - - - - - - - - - - - - - - - - - - - -
53			die() {
54
55			local e=$1 # first parameter is the exit code
56			shift
57
58			echo "$@" # print error message
59			exit $e
60			}
61	lange	2566	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
62			excludeclass() {
63
64			# removes/excludes all classes in $* from $classes
65			local insert eclasses newclass c e
66
67			eclasses=$(echo $* \| sed "s/,/ /g")
68
69			for c in $classes; do
70			insert=1
71			for e in $eclasses; do
72			[ $c = $e ] && insert=0
73			done
74			[ $insert = 1 ] && newclass="$newclass $c"
75			done
76			classes="$newclass"
77			}
78			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
79			get_addpackages() {
80
81			# add packages (mostly kernels) which are defined in the variable $addpackages
82	lange	2581	# TODO: maybe better to create a tmp file with all these package
83	lange	2566	# names, then to call install_packages for the tmp file
84
85			local f p pkg pkglist
86
87			echo -n "Adding packages from variable \$addpackages:"
88			for f in $(echo $FAI_CONFIGDIR/class/*.var); do
89			pkg=$(egrep ^addpackages= $f \| sed 's/addpackages=//'\|sed 's/"//g'\| perl -pe 's/\$[\w_-]+//g')
90			pkglist="$pkglist $pkg"
91			done
92
93			# loop over the list, because maybe some packages doesn't exist in the
94			# partitial mirror, but only in the local repository. These can't be
95			# downloaded via the normal method
96			for p in $pkglist; do
97			echo -n " $p"
98			# test if .deb file is available
99			if [ ! -f $FAI_CONFIG/files/packages/${p}_*.deb ]; then
100			apt-get -qq -d $aptoptions -y --force-yes --fix-missing install $p
101			fi
102			done
103			echo ""
104			}
105			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
106			umount_dirs() {
107
108			[ "$FAI_DEBMIRROR" ] && umount $FAI_ROOT/$MNTPOINT 2>/dev/null 1>&2
109			}
110			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
111			cleandirs() {
112
113	lange	2576	return # currently nothing to do
114	lange	2566	# rm $statefile
115			# rm -rf $mirrordir/.apt-move $statefile
116			# [ $debug -eq 1 ] \|\| rm -rf $aptcache $archivedir
117			# rm -rf $aptcache $archivedir
118			}
119			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
120			initialize() {
121
122			# TODO: root is only needed when FAI_DEBMIRROR is defined. Then we
123			# must mount a directory
124
125			if [ ! -f /etc/fai/install_packages.conf ]; then
126			echo "/etc/fai/install_packages.conf not found. Aborting."
127			exit 3
128			fi
129
130			# store all packages temporary in the mirror partition so we need only a move,
131			# not a copy later during apt-move
132			aptcache=$mirrordir/aptcache # holds the package cache data
133			archivedir=$aptcache/var/cache/apt/archives
134			aptmovefile=$aptcache/etc/apt-move.conf # stores apt-move.conf
135			statefile=$aptcache/statefile # also used in install_packages.conf
136
137			# also used in install_pacakges.conf
138			export aptoptions=" \
139			-o Dir::State::status=$statefile \
140			-o Dir::Cache=$aptcache/var/cache/apt \
141			-o Dir::State=$aptcache/var/cache/apt \
142			-o Dir::Cache::Archives=$archivedir \
143			-o Dir::Etc::sourcelist=$aptcache/etc/apt/sources.list \
144			-o Dir::State::Lists=$aptcache/var/lib/apt/lists/"
145
146			# not needed
147			# -o APT::Get::ReInstall
148			# ."-o APT::Get::Download-Only=true -o Aptitude::Cmd-Line::Download-Only=true "
149
150			# we only need some empty dirs
151			mkdir -p $archivedir/partial $aptcache/etc/apt
152			mkdir -p $aptcache/var/lib/apt/lists/partial
153			> $statefile
154			}
155			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
156			delete_base_packages() {
157
158			# now delete all packages that are already included in base.tgz
159			local p
160
161	lange	2590	if [ ! -f $NFSROOT/var/tmp/base-pkgs.lis ]; then
162			echo "$NFSROOT/var/tmp/base-pkgs.lis not available."
163			echo "Can't remove wasteful packages that are already in base.tgz."
164			return
165			fi
166	lange	2566	echo "Removing packages that are already included in base.tgz"
167			for p in $(cat $NFSROOT/var/tmp/base-pkgs.lis); do
168			if [ -f $archivedir/${p}_*.deb ]; then
169			[ $verbose -eq 1 ] && echo "deleting package $p"
170			rm $archivedir/${p}_*.deb
171			# else commands only for debugging
172			# else
173			# echo "package $p not found"
174			# ls $archivedir/${p}_*.deb
175			fi
176			done
177			}
178			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
179	lange	2590	set-classes() {
180	lange	2566
181	lange	2590	# if -c is given, ignore -x
182	lange	2592	if [ -n "$cclasses" ]; then
183	lange	2590	export classes=${cclasses//,/ }
184			return
185			fi
186
187	lange	2592	set +e
188	lange	2590	# all available file names are classes
189			classes=$(cd $FAI_CONFIGDIR/package_config; ls -1 \| egrep "^[A-Z0-9_]+$")
190			addclasses=$(grep -h PACKAGES $FAI_CONFIGDIR/package_config/* \| awk '{printf $3"\n"$4"\n"$5"\n"$6"\n"}')
191			export classes=$(echo -e "$classes\n$addclasses\n" \| sort \| uniq)
192			[ -n "$exclasses" ] && excludeclass $exclasses
193	lange	2592	set -e
194	lange	2590	}
195			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
196
197	lange	2566	preserve=0
198			verbose=0
199	lange	2590	while getopts "vhx:pc:" opt ; do
200	lange	2566	case "$opt" in
201			h) usage ;;
202			x) exclasses="$OPTARG";;
203	lange	2590	c) cclasses="$OPTARG";;
204	lange	2566	p) preserve=1;;
205			v) verbose=1; vflag=-v;;
206			?) echo "Unknown option"; exit 1;;
207			esac
208			done
209			shift $(($OPTIND - 1))
210
211	lange	2590	[ -n "$exclasses" -a -n "$cclasses" ] && die 3 "Options -x and -c not allowed at the same time."
212	lange	2566	debug=0
213			[ $debug -eq 0 ] && quiet=-q
214
215			mirrordir=$1
216	lange	2576	if [ -z "$mirrordir" ]; then
217			echo "Please give the directory for the mirror."
218			exit 2
219			fi
220	lange	2566	if [ -d $mirrordir/pool -o -d $mirrordir/dists ]; then
221			echo "Please first remove $mirrordir/pool and $mirrordir/dists"
222			exit 3
223			fi
224
225			initialize
226
227			# if we are using nfs mounts for Debian mirror, this may fail here, since inside a chroot environment different dir are used
228
229			# if sources.list includes file AND FAI_DEBMIRROR is defined we have to mount
230			# otherwise mounting is not needed. call task_mirror
231
232	lange	2588	[ "$FAI_DEBMIRROR" ] && mount -r $FAI_DEBMIRROR $FAI_ROOT/$MNTPOINT \|\| exit 9
233	lange	2566	# TODO: use -p to preserve sources.list
234			perl -p -e 's/file:/copy:/' /etc/fai/sources.list > $aptcache/etc/apt/sources.list
235
236			echo "Getting package information"
237			apt-get $quiet $aptoptions update >/dev/null
238
239	lange	2590	set-classes
240	lange	2566	echo "Downloading packages for classes:" $classes
241	lange	2590	# flag -i for install_packages seems to be silly. Remove it ASAP
242			FAI=$FAI_CONFIGDIR install_packages $vflag -f /etc/fai
243	lange	2566	get_addpackages
244			umount_dirs
245	lange	2576	trap "" EXIT ERR
246	lange	2566	delete_base_packages
247
248			# create mirror directory structure
249			echo "Calling apt-move"
250			cat > $aptmovefile <<EOF # generate apt-move.conf
251			APTSITES=*
252			LOCALDIR=$mirrordir
253			DIST=sarge
254			FILECACHE=$archivedir
255			LISTSTATE=$aptcache/var/lib/apt/lists
256			DELETE=no
257			CONTENTS=no
258			EOF
259			apt-move $quiet -c $aptmovefile update
260			echo "$0 finished."
261			echo -n "Mirror size and location: ";du -sh $mirrordir
262			cleandirs
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision