Contents of /trunk/scripts/fai-mirror

#! /bin/bash

# $Id$
#*********************************************************************
#
# fai-mirror -- create and manage a partitial mirror for FAI
#
# This script is part of FAI (Fully Automatic Installation)
# (c) 2004-2005, Thomas Lange, lange@informatik.uni-koeln.de
#
#*********************************************************************
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; see the file COPYING. If not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
# MA 02111-1307, USA.
#*********************************************************************

version="Version 1.2, 12-jan-2005"

# additional files: install_packages.conf
# variables: NFSROOT, FAI_CONFIGDIR

# TODO:
# - cleanup code
# - add support for package that will be removed like: lilo-
#   currently you must remove those package from you package_config files

set -a 
. /etc/fai/fai.conf
FAI_ROOT=/ # do not execute in chroot
[ -z "$MAXPACKAGES" ] && MAXPACKAGES=9999
set +a

trap "umount_dirs" EXIT ERR
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
usage() {

    echo "fai-mirror -- create and manage a partitial mirror for FAI"
    echo "$0 $version"
    exit
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
excludeclass() {

    # removes/excludes all classes in $* from $classes
    local insert eclasses newclass c e

    eclasses=$(echo $* | sed "s/,/ /g")
   
    for c in $classes; do
        insert=1
        for e in $eclasses; do
          [ $c = $e ] && insert=0
        done
        [ $insert = 1 ] && newclass="$newclass $c"
    done
    classes="$newclass"
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
get_addpackages() {

    # add packages (mostly kernels) which are defined in the variable $addpackages
    # TODO: maybe better to craeate a tmp file with all these package
    # names, then to call install_packages for the tmp file

    local f p pkg pkglist

    echo -n "Adding packages from variable \$addpackages:"
    for f in $(echo $FAI_CONFIGDIR/class/*.var); do
        pkg=$(egrep ^addpackages= $f | sed 's/addpackages=//'|sed 's/"//g'| perl -pe 's/\$[\w_-]+//g')
        pkglist="$pkglist $pkg"
    done

    # loop over the list, because maybe some packages doesn't exist in the
    # partitial mirror, but only in the local repository. These can't be
    # downloaded via the normal method
    for p in $pkglist; do
        echo -n " $p"
        # test if .deb file is available
        if [ ! -f $FAI_CONFIG/files/packages/${p}_*.deb ]; then
            apt-get -qq -d $aptoptions -y --force-yes --fix-missing install $p
        fi
    done
    echo ""
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
umount_dirs() {

    [ "$FAI_DEBMIRROR" ] && umount $FAI_ROOT/$MNTPOINT 2>/dev/null 1>&2
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
cleandirs() {

    return # currently nothing to do
#    rm $statefile
#    rm -rf $mirrordir/.apt-move  $statefile
#    [ $debug -eq 1 ] || rm -rf $aptcache $archivedir
#    rm -rf $aptcache $archivedir
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
initialize() {

    # TODO: root is only needed when FAI_DEBMIRROR is defined. Then we
    # must mount a directory

    if [ ! -f /etc/fai/install_packages.conf ]; then
        echo "/etc/fai/install_packages.conf not found. Aborting."
        exit 3
    fi

    # store all packages temporary in the mirror partition so we need only a move,
    # not a copy later during apt-move
    aptcache=$mirrordir/aptcache   # holds the package cache data
    archivedir=$aptcache/var/cache/apt/archives
    aptmovefile=$aptcache/etc/apt-move.conf # stores apt-move.conf
    statefile=$aptcache/statefile   # also used in install_packages.conf

    # also used in install_pacakges.conf
    export aptoptions=" \
      -o Dir::State::status=$statefile \
      -o Dir::Cache=$aptcache/var/cache/apt \
      -o Dir::State=$aptcache/var/cache/apt \
      -o Dir::Cache::Archives=$archivedir \
      -o Dir::Etc::sourcelist=$aptcache/etc/apt/sources.list \
      -o Dir::State::Lists=$aptcache/var/lib/apt/lists/"

    # not needed
    # -o APT::Get::ReInstall
    # ."-o APT::Get::Download-Only=true -o Aptitude::Cmd-Line::Download-Only=true "

    # we only need some empty dirs
    mkdir -p $archivedir/partial $aptcache/etc/apt
    mkdir -p $aptcache/var/lib/apt/lists/partial 
    > $statefile
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
delete_base_packages() {

    # now delete all packages that are already included in base.tgz
    local p

    echo "Removing packages that are already included in base.tgz"
    for p in $(cat $NFSROOT/var/tmp/base-pkgs.lis); do
        if [ -f $archivedir/${p}_*.deb ]; then
            [ $verbose -eq 1 ] && echo "deleting package $p"
            rm $archivedir/${p}_*.deb
        # else commands only for debugging
        #    else
        #       echo "package $p not found"
        #       ls $archivedir/${p}_*.deb
        fi
    done
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

preserve=0
verbose=0
while getopts "vhx:p" opt ; do
    case "$opt" in
        h) usage ;;
        x) exclasses="$OPTARG";;
        p) preserve=1;;
        v) verbose=1; vflag=-v;;
        ?) echo "Unknown option"; exit 1;;
    esac
done
shift $(($OPTIND - 1))

debug=0
[ $debug -eq 0 ] && quiet=-q

mirrordir=$1
if [ -z "$mirrordir" ]; then
    echo "Please give the directory for the mirror."
    exit 2
fi
if [ -d $mirrordir/pool -o -d $mirrordir/dists ]; then
    echo "Please first remove $mirrordir/pool and $mirrordir/dists"
    exit 3
fi

initialize

# if we are using nfs mounts for Debian mirror, this may fail here, since inside a chroot environment different dir are used

# if sources.list includes file AND FAI_DEBMIRROR is defined we have to mount
# otherwise mounting is not needed. call task_mirror

[ "$FAI_DEBMIRROR" ] && mount -r $FAI_DEBMIRROR $FAI_ROOT/$MNTPOINT
# TODO: use -p to preserve sources.list
perl -p -e 's/file:/copy:/' /etc/fai/sources.list > $aptcache/etc/apt/sources.list

echo "Getting package information"
apt-get $quiet $aptoptions update >/dev/null

# all available file names are classes
classes=$(cd $FAI_CONFIGDIR/package_config; ls -1 | egrep "^[A-Z0-9_]+$")
addclasses=$(grep -h PACKAGES $FAI_CONFIGDIR/package_config/* | awk '{print $3}')
export classes=$(echo -e "$classes\n$addclasses\n" | sort | uniq)
[ -n "$exclasses" ] && excludeclass $exclasses

echo "Downloading packages for classes:" $classes
FAI=$FAI_CONFIGDIR install_packages -i $vflag -f /etc/fai
get_addpackages
umount_dirs
trap "" EXIT ERR
delete_base_packages

# create mirror directory structure
echo "Calling apt-move"
cat > $aptmovefile <<EOF   # generate apt-move.conf
APTSITES=*
LOCALDIR=$mirrordir
DIST=sarge
FILECACHE=$archivedir
LISTSTATE=$aptcache/var/lib/apt/lists
DELETE=no
CONTENTS=no
EOF
apt-move $quiet -c $aptmovefile update
echo "$0 finished."
echo -n "Mirror size and location: ";du -sh $mirrordir
cleandirs
1	lange	2566	#! /bin/bash
2
3			# $Id$
4			#*********************************************************************
5			#
6			# fai-mirror -- create and manage a partitial mirror for FAI
7			#
8			# This script is part of FAI (Fully Automatic Installation)
9			# (c) 2004-2005, Thomas Lange, lange@informatik.uni-koeln.de
10			#
11			#*********************************************************************
12			# This program is free software; you can redistribute it and/or modify
13			# it under the terms of the GNU General Public License as published by
14			# the Free Software Foundation; either version 2 of the License, or
15			# (at your option) any later version.
16			#
17			# This program is distributed in the hope that it will be useful, but
18			# WITHOUT ANY WARRANTY; without even the implied warranty of
19			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20			# General Public License for more details.
21			#
22			# You should have received a copy of the GNU General Public License
23			# along with this program; see the file COPYING. If not, write to the
24			# Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
25			# MA 02111-1307, USA.
26			#*********************************************************************
27
28	lange	2576	version="Version 1.2, 12-jan-2005"
29	lange	2566
30			# additional files: install_packages.conf
31			# variables: NFSROOT, FAI_CONFIGDIR
32
33			# TODO:
34			# - cleanup code
35			# - add support for package that will be removed like: lilo-
36			# currently you must remove those package from you package_config files
37
38			set -a
39			. /etc/fai/fai.conf
40			FAI_ROOT=/ # do not execute in chroot
41			[ -z "$MAXPACKAGES" ] && MAXPACKAGES=9999
42			set +a
43
44			trap "umount_dirs" EXIT ERR
45			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
46			usage() {
47
48			echo "fai-mirror -- create and manage a partitial mirror for FAI"
49			echo "$0 $version"
50			exit
51			}
52			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
53			excludeclass() {
54
55			# removes/excludes all classes in $* from $classes
56			local insert eclasses newclass c e
57
58			eclasses=$(echo $* \| sed "s/,/ /g")
59
60			for c in $classes; do
61			insert=1
62			for e in $eclasses; do
63			[ $c = $e ] && insert=0
64			done
65			[ $insert = 1 ] && newclass="$newclass $c"
66			done
67			classes="$newclass"
68			}
69			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
70			get_addpackages() {
71
72			# add packages (mostly kernels) which are defined in the variable $addpackages
73			# TODO: maybe better to craeate a tmp file with all these package
74			# names, then to call install_packages for the tmp file
75
76			local f p pkg pkglist
77
78			echo -n "Adding packages from variable \$addpackages:"
79			for f in $(echo $FAI_CONFIGDIR/class/*.var); do
80			pkg=$(egrep ^addpackages= $f \| sed 's/addpackages=//'\|sed 's/"//g'\| perl -pe 's/\$[\w_-]+//g')
81			pkglist="$pkglist $pkg"
82			done
83
84			# loop over the list, because maybe some packages doesn't exist in the
85			# partitial mirror, but only in the local repository. These can't be
86			# downloaded via the normal method
87			for p in $pkglist; do
88			echo -n " $p"
89			# test if .deb file is available
90			if [ ! -f $FAI_CONFIG/files/packages/${p}_*.deb ]; then
91			apt-get -qq -d $aptoptions -y --force-yes --fix-missing install $p
92			fi
93			done
94			echo ""
95			}
96			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
97			umount_dirs() {
98
99			[ "$FAI_DEBMIRROR" ] && umount $FAI_ROOT/$MNTPOINT 2>/dev/null 1>&2
100			}
101			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
102			cleandirs() {
103
104	lange	2576	return # currently nothing to do
105	lange	2566	# rm $statefile
106			# rm -rf $mirrordir/.apt-move $statefile
107			# [ $debug -eq 1 ] \|\| rm -rf $aptcache $archivedir
108			# rm -rf $aptcache $archivedir
109			}
110			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
111			initialize() {
112
113			# TODO: root is only needed when FAI_DEBMIRROR is defined. Then we
114			# must mount a directory
115
116			if [ ! -f /etc/fai/install_packages.conf ]; then
117			echo "/etc/fai/install_packages.conf not found. Aborting."
118			exit 3
119			fi
120
121			# store all packages temporary in the mirror partition so we need only a move,
122			# not a copy later during apt-move
123			aptcache=$mirrordir/aptcache # holds the package cache data
124			archivedir=$aptcache/var/cache/apt/archives
125			aptmovefile=$aptcache/etc/apt-move.conf # stores apt-move.conf
126			statefile=$aptcache/statefile # also used in install_packages.conf
127
128			# also used in install_pacakges.conf
129			export aptoptions=" \
130			-o Dir::State::status=$statefile \
131			-o Dir::Cache=$aptcache/var/cache/apt \
132			-o Dir::State=$aptcache/var/cache/apt \
133			-o Dir::Cache::Archives=$archivedir \
134			-o Dir::Etc::sourcelist=$aptcache/etc/apt/sources.list \
135			-o Dir::State::Lists=$aptcache/var/lib/apt/lists/"
136
137			# not needed
138			# -o APT::Get::ReInstall
139			# ."-o APT::Get::Download-Only=true -o Aptitude::Cmd-Line::Download-Only=true "
140
141			# we only need some empty dirs
142			mkdir -p $archivedir/partial $aptcache/etc/apt
143			mkdir -p $aptcache/var/lib/apt/lists/partial
144			> $statefile
145			}
146			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
147			delete_base_packages() {
148
149			# now delete all packages that are already included in base.tgz
150			local p
151
152			echo "Removing packages that are already included in base.tgz"
153			for p in $(cat $NFSROOT/var/tmp/base-pkgs.lis); do
154			if [ -f $archivedir/${p}_*.deb ]; then
155			[ $verbose -eq 1 ] && echo "deleting package $p"
156			rm $archivedir/${p}_*.deb
157			# else commands only for debugging
158			# else
159			# echo "package $p not found"
160			# ls $archivedir/${p}_*.deb
161			fi
162			done
163			}
164			# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
165
166			preserve=0
167			verbose=0
168	lange	2576	while getopts "vhx:p" opt ; do
169	lange	2566	case "$opt" in
170			h) usage ;;
171			x) exclasses="$OPTARG";;
172			p) preserve=1;;
173			v) verbose=1; vflag=-v;;
174			?) echo "Unknown option"; exit 1;;
175			esac
176			done
177			shift $(($OPTIND - 1))
178
179			debug=0
180			[ $debug -eq 0 ] && quiet=-q
181
182			mirrordir=$1
183	lange	2576	if [ -z "$mirrordir" ]; then
184			echo "Please give the directory for the mirror."
185			exit 2
186			fi
187	lange	2566	if [ -d $mirrordir/pool -o -d $mirrordir/dists ]; then
188			echo "Please first remove $mirrordir/pool and $mirrordir/dists"
189			exit 3
190			fi
191
192			initialize
193
194			# if we are using nfs mounts for Debian mirror, this may fail here, since inside a chroot environment different dir are used
195
196			# if sources.list includes file AND FAI_DEBMIRROR is defined we have to mount
197			# otherwise mounting is not needed. call task_mirror
198
199			[ "$FAI_DEBMIRROR" ] && mount -r $FAI_DEBMIRROR $FAI_ROOT/$MNTPOINT
200			# TODO: use -p to preserve sources.list
201			perl -p -e 's/file:/copy:/' /etc/fai/sources.list > $aptcache/etc/apt/sources.list
202
203			echo "Getting package information"
204			apt-get $quiet $aptoptions update >/dev/null
205
206			# all available file names are classes
207			classes=$(cd $FAI_CONFIGDIR/package_config; ls -1 \| egrep "^[A-Z0-9_]+$")
208			addclasses=$(grep -h PACKAGES $FAI_CONFIGDIR/package_config/* \| awk '{print $3}')
209			export classes=$(echo -e "$classes\n$addclasses\n" \| sort \| uniq)
210			[ -n "$exclasses" ] && excludeclass $exclasses
211
212			echo "Downloading packages for classes:" $classes
213			FAI=$FAI_CONFIGDIR install_packages -i $vflag -f /etc/fai
214			get_addpackages
215			umount_dirs
216	lange	2576	trap "" EXIT ERR
217	lange	2566	delete_base_packages
218
219			# create mirror directory structure
220			echo "Calling apt-move"
221			cat > $aptmovefile <<EOF # generate apt-move.conf
222			APTSITES=*
223			LOCALDIR=$mirrordir
224			DIST=sarge
225			FILECACHE=$archivedir
226			LISTSTATE=$aptcache/var/lib/apt/lists
227			DELETE=no
228			CONTENTS=no
229			EOF
230			apt-move $quiet -c $aptmovefile update
231			echo "$0 finished."
232			echo -n "Mirror size and location: ";du -sh $mirrordir
233			cleandirs
Name	Value
svn:eol-style	native
svn:executable	*
svn:keywords	Author Date Id Revision