#!/bin/bash

# Mirroring the SuSE distribution and security updates
# See below, destination directory, for what is mirrored.
# Run this as a cron job every day.  Suggested to skip Saturday, so it 
# will run on Sunday and Wednesday AM every week.

# Options:
#   -f		Force it to run whether or not it's the correct day.
#   -A		Download all releases and architectures whether or not any 
#		machines are in the respective hostgroups.  

# See the end of the script for the URL(s) to download from.

# This is the CouchNet version.  Mathnet is different.  

while [ "X$1" != "X" ] ; do
    case "X$1" in
	X-[fA] )	eval "opt_${1:1:1}=1" ; ;;
	* )		echo "Unrecognized option '$1'"
			exit 4
			;;
    esac
    shift
done

# Number of days between mirror runs (minus 1, i.e. set to 2, run on Monday,
# it won't run on Tues or Wed, will run on Thurs.)  Beware of race condition:
# if the script starts at 22:01 and finishes at 22:15, fine, but if it finishes
# at 22:01, the count of days may or may not be off by 1.
# Remove the log file to force a run to occur.
days=2

# For each arch in $allarchs, if `hostgroup -a ${arch}@${version}` is 
# nonempty then we want that arch (otherwise it's excluded).  If we
# get a new architecture it must be added to this list.  noarch
# is added as a special case.  Source RPMs are not included.
# (The program creates empty directories for the unwanted architectures;
# too hard to suppress this behavior.)
allarchs="i686 i586 x86_64"

# Destination directory.  update/ should be an immediate subdir of $dest.
# Scripts are also in $dest.  The subdirectories that are brought in are:
# $dest/update/$VERS (where $VERS refers to all numbered subdirs that look like
# version numbers).  When a new version comes out, do this:
#   mkdir $dest/update/$VERS 
#   chown $user $dest/update/$VERS		# user=wwwrun
#   chmod 755 $dest/update/$VERS
# When you abandon an old version, just delete the update and install dirs.
for f in /h1/www/htdocs/SuSE/. /home/httpd/htdocs/SuSE/. ; do
    if [ -d $f ] ; then
	dest=$f
	break
    fi
done
cd $dest || exit 4

# User who is supposed to own the files.
user=wwwrun

umask 022

ulimit -f unlimited

shopt -s nullglob

etc=root
for f in /usr/math/default/path.sh /usr/diklo/default/path.sh ; do
    if [ -r $f ] ; then
	. $f
	break
    fi
done

# Name of log file
logdir=/tmp/system
logfil=rsyncsuse.log
fil=$logdir/$logfil
scr=$logdir/rsyncsuse.tmp

# Check if it's time to do the mirror run.  Exit if not.
if [ -n "$opt_f" -o -z "`find $logdir -name $logfil -mtime -$days -print`" ] ; then
    date > $fil.st
    echo "    Starting mirror run from $url" >> $fil.st
else
    date >> $fil.st
    echo "    Mirror is <= $days days old, skipping" >> $fil.st
    exit 0
fi
exec > $fil 2>&1

# Emits on stdout the wanted architectures, given a set of possible
# architectures as args. $1 should be the OS version, e.g. '10.1'.  Output is
# not sorted, usually contains duplicates.
function wantarch () {
    local f
    local t
    local vers=$1
    shift
    for f in $* ; do
		# Example: hostgroup x86_64@v10.3-rogue (@ means "and")
		# Formerly it said `hostgroup -a` (include machines that are
		# down); now it doesn't.  i*86 is a special case, if you have
		# either you need both i586 and i686.  -A option causes all
		# architectures in all versions to be wanted unconditionally. 
	t=`hostgroup "${f}@v${vers}-rogue" | wc -w`
	if [ "$t" -gt 0 -o -n "$opt_A" ] ; then
	    echo $f
	    case $f in
		i686 )
		    echo i586
		    ;;
		i586 )
		    echo i686
		    ;;
	    esac
	fi
    done
}

# Gets directories from a repository.
#   $1		URL to download from
#   $2		Destination directory relative to $dest (= $di/SuSE), normally
#		just "update"
#   $3 ...	List of versions to download.  If absent (recommended), all 
#		the subdirs of $2 that begin with a digit will be used, e.g.
#		11.4 or 12.2rc1 .
# The source URL will be $1/$3/ and the destination will be $dest/$2/$3/ .
function getdir () {
    local include
    local url=$1
    shift
    local updir=$1
    shift
    local verss=($*)
    if [ ${#verss[*]} -le 0 ] ; then
	verss=(`cd $dest/$updir && echo [0-9]*`)
    fi
    for vers in ${verss[*]} ; do
	local destdir=$dest/$updir/$vers
                # Suppress unwanted files.  Retain wanted archs.  In
		# --include/--exclude, earlier parameters override later ones.
	archs="`wantarch $vers $allarchs | sort -r -u | tr '\012' ' '`"
	if [ -z "$archs" ] ; then
	    echo "OS version $vers skipped due to lack of interest (no hosts with this version)."
	    continue
	fi

	archs="noarch $archs"
	include="--exclude *.rpm --exclude *.drpm --exclude *.spm --exclude *.info"
	for f in $archs ; do
	    include="--include *.$f.rpm $include"
	done
    
	cat - <<EOF

`date` -- Transferring 
    $url/$vers architectures $archs 
    to $destdir

EOF

                # Note: Mathnet default shell sets a filesize limit of 20 MB,
		# but if you specify /bin/sh the limit of the parent process is
		# used, i.e. unlimited. Can't use "rsync -a" because we don't
		# want to set the user and group.  Nor permissions (should be
		# 644 or 755 per umask).  -a is a synonym for -rlptgoD ; will
		# use -rlt (recurse, symlinks, set times).
	cmdline="umask 022 ; \
	    rsync -rlt --keep-dirlinks --log-format='%o %f' $include \
		    $url/$vers/ $destdir/"
	echo "DEBUG command line: $cmdline"
	export SHELL=/bin/sh
                # The server sometimes has too many users.  Retry with
		# exponential backoff.  Times in secs.  dt = half the initial
		# wait (10 mins).  dtmax can be interpreted in 2 ways: give up
		# if next wait or total wait so far exceeds this value (4
		# hours).
	dt=300
	dtmax=14400
	while /bin/true ; do
		# Actually executing the command line
	    su $user -s /bin/sh -c "$cmdline"
	    rc=$?
	    if [ $rc -eq 0 ] ; then break ; fi
	    echo "`date` -- rsync exited with code $rc"
	    dt=`expr $dt + $dt`
	    if [ $dt -gt $dtmax ] ; then
		echo "Giving up."
		break
	    fi
	    echo "Waiting $dt secs before retrying"
	    sleep $dt
	done
		# Don't need to keep index files from the previous batch. 
		# Retain only files mentioned in repomd.xml.
	export LC_ALL=C
	export LC_COLLATE=C
	(cd $destdir/repodata && ls -1 repomd.* ) > $scr
	sed -e '/repodata\//!d' -e 's/.*repodata\///' -e 's/".*//' $destdir/repodata/repomd.xml >> $scr
	sort -o $scr $scr
	if [ -s $scr ] ; then
	    (cd $destdir/repodata && ls -1 | sort | comm -23 - $scr | xargs -r -n 25 rm)
	fi
	
		# Rebuild the package index.  On versions that use zypper this
		# is no longer necessary and there is no Makefile.  
	if [ -r $destdir/rpm/Makefile ] ; then
	    (cd $destdir/rpm && make)
	fi
	
    done	# End of loop over versions
}

# To download the updates, insert one or more commands of the form
#   getdir $URL update 
#   getdir $URL update version...
# See function getdir for details.  The arguments have changed through the 
# years to accomodate changes in the directory structure.  

# OpenSuSE mirror site list: http://mirrors.opensuse.org/

# 2006-06-21	For reference, this is the URL on the SuSE server for v10.1.
# url="ftp://ftp.suse.com/pub/suse"

# 2004-06-01	USC stopped getting updates
# 2006-06-21	They're getting updates now.
# url="rsync://mirrors.usc.edu/suse"

# 2004-06-01	Argonne became unreachable so we switched to USC.
# 2004-07-09	Argonne has recovered; switching back.
# 2006-06-21	Server is in a strange state; trying USC again.
# 2006-07-26	Argonne has recovered; switching back.
# 2007-11-01	Neither Argonne nor USC are mirroring v10.3,
#		but the SuSE site has been located.  
#		rsync://download.opensuse.org/opensuse-updates
#		But only official mirrors have rsync access.  For info see
#		http://en.opensuse.org/Mirror_Infrastructure 
# 2007-11-07	rsync://mirrors.kernel.org/opensuse has the 10.3 stuff.
#		And also v10.1, 2.
# 2008-01-02	Argonne can't do the chroot to send out rsync.  Switch to USC.
# 2010-02-09	Switched over to SuSE 11.2, omit patches for older versions.

#getdir rsync://mirror.anl.gov/suse update/10.{1,2}
#getdir rsync://mirrors.usc.edu/suse update/10.{1,2} (obsolete versions)
#getdir rsync://mirrors.kernel.org/suse update/10.{1,2}

# getdir rsync://mirrors.kernel.org/opensuse update/11.2
# 2011-02-02	mirrors.kernel.org max clients: 100
#		mirror.anl.gov max clients: 17
#		mirrors.usc.edu seems to be out of business.
# 2011-02-02	We've been using mirrors.kernel.org for a long time, but
#		suddenly their server is wacked.  Argonne is too busy. 

#getdir rsync://opensuse.cs.utah.edu/opensuse update

# 2013-08-06	opensuse.cs.utah.edu is out of action (temporary? permanent?)
#		All the other mirrors above answer ping.  
#		mirrors.kernel.org is too busy to give us service.
#		Argonne is very busy and doesn't have suse/update/12.3
#		USC...  Last update 2008, service not available.
# 2013-08-13	Utah is down again.  Trying a new mirror: Linux Foundation, 
#		Portland, Oregon, evidently hosted on mobile-central.org.  
#getdir rsync://suse.mobile-central.org/opensuse/ update
#		*** Utah's problem was that my firewall blocked port 873. ***
#		Reverting to Utah.  

# 2015-12-01	For "leap-42.1" they've added a subdir "leap" under "update".
#		Have to change getdir, then download 13.1 and 42.1 separately.  
getdir rsync://opensuse.cs.utah.edu/opensuse/update update 13.1
getdir rsync://opensuse.cs.utah.edu/opensuse/update/leap update 42.1

cat - <<EOF

`date` -- Transfer finished

EOF