#!/bin/bash # # rsync-fast: syncs remote MIR site with local copy, fast version # for copyright notice, see the file "production" # CONFIG="$1" BASENAME=`basename $0` function usage { echo "usage: $BASENAME config-file" exit 1 } if [ -z "$1" ]; then usage elif [ ! -d "$1" ]; then $BASENAME: config file not found: $1 fi THIS_MONTH="`date +%Y\/%m`" LAST_MONTH="`date +%Y\/%m -d "1 month ago"`" LAST_YEAR="`date +%Y -d "1 year ago"`" THIS_YEAR="`date +%Y`" EXCLUDES="" INCLUDES="" function count_chars { echo $1 | sed -e 's/\(.\)/\n\1/g' | grep $2 | wc -l } function decho { echo "" echo "####################################################################################" echo $* echo "####################################################################################" echo "" } function eval_param { local param if [ -z "$1" ]; then return fi if [ ! -f "$CONFIG" ]; then decho "Fatal: config file $CONFIG does not exist." exit 1 else param="`grep -e "^$1=" $CONFIG | cut -d = -f 2 | sed -e 's/"//g' -e "s/'//g" | cut -d "#" -f 1`" if [ ! -z "$param" ]; then echo $param else if [ "$2" == "--required" ]; then echo "$BASENAME: fatal: config setting $1 not defined in $CONFIG, aborting" exit 1 else shift echo $* fi fi fi } function eval_config { SITE="`eval_param SITE --required`" SOURCEBASE="`eval_param SOURCEBASE --required`" RUN="`eval_param RUN --required`" LOCKFILE="`eval_param LOCKFILE $RUN/rsynclock-$SITE-fast`" STYLES="`eval_param STYLES --required`" LANGS="`eval_param LANGS --required`" INITIAL_YEAR="`eval_param FIRST_YEAR --required`" HOSTS="`eval_param HOSTS --required`" RSYNC="`eval_param RSYNC nice -n 19 /usr/bin/rsync --timeout=120 -Cazv --temp-dir=~/tmp`" RSYNCMEDIA="`eval_param RSYNCMEDIA nice -n 19 /usr/bin/rsync --timeout=120 -Cav --size-only --temp-dir=~/tmp`" } function sync_fast { for k in $LANGS; do for i in $STYLES; do INCLUDES="${INCLUDES} --include=/$k/$i/${THIS_MONTH}/ --include=/$k/$i/${LAST_MONTH}/" for ((j = $INITIAL_YEAR; j <= $LAST_YEAR; j++)); do EXCLUDES="${EXCLUDES} --exclude=/$k/$i/$j/*/" done done done for ((j = $INITIAL_YEAR; j <= $LAST_YEAR; j++)); do EXCLUDES="${EXCLUDES} --exclude=/content/$j/*/" done # everything but the media decho "STARTING AT: $(date)" using $DESTINATION_BASE decho everything except the optimized stuff $RSYNC -e "ssh -2 -c blowfish $PORT" --exclude="/icon" --exclude="/images" --exclude="/media" --exclude="/rtsp" --exclude="robots.txt" ${INCLUDES} \ --include="/content/${THIS_MONTH}/" --include="/content/${LAST_MONTH}/" ${EXCLUDES} ${SOURCE_BASE}/ ${DESTINATION_BASE}/ # all media: only the latest month: echo " " echo "media" echo " " for media in images icon rtsp media; do $RSYNCMEDIA -e "ssh -2 -c blowfish $PORT" --size-only ${SOURCE_BASE}/$media/${THIS_MONTH}/ ${DESTINATION_BASE}/$media/${THIS_MONTH}/ done decho "FINISHING AT `date`" } eval_config mkdir -p $RUN if [ -f $LOCKFILE ]; then echo Locked! exit 1 else touch $LOCKFILE || echo "Fatal: could not write $LOCKFILE." fi for DESTINATION_BASE in $HOSTS; do # HOST definition: # USER@HOST:PORT/FOLDER # check if destination is fine if [ "`cont_chars $DESTINATION_BASE @`" != "1" ] || \ [ `cont_chars $DESTINATION_BASE :`!= 1 ]; then echo $BASENAME: error: please check your $HOSTS definitions at $CONFIG echo $BASENAME: failed at $DESTINATION_BASE exit 1 fi # check if host definition use a non-standard ssh port if echo $DESTINATION_BASE | grep -q -e ":[[:digit:]]"; then PORT="`echo $DESTINATION_BASE | cut -d : -f 2 | cut -d / -f 1`" DESTINATION_BASE="`echo $DESTINATION_BASE | sed -e "s/:$PORT\//:\//"`" PORT="-p $PORT" else PORT="-p 22" fi sync_fast done rm -f $LOCKFILE