new archlinux.sh and less verbose logs

archlinux-old.sh

@@ -0,0 +1,25 @@
+#/bin/bash
+
+fatal() {
+    echo "$1"
+    exit 1
+}
+
+#RSYNCSOURCE=rsync://ftp5.gwdg.de/pub/linux/archlinux/
+#RSYNCSOURCE=rsync://mirror.nl.leaseweb.net/archlinux/
+#RSYNCSOURCE=rsync://mirror.one.com/archlinux/
+#RSYNCSOURCE=rsync://mirror.neuf.no/archlinux/
+#RSYNCSOURCE=rsync://mirror.f4st.host/archlinux/
+#RSYNCSOURCE=rsync://ftp.nluug.nl/archlinux/
+RSYNCSOURCE=rsync://fooo.biz/archlinux/
+
+# Define where you want the mirror-data to be on your mirror
+BASEDIR=/store/mirror/archlinux
+
+rsync --verbose --recursive --times --links --hard-links \
+  --stats --delete-after --ignore-errors \
+  ${RSYNCSOURCE} ${BASEDIR} &>> /opt/scripts/archlinux.log || fatal "Failed to rsync from ${RSYNCSOURCE}."
+
+date &>> /opt/scripts/archlinux.log
+echo "############################" &>> /opt/scripts/archlinux.log
+

archlinux.sh

@@ -1,23 +1,41 @@
-#/bin/bash
+#!/bin/bash
 
 fatal() {
     echo "$1"
     exit 1
 }
 
-#RSYNCSOURCE=rsync://ftp5.gwdg.de/pub/linux/archlinux/
-#RSYNCSOURCE=rsync://mirror.nl.leaseweb.net/archlinux/
-#RSYNCSOURCE=rsync://mirror.one.com/archlinux/
-#RSYNCSOURCE=rsync://mirror.neuf.no/archlinux/
-RSYNCSOURCE=rsync://mirror.f4st.host/archlinux/
+home="/store"
+target="${home}/mirror/archlinux"
+tmp="${home}/tmp"
+lock='/tmp/mirrorsync.lck'
+bwlimit=4096
+# NOTE: you very likely need to change this since rsync.archlinux.org requires you to be a tier 1 mirror
+source='rsync://rsync.archlinux.org/ftp_tier1'
+lastupdate_url="http://rsync.archlinux.org/lastupdate"
 
-# Define where you want the mirror-data to be on your mirror
-BASEDIR=/store/mirror/archlinux
+[ ! -d "${target}" ] && mkdir -p "${target}"
+[ ! -d "${tmp}" ] && mkdir -p "${tmp}"
 
-rsync --verbose --recursive --times --links --hard-links \
-  --stats --delete-after --ignore-errors \
-  ${RSYNCSOURCE} ${BASEDIR} &>> /opt/scripts/archlinux.log || fatal "Failed to rsync from ${RSYNCSOURCE}."
+exec 9>"${lock}"
+flock -n 9 || exit
 
-date &>> /opt/scripts/archlinux.log
-echo "############################" &>> /opt/scripts/archlinux.log
+# if we are called without a tty (cronjob) only run when there are changes
+if ! tty -s && diff -b <(curl -s "$lastupdate_url") "$target/lastupdate" >/dev/null; then
+	echo "Sync was not needed, exiting. $(date)" >> /opt/scripts/archlinux.log
+	exit 0
+fi
+
+if ! stty &>/dev/null; then
+    QUIET="-q"
+fi
+
+rsync -rtlvH --safe-links --delete-after --progress -h ${QUIET} --timeout=600 --contimeout=60 -p \
+	--delay-updates --no-motd --bwlimit=$bwlimit \
+	--temp-dir="${tmp}" \
+	--exclude='*.links.tar.gz*' \
+	${source} \
+	"${target}" &>> /opt/scripts/archlinux.log || fatal "Failed to sync, se /opt/scripts/archlinux.log for more info." 
+
+echo "Last sync was $(date -d @$(cat ${target}/lastsync))" >> /opt/scripts/archlinux.log
 

ubuntu_archive_two_stage.sh

@@ -24,17 +24,17 @@ if [ ! -d ${BASEDIR} ]; then
   mkdir -p ${BASEDIR} || fatal "Creation of ${BASEDIR} failed."
 fi
 
-rsync -4 --verbose --recursive --times --links --hard-links \
+rsync -4 -q --recursive --times --links --hard-links \
   --stats --chmod=a+rx \
   --exclude "Packages*" --exclude "Sources*" \
   --exclude "Release*" \
   ${RSYNCSOURCE} ${BASEDIR} &>> /opt/scripts/ubuntu_archive_two_stage.log || fatal "First stage of sync failed."
 
-rsync -4 --verbose --recursive --times --links --hard-links \
+rsync -4 -q --recursive --times --links --hard-links \
   --stats --delete --chmod=a+rx --delete-after \
   ${RSYNCSOURCE} ${BASEDIR} &>> /opt/scripts/ubuntu_archive_two_stage.log || fatal "Second stage of sync failed."
 
 date -u > ${BASEDIR}/project/trace/$(hostname -f)
 
 date &>> /opt/scripts/ubuntu_archive_two_stage.log
-echo "############################" &>> /opt/scripts/ubuntu_archive_two_stage.log
+

ubuntu_releases.sh

@@ -24,11 +24,10 @@ if [ ! -d ${BASEDIR} ]; then
   mkdir -p ${BASEDIR} || fatal "Creation of ${BASEDIR} failed."
 fi
 
-rsync --verbose --recursive --times --links --hard-links \
+rsync -q --recursive --times --links --hard-links \
   --stats --delete-after \
   ${RSYNCSOURCE} ${BASEDIR} &>> /opt/scripts/ubuntu_releases.log || fatal "Failed to rsync from ${RSYNCSOURCE}."
 
 date -u > ${BASEDIR}/.trace/$(hostname -f)
 
 date &>> /opt/scripts/ubuntu_releases.log
-echo "############################" &>> /opt/scripts/ubuntu_releases.log