New shell script that will create deltas for all consecutive

revisions of all databases on ftp.ensembl.org.  Uses the files
in /pub/<species>/data/mysql/<db>/*

misc-scripts/binary_delta/autorunner.ksh

+#/bin/ksh
+
+# $Id$
+#
+# Creates delta files between all consecutive revisions of all
+# databases on ftp.ensembl.org using build.pl and apply.pl.
+#
+# Author:  Andreas Kahari <andreas.kahari@ebi.ac.uk>
+#
+
+# Gets a database from the FTP site, puts it in ./databases/
+function getdb
+{
+    typeset path=$1
+    typeset db=$2
+    typeset ver=$3
+
+    typeset url='ftp://ftp.ensembl.org'${path}/${db}_${ver}.tar
+
+    if [[ ! -d databases ]]; then
+	mkdir databases
+    fi
+    if [[ ! -d databases/${db}_${ver} ]]; then
+	lynx -source $url | tar xvf - -C databases
+    fi
+}
+
+# Builds delta files, and applies them for verification
+function do_delta
+{
+    typeset db=$1
+    typeset v1=$2
+    typeset v2=$3
+
+    if [[ ! -d deltas ]]; then
+	mkdir deltas
+    fi
+
+    typeset build_out=deltas/${db}_${v1}_delta_${v2}_build.out
+    typeset apply_out=deltas/${db}_${v1}_delta_${v2}_apply.out
+
+    if [[ ! -f $build_out ]]; then
+	/usr/bin/time perl -w ./build.pl -c ./xdelta.osf \
+	    -s databases -d deltas \
+	    $db $v1 $v2 1>&2 | tee $build_out
+    fi
+    if [[ ! -f $apply_out ]]; then
+	/usr/bin/time perl -w ./apply.pl -c ./xdelta.osf \
+	    -d databases -s deltas \
+	    $db $v1 $v2 1>&2 | tee $apply_out
+    fi
+}
+
+# Removes a database that was fetched
+function cleandb
+{
+    typeset db=$1
+    typeset ver=$2
+
+    if [[ -d databases && -n $db && -n $ver ]]; then
+	rm -rf databases/${db}_${ver}*
+    fi
+}
+
+# ---------------  Main
+
+# For debugging
+typeset -ft getdb
+typeset -ft do_delta
+typeset -ft cleandb
+
+version_re='[0-9][0-9]*_[0-9][0-9]*'
+
+# Use ftp://ftp.ensembl.org/ls-lR.Z to figure out what files are
+# available
+lynx -source ftp://ftp.ensembl.org/ls-lR.Z | \
+    sed -n 's/^\.\(.*data\/mysql\)\/\(.*\)_\('"$version_re"'\):$/\1 \2 \3/p' | \
+    sort -k2 |&
+
+while read path db ver; do
+    if [[ $db != $this_db ]]; then
+	cleandb $this_db $ver
+	this_db=$db
+	old_ver=$ver
+	old_path=$path
+	continue
+    fi
+
+    getdb $old_path $this_db $old_ver
+    getdb $path $this_db $ver
+
+    do_delta $this_db $old_ver $ver
+
+    cleandb $this_db $old_ver
+
+    old_ver=$ver
+    old_path=$path
+done <&p
+
+cleandb $this_db $ver