notorious

#!/usr/bin/env bash
################################################################################
# notorious - scp for big quantities of small files. This script automates the
# process of tarring up a directory of small files, transferring them, and then
# untarring them on the other side. This can improve performance when
# transferring tons of tiny, tiny files.
#
# Version 1
# Matthew Malensek <matt@malensek.net>
################################################################################

progress_size=25

print_usage() {
cat <<EOM
Usage: $(basename ${0}) source dest-host:dest-dir
EOM
}

progress() {
    perc=$(awk "BEGIN { print int(${1} / ${2} * 100) } ")
    bars=$(( perc / (100 / progress_size) ))
    echo -ne '\r['
    for (( i=0; i < ${progress_size}; ++i, --bars )); do
        if [[ ${bars} -gt 0 ]]; then echo -n '='; else echo -n ' '; fi
    done
    echo -n "]  ${1}/${2} (${perc}%) "
}

if [[ ${#} -ne 2 ]]; then
    print_usage
    exit 1
fi

source=${1}
read host dest <<< $(IFS=":"; echo ${2})

num_files=$(ls -1 "${source}" | wc -l)
num_finished=0
timer=0

tar -c ${source} | ssh -c arcfour "${host}" "tar -C '${dest}' -vxf -" 2>&1 | \
    while read; do
        (( ++num_finished ))
        if [[ ${SECONDS} -gt ${timer} ]]; then
            timer=${SECONDS}
            progress ${num_finished} ${num_files} &
        fi
    done