#!/bin/bash

# Unofficial Bash Strict Mode (http://redsymbol.net/articles/unofficial-bash-strict-mode/)
set -euo pipefail
IFS=$'\n\t'


AWS_BUCKET="datasets.pacb.com"
AWS_BASEPATH="2014/Arabidopsis/raw/2420292"


function list_files()
{
    # retrieved from http://datasets.pacb.com.s3.amazonaws.com/2014/Arabidopsis/raw/list.html
    echo 0001/Analysis_Results/m131019_072530_42175_c100583702550000001823087704281410_s1_p0.{1,2,3}.bax.h5
    echo 0002/Analysis_Results/m131019_104730_42175_c100583702550000001823087704281411_s1_p0.{1,2,3}.bax.h5
    echo 0003/Analysis_Results/m131019_141039_42175_c100583702550000001823087704281412_s1_p0.{1,2,3}.bax.h5
    echo 0004/Analysis_Results/m131019_205608_42175_c100583702550000001823087704281414_s1_p0.{1,2,3}.bax.h5
    echo 0005/Analysis_Results/m131019_173349_42175_c100583702550000001823087704281413_s1_p0.{1,2,3}.bax.h5
    echo 0006/Analysis_Results/m131020_001913_42175_c100583702550000001823087704281415_s1_p0.{1,2,3}.bax.h5
    echo 0007/Analysis_Results/m131020_065129_42175_c100583802550000001823087704281480_s1_p0.{1,2,3}.bax.h5
    echo 0008/Analysis_Results/m131020_111406_42175_c100583802550000001823087704281481_s1_p0.{1,2,3}.bax.h5
    echo 0009/Analysis_Results/m131020_153743_42175_c100583802550000001823087704281482_s1_p0.{1,2,3}.bax.h5
    echo 0010/Analysis_Results/m131020_200125_42175_c100583802550000001823087704281483_s1_p0.{1,2,3}.bax.h5
    echo 0011/Analysis_Results/m131021_002338_42175_c100583802550000001823087704281484_s1_p0.{1,2,3}.bax.h5
    echo 0012/Analysis_Results/m131021_044647_42175_c100583802550000001823087704281485_s1_p0.{1,2,3}.bax.h5
    echo 0013/Analysis_Results/m131021_091236_42175_c100583802550000001823087704281486_s1_p0.{1,2,3}.bax.h5
    echo 0014/Analysis_Results/m131021_133552_42175_c100583802550000001823087704281487_s1_p0.{1,2,3}.bax.h5
    echo 0015/Analysis_Results/m131023_015910_42175_c100583712550000001823087704281400_s1_p0.{1,2,3}.bax.h5
    echo 0016/Analysis_Results/m131023_062359_42175_c100583712550000001823087704281401_s1_p0.{1,2,3}.bax.h5
    echo 0017/Analysis_Results/m131023_105030_42175_c100583712550000001823087704281402_s1_p0.{1,2,3}.bax.h5
    echo 0018/Analysis_Results/m131023_151704_42175_c100583712550000001823087704281403_s1_p0.{1,2,3}.bax.h5
    echo 0019/Analysis_Results/m131023_194332_42175_c100583712550000001823087704281404_s1_p0.{1,2,3}.bax.h5
    echo 0020/Analysis_Results/m131024_001007_42175_c100583712550000001823087704281405_s1_p0.{1,2,3}.bax.h5
    echo 0021/Analysis_Results/m131024_043828_42175_c100583712550000001823087704281406_s1_p0.{1,2,3}.bax.h5
    echo 0022/Analysis_Results/m131024_090506_42175_c100583712550000001823087704281407_s1_p0.{1,2,3}.bax.h5
    echo 0023/Analysis_Results/m131024_162402_42175_c100583982550000001823087704281470_s1_p0.{1,2,3}.bax.h5
    echo 0024/Analysis_Results/m131024_204857_42175_c100583982550000001823087704281471_s1_p0.{1,2,3}.bax.h5
    echo 0025/Analysis_Results/m131025_011527_42175_c100583982550000001823087704281472_s1_p0.{1,2,3}.bax.h5
    echo 0026/Analysis_Results/m131025_054213_42175_c100583982550000001823087704281473_s1_p0.{1,2,3}.bax.h5
    echo 0027/Analysis_Results/m131025_100934_42175_c100583982550000001823087704281474_s1_p0.{1,2,3}.bax.h5
    echo 0028/Analysis_Results/m131025_143613_42175_c100583982550000001823087704281475_s1_p0.{1,2,3}.bax.h5
    echo 0029/Analysis_Results/m131025_190453_42175_c100583982550000001823087704281476_s1_p0.{1,2,3}.bax.h5
    echo 0030/Analysis_Results/m131025_233209_42175_c100583982550000001823087704281477_s1_p0.{1,2,3}.bax.h5
    echo 0031/Analysis_Results/m131026_062217_42175_c100583752550000001823087704281460_s1_p0.{1,2,3}.bax.h5
    echo 0032/Analysis_Results/m131026_104626_42175_c100583752550000001823087704281461_s1_p0.{1,2,3}.bax.h5
    echo 0033/Analysis_Results/m131026_151302_42175_c100583752550000001823087704281462_s1_p0.{1,2,3}.bax.h5
    echo 0034/Analysis_Results/m131026_193952_42175_c100583752550000001823087704281463_s1_p0.{1,2,3}.bax.h5
    echo 0035/Analysis_Results/m131027_000628_42175_c100583752550000001823087704281464_s1_p0.{1,2,3}.bax.h5
    echo 0036/Analysis_Results/m131027_043256_42175_c100583752550000001823087704281465_s1_p0.{1,2,3}.bax.h5
    echo 0037/Analysis_Results/m131027_090108_42175_c100583752550000001823087704281466_s1_p0.{1,2,3}.bax.h5
    echo 0038/Analysis_Results/m131027_132828_42175_c100583752550000001823087704281467_s1_p0.{1,2,3}.bax.h5
    echo 0039/Analysis_Results/m131027_200215_42175_c100583692550000001823087704281450_s1_p0.{1,2,3}.bax.h5
    echo 0040/Analysis_Results/m131028_002525_42175_c100583692550000001823087704281451_s1_p0.{1,2,3}.bax.h5
    echo 0041/Analysis_Results/m131028_045128_42175_c100583692550000001823087704281452_s1_p0.{1,2,3}.bax.h5
    echo 0042/Analysis_Results/m131028_091947_42175_c100583692550000001823087704281453_s1_p0.{1,2,3}.bax.h5
    echo 0043/Analysis_Results/m131028_134714_42175_c100583692550000001823087704281454_s1_p0.{1,2,3}.bax.h5
    echo 0044/Analysis_Results/m131028_181351_42175_c100583692550000001823087704281455_s1_p0.{1,2,3}.bax.h5
    echo 0045/Analysis_Results/m131028_224232_42175_c100583692550000001823087704281456_s1_p0.{1,2,3}.bax.h5
    echo 0046/Analysis_Results/m131029_030920_42175_c100583692550000001823087704281457_s1_p0.{1,2,3}.bax.h5
}


function get_urls_from_list()
{
    list_files | tr ' ' '\n' | while read FILENAME;
    do
        echo "s3://$AWS_BUCKET/$AWS_BASEPATH/$FILENAME"
    done
}


function get_state_of()
{
    local URL="$1"
    local FILE="$(get_filename_for_url "$URL")"

    if [[ -e "$FILE" ]];
    then
        echo 'skip' && return
    else
        echo 'download' && return
    fi

    return 1
}


function download_file()
{
    local URL="$1"
    local FILENAME="$2"

    aws s3 cp --no-sign-request "$URL" "$FILENAME" 1>&2
}


function log()
{
    echo "-- $@"
} >&2


function call_hook()
{
    if [[ "$1" == "--optional" ]];
    then
        local OPTIONAL=yes
        shift
    fi

    if { declare -F "$1" || alias "$1" || which "$1"; } &> /dev/null;
    then
        eval "$@"
    else
        if [[ "${OPTIONAL:-no}" == no ]];
        then
            error "missing definition for hook: $1"
            exit 1
        fi
    fi
}


function get_filename_for_url()
{
    local FILENAME="${1##*/}"
    FILENAME="${FILENAME%.1}"

    echo "$FILENAME"
}


function print_job_status_line()
{
    tail -c80 "$1" >&2
}


function init_job_control()
{
    export JOBS_FILE="$(mktemp .jobs-XXXXXX)"

    trap "kill -SIGTERM \$(job_pids)" err
    trap "rm -f .job-*.log &> /dev/null; rm -f $JOBS_FILE" exit
}


function track_job()
{
    local JOBID="$1"
    local PID="$2"
    local LOG="$3"

    echo -e "$JOBID\t$PID\t$LOG\trunning" >> "$JOBS_FILE"
}


function new_jobid()
{
    awk -F'\t' '{ if ($1 > jobid) jobid = $1 } END { print jobid + 1 }' "$JOBS_FILE"
}


function enqueue_download_job()
{
    wait_for_queue

    local URL="$1"
    local CONTINUE="$2"
    local JOBID="$(new_jobid)"
    local LOG=".job-$JOBID.log"
    local FILENAME="$(call_hook get_filename_for_url "$URL")"

    {
        call_hook download_file "$URL" "$FILENAME" "$CONTINUE" 2> "$LOG" || job_failed "$URL" "$LOG"
    } &

    track_job "$JOBID" "$!" "$LOG"
}


function do_download_job()
{
    local URL="$1"
    local CONTINUE="$2"
    local FILENAME="$(call_hook get_filename_for_url "$URL")"

    call_hook download_file "$URL" "$FILENAME" "$CONTINUE"
}


function log_job_status()
{
    local PID="$1"
    local LOG="$(get_logfile "$PID")"
    local JOBID="${LOG%.log}"
    JOBID="${JOBID##*-}"

    log "job $JOBID status:"
    call_hook print_job_status_line "$LOG" "$JOBID" "$PID" >&2
}


function is_job_running()
{
    [[ -e "/proc/$1" ]]
}


function check_job_states()
{
    local NEW_STATES="$( awk -F'\t' '
        BEGIN {
            OFS = "\t";
        }

        {
            process_exe = "/proc/"$2"/exe";

            if ( (getline dummy < process_exe) > 0 ) {
                $4 = "running";
                close(process_exe)
            } else {
                $4 = "stopped"
            }

            print
        }
    ' "$JOBS_FILE" )";

    echo "$NEW_STATES" > "$JOBS_FILE"
}


function num_jobs()
{
    if dryrun;
    then
        echo 0
    else
        check_job_states

        awk -F'\t' '
            ($4 == "running") {
                ++num_jobs;
            }

            END {
                print 0 + num_jobs
            }
        ' "$JOBS_FILE"
    fi
}


function job_pids()
{
    awk -F'\t' '
        ($4 == "running") {
            print $2;
        }
    ' "$JOBS_FILE"
}


function get_logfile()
{
    awk -F'\t' '
        ($2 == "'$1'") {
            print $3;
        }
    ' "$JOBS_FILE"
}


function wait_for_queue()
{
    if parallel;
    then
        while (( $(num_jobs) >= PARALLEL ));
        do
            for PID in $(job_pids);
            do
                log_job_status "$PID"
            done

            sleep $PARALLEL
        done
    fi
}


function wait_jobs()
{
    if parallel
    then
        while (( $(num_jobs) > 0 ));
        do
            for PID in $(job_pids);
            do
                log_job_status "$PID"
            done

            sleep $PARALLEL
        done
    fi
}


function job_failed()
{
    local URL="$1"
    local LOG="$2"
    local JOBID="${LOG%.log}"
    JOBID="${JOBID##*-}"
    local ERRLOG="error.$JOBID.log"

    mv "$LOG" "$ERRLOG"
    error "failed to download $URL; inspect $ERRLOG for details"
}


function download_list()
{
    local URL

    log "syncing list"

    call_hook get_urls_from_list | while read URL;
    do
        local FILE_STATE="$(call_hook get_state_of "$URL")"
        case "$FILE_STATE" in
            skip)
                log "skipping $URL"
                DOWNLOAD=no
                ;;
            resume)
                log "resuming $URL"
                CONTINUE=yes
                DOWNLOAD=yes
                ;;
            replace)
                log "removing corrupted file for $URL"
                dryrun || rm -f "$FILENAME"
                DOWNLOAD=yes
                ;;
            download)
                DOWNLOAD=yes
                ;;
            *)
                error "illegal state: $FILE_STATE"
        esac

        if [[ "${DOWNLOAD:-yes}" != "no" ]];
        then
            log "downloading $URL (job $(num_jobs))"
            if parallel;
            then
                dryrun || enqueue_download_job "$URL" "${CONTINUE:-no}"
            else
                dryrun || do_download_job "$URL" "${CONTINUE:-no}"
            fi
        fi
    done

    call_hook --optional on_list_downloaded
}


PROG="$0"


function error()
{
    echo "$PROG: error: $@"
} >&2


function usage()
{
    echo "USAGE:  $PROG [-hn] [-p<num>]"
} >&2


function parse_args()
{
    while getopts "hnp:" OPTION; do
        case "$OPTION" in
            p)
                PARALLEL="$OPTARG"

                if ! (( OPTARG ));
                then
                    error "-p must have an integer value"
                    exit 1
                elif ! (( OPTARG > 0 ));
                then
                    error "-p must be positive"
                    exit 1
                fi
                ;;
            n)
                DRYRUN='yes'
                ;;
            h)
                usage

                exit
                ;;
            *)
                error "unkown option $OPTION"
                usage

                exit 1
                ;;
        esac
    done
    shift $(($OPTIND - 1))
}


function parallel()
{
    [[ -v PARALLEL ]]
}


function dryrun()
{
    [[ -v DRYRUN ]]
}


function main()
{
    call_hook --optional before_parse_args
    parse_args "$@"

    if parallel;
    then
        init_job_control
    fi

    call_hook --optional before_all_downloads
    download_list

    wait_jobs
}


main "$@"
