#!/bin/bash

# Unofficial Bash Strict Mode (http://redsymbol.net/articles/unofficial-bash-strict-mode/)
set -euo pipefail
IFS=$'\n\t'


AWS_BUCKET="genomeark"
AWS_BASEPATH="species/Calypte_anna/bCalAnn1/genomic_data/pacbio"


function list_files()
{
    # retrieved by aws s3 --no-sign-request ls s3://genomeark/species/Calypte_anna/bCalAnn1/genomic_data/pacbio/
    echo m150407_133846_sherri_c100712082550000001823142104221563_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150408_041513_42142_c100709632550000001823139404221560_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150409_034551_42142_c100759822550000001823163407221540_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150410_022755_42142_c100787282550000001823152608251550_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150410_150844_42142_c100787282550000001823152608251552_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150411_034422_42142_c100787282550000001823152608251554_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150411_100402_42142_c100787282550000001823152608251555_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150411_162722_42142_c100787282550000001823152608251556_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150412_070933_42142_c100759822550000001823163407221545_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150413_234253_sherri_c100761362550000001823161607221560_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150413_234416_42141_c100761652550000001823161607221580_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150414_060429_sherri_c100761362550000001823161607221561_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150414_061217_42141_c100761652550000001823161607221581_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150414_122654_sherri_c100761362550000001823161607221562_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150414_124052_42141_c100761652550000001823161607221582_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150414_184922_sherri_c100761362550000001823161607221563_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150414_190937_42141_c100761652550000001823161607221583_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_005921_42142_c100759932550000001823166007221590_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_011152_sherri_c100761362550000001823161607221564_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_013720_42141_c100761652550000001823161607221584_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_073436_sherri_c100761362550000001823161607221565_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_080638_42141_c100761652550000001823161607221585_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_133721_42142_c100759932550000001823166007221592_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_140003_sherri_c100761362550000001823161607221566_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_143608_42141_c100761652550000001823161607221586_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_195645_42142_c100759932550000001823166007221593_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_202229_sherri_c100761362550000001823161607221567_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_210331_42141_c100761652550000001823161607221587_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150415_214500_42161_c100761482550000001823161607221510_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_021818_42142_c100759932550000001823166007221594_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_040402_42161_c100761482550000001823161607221511_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_044928_sherri_c100761382550000001823161607221540_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_053656_42141_c100761372550000001823161607221550_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_083729_42142_c100759932550000001823166007221595_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_102315_42161_c100761482550000001823161607221512_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_111217_sherri_c100761382550000001823161607221541_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_120511_42141_c100761372550000001823161607221551_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_145933_42142_c100759932550000001823166007221596_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_164229_42161_c100761482550000001823161607221513_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_211825_42142_c100759932550000001823166007221597_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_225827_sherri_c100777832550000001823161908051560_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_230141_42161_c100761482550000001823161607221514_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150416_230404_42141_c100761372550000001823161607221552_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_051917_sherri_c100777832550000001823161908051561_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_052054_42161_c100761482550000001823161607221515_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_053211_42141_c100761372550000001823161607221553_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_054245_42142_c100761432550000001823161607221560_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_114250_sherri_c100777832550000001823161908051562_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_114352_42161_c100761482550000001823161607221516_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_120115_42141_c100761372550000001823161607221554_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_120227_42142_c100761432550000001823161607221561_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_180258_42161_c100761482550000001823161607221517_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_180637_sherri_c100777832550000001823161908051563_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150417_182824_42141_c100761372550000001823161607221555_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_003106_sherri_c100777832550000001823161908051564_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_005600_42141_c100761372550000001823161607221556_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_022452_42161_c100761412550000001823161607221580_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_065436_sherri_c100777832550000001823161908051565_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_084155_42161_c100761412550000001823161607221581_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_132111_sherri_c100777832550000001823161908051566_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_133235_42141_c100761382550000001823161607221542_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_194427_sherri_c100777832550000001823161908051567_s1_p0.{1,2,3}.subreads.bam{,.pbi}
    echo m150418_200009_42141_c100761382550000001823161607221543_s1_p0.{1,2,3}.subreads.bam{,.pbi}
}


function get_urls_from_list()
{
    list_files | tr ' ' '\n' | while read FILENAME;
    do
        echo "s3://$AWS_BUCKET/$AWS_BASEPATH/$FILENAME"
    done
}


function get_state_of()
{
    local URL="$1"
    local FILE="$(get_filename_for_url "$URL")"

    if [[ -e "$FILE" ]];
    then
        echo 'skip' && return
    else
        echo 'download' && return
    fi

    return 1
}


function download_file()
{
    local URL="$1"
    local FILENAME="$2"

    aws s3 cp --no-sign-request "$URL" "$FILENAME" 1>&2
}


function log()
{
    echo "-- $@"
} >&2


function call_hook()
{
    if [[ "$1" == "--optional" ]];
    then
        local OPTIONAL=yes
        shift
    fi

    if { declare -F "$1" || alias "$1" || which "$1"; } &> /dev/null;
    then
        eval "$@"
    else
        if [[ "${OPTIONAL:-no}" == no ]];
        then
            error "missing definition for hook: $1"
            exit 1
        fi
    fi
}


function get_filename_for_url()
{
    local FILENAME="${1##*/}"
    FILENAME="${FILENAME%.1}"

    echo "$FILENAME"
}


function print_job_status_line()
{
    tail -c80 "$1" >&2
}


function init_job_control()
{
    export JOBS_FILE="$(mktemp .jobs-XXXXXX)"

    trap "kill -SIGTERM \$(job_pids)" err
    trap "rm -f .job-*.log &> /dev/null; rm -f $JOBS_FILE" exit
}


function track_job()
{
    local JOBID="$1"
    local PID="$2"
    local LOG="$3"

    echo -e "$JOBID\t$PID\t$LOG\trunning" >> "$JOBS_FILE"
}


function new_jobid()
{
    awk -F'\t' '{ if ($1 > jobid) jobid = $1 } END { print jobid + 1 }' "$JOBS_FILE"
}


function enqueue_download_job()
{
    wait_for_queue

    local URL="$1"
    local CONTINUE="$2"
    local JOBID="$(new_jobid)"
    local LOG=".job-$JOBID.log"
    local FILENAME="$(call_hook get_filename_for_url "$URL")"

    {
        call_hook download_file "$URL" "$FILENAME" "$CONTINUE" 2> "$LOG" || job_failed "$URL" "$LOG"
    } &

    track_job "$JOBID" "$!" "$LOG"
}


function do_download_job()
{
    local URL="$1"
    local CONTINUE="$2"
    local FILENAME="$(call_hook get_filename_for_url "$URL")"

    call_hook download_file "$URL" "$FILENAME" "$CONTINUE"
}


function log_job_status()
{
    local PID="$1"
    local LOG="$(get_logfile "$PID")"
    local JOBID="${LOG%.log}"
    JOBID="${JOBID##*-}"

    log "job $JOBID status:"
    call_hook print_job_status_line "$LOG" "$JOBID" "$PID" >&2
}


function is_job_running()
{
    [[ -e "/proc/$1" ]]
}


function check_job_states()
{
    local NEW_STATES="$( awk -F'\t' '
        BEGIN {
            OFS = "\t";
        }

        {
            process_exe = "/proc/"$2"/exe";

            if ( (getline dummy < process_exe) > 0 ) {
                $4 = "running";
                close(process_exe)
            } else {
                $4 = "stopped"
            }

            print
        }
    ' "$JOBS_FILE" )";

    echo "$NEW_STATES" > "$JOBS_FILE"
}


function num_jobs()
{
    if dryrun;
    then
        echo 0
    else
        check_job_states

        awk -F'\t' '
            ($4 == "running") {
                ++num_jobs;
            }

            END {
                print 0 + num_jobs
            }
        ' "$JOBS_FILE"
    fi
}


function job_pids()
{
    awk -F'\t' '
        ($4 == "running") {
            print $2;
        }
    ' "$JOBS_FILE"
}


function get_logfile()
{
    awk -F'\t' '
        ($2 == "'$1'") {
            print $3;
        }
    ' "$JOBS_FILE"
}


function wait_for_queue()
{
    if parallel;
    then
        while (( $(num_jobs) >= PARALLEL ));
        do
            for PID in $(job_pids);
            do
                log_job_status "$PID"
            done

            sleep $PARALLEL
        done
    fi
}


function wait_jobs()
{
    if parallel
    then
        while (( $(num_jobs) > 0 ));
        do
            for PID in $(job_pids);
            do
                log_job_status "$PID"
            done

            sleep $PARALLEL
        done
    fi
}


function job_failed()
{
    local URL="$1"
    local LOG="$2"
    local JOBID="${LOG%.log}"
    JOBID="${JOBID##*-}"
    local ERRLOG="error.$JOBID.log"

    mv "$LOG" "$ERRLOG"
    error "failed to download $URL; inspect $ERRLOG for details"
}


function download_list()
{
    local URL

    log "syncing list"

    call_hook get_urls_from_list | while read URL;
    do
        local FILE_STATE="$(call_hook get_state_of "$URL")"
        case "$FILE_STATE" in
            skip)
                log "skipping $URL"
                DOWNLOAD=no
                ;;
            resume)
                log "resuming $URL"
                CONTINUE=yes
                DOWNLOAD=yes
                ;;
            replace)
                log "removing corrupted file for $URL"
                dryrun || rm -f "$FILENAME"
                DOWNLOAD=yes
                ;;
            download)
                DOWNLOAD=yes
                ;;
            *)
                error "illegal state: $FILE_STATE"
        esac

        if [[ "${DOWNLOAD:-yes}" != "no" ]];
        then
            log "downloading $URL (job $(num_jobs))"
            if parallel;
            then
                dryrun || enqueue_download_job "$URL" "${CONTINUE:-no}"
            else
                dryrun || do_download_job "$URL" "${CONTINUE:-no}"
            fi
        fi
    done

    call_hook --optional on_list_downloaded
}


PROG="$0"


function error()
{
    echo "$PROG: error: $@"
} >&2


function usage()
{
    echo "USAGE:  $PROG [-hn] [-p<num>]"
} >&2


function parse_args()
{
    while getopts "hnp:" OPTION; do
        case "$OPTION" in
            p)
                PARALLEL="$OPTARG"

                if ! (( OPTARG ));
                then
                    error "-p must have an integer value"
                    exit 1
                elif ! (( OPTARG > 0 ));
                then
                    error "-p must be positive"
                    exit 1
                fi
                ;;
            n)
                DRYRUN='yes'
                ;;
            h)
                usage

                exit
                ;;
            *)
                error "unkown option $OPTION"
                usage

                exit 1
                ;;
        esac
    done
    shift $(($OPTIND - 1))
}


function parallel()
{
    [[ -v PARALLEL ]]
}


function dryrun()
{
    [[ -v DRYRUN ]]
}


function main()
{
    call_hook --optional before_parse_args
    parse_args "$@"

    if parallel;
    then
        init_job_control
    fi

    call_hook --optional before_all_downloads
    download_list

    wait_jobs
}


main "$@"
