#!/bin/bash

export LC_ALL="POSIX"


Usage() {
    echo ""
    echo "Usage: `basename $0` options"
    echo ""
    echo "   Options:"
    echo "      General:"
    echo "        -u <dir>      Directory, where the data will be generated/originates"
    echo "        -n <name>     Run name = main directory name"
    echo "        -j <int>      Number of jobs"
    echo "        -J            Restart the run (e.g. add more jobs via a new increased max -j)"
    echo "        -l <int>      Limit the number of running jobs to this number"
    echo "        -q <int>      Submit jobs with qsub with a certain delay between the jobs in sec"
    echo "        -g <name>     Geometry file name"
    echo "        -z            gzip output files"
    echo "        -a            Concatenate sim and tra files at the end"
    echo "        -y <int>      Type of queue master: 0 = torque/maui (default), 1 = sun grid engine, 2 = LSF" 
    echo "                      Without this argument the jobs are run locally on this machine" 
    echo "        -v <dir>      Temporary directory"
    echo "        -N <int>      Nice level of all jobs (default: 19)"
    echo "        -D            Delete low-level files (e.g. sim) after high level file (e.g. tra) have been generated"
    echo "        -h            Help"
    echo " "
    echo "      Simulation (MGGPOD or Cosima): needs -u, -n, -g"
    echo "        -s <int>      Do MGGGPOD simulations (0: ggod0 1: ggod1, 2: ggod2)"
    echo "                      If you want to do ggod2, do"
    echo "                          --> first ggod1 (-s 1) without conversion"
    echo "                          --> then start ggod2 (-s 2) normally (use -a here!)"
    echo "                      Do Cosima simulations:runs"
    echo "                          --> s -3: Standard cosima stage"
    echo "                          --> s -4: Activation calculation stage: Requires -d as the directory for the isotopes files!"
    echo "        -t <int>      ggod0 & ggod1:  Number of triggers per job (OTIM if negative double, keep what's in file if zero!)"
    echo "                      ggod2:          Observation time per job as DOUBLE (keep what's in file does not apply!)!"
    echo "                      cosima:         Ignored. Use what's in the source file"
    echo "        -w <double>   Irradiation time for ggod2 in years (if parameter is not given: 1.0 years)"
    echo "        -C <double>   Cool-down time in years (if not given no cool-down, constant exposure mode is assumed (i.e. intori instead of intorid))"
    echo "        -d <name>     Name of data directory - all content of the directory will be copied to the target directory"
    echo "        -F <name>     Name of the Cosima source file"
    echo "        -k            Do *NOT* recompile MGGPOD, do *NOT* recreate geometry files"
    echo "        -c            Run ConvertMGGPOD"
    echo " "
    echo "      Revan:          needs -u, -n, -g + -r, -f"
    echo "        -r            Run Revan"
    echo "        -f <name>     Use this revan *.cfg file"
    echo " "
    echo "      Response:       needs -u, -n, -g  +  -p, -f, -x"
    echo "        -p <char>     Generate response in mode <char>={t, c, il, ib, e}"
    echo "        -x <name>     Use this mimrec *.cfg file"
    echo " "
    echo "      NuSTAR:         needs -u, -n, -g + -S"
    echo "        -S <int>      Run NuSTAR compactor with <int> as background type ID"
    echo " "
    echo "      Rename:         needs -u & -n"
    echo "        -e \"<char>\"   \"<char>\" has the form \"<from> <to> <file selector>\""  
    echo "                      Replace in all names of files described <file selector> <from> with <to>"
    echo "                      Don't forget the \"\" -  the arguments itself are not allowed to have blanks!"
    echo "                      Example: -e \"CosmciPhotons CosmicPhotons *\""
    echo " "
    echo "      Merge rsp's:    needs -u & -n"
    echo "        -m            Merge rsp's. They need to exist!"
    echo " "
    echo "      Transfer:       needs -u & -n"
    echo "        -A <string>   Pack and transfer all data to a remote directory"
    echo "                      Password will be written and stored unencrypted!"
    echo "                      -i \"<username>@<machine>:<directory>|<password>\""
    echo "                      -i \"fred@feuerstein.de:~/steinbruch|wilma1963\""
    echo "                      -i \"steinbruch\" for a local directory"
    echo " "
    echo "      Clean temp:     needs -j, -q, -n & -v"
    echo "        -i            Clean the temporary directory"
    echo " "
    echo "      Special run:    needs -j, -q, -n & -u"
    echo "        -b \"<char>\"   Execute a special program"
    echo ""
}


jecho() {
    text="$1"
    echo "${text}"
    echo "${text}" 1>&2
    if [ `test ! -d $logfile` ]; then
        echo "${text}" >> ${logfile}
    fi
}


getrunningjobs() {
    runningjobs=0
    if ([ ${queuemaster} -eq 0 ]); then
        lines=`showq`
        for i in ${lines}; do
            if [ ${i} == ${USER} ]; then
                runningjobs=`expr $runningjobs + 1`
            fi
        done
    elif ([ ${queuemaster} -eq 1 ]); then
        lines=`qstat`
        for i in ${lines}; do
            if [ ${i} == ${USER} ]; then
                runningjobs=`expr ${runningjobs} + 1`
            fi
        done
    elif ([ ${queuemaster} -eq 2 ]); then
        lines=`bstat`
        for i in ${lines}; do
            if [ ${i} == ${USER} ]; then
                runningjobs=`expr ${runningjobs} + 1`
            fi
        done
    fi
}


getqueuedjobs() {
    queuedjobs=0
    if ([ ${queuemaster} -eq 0 ]); then
        Scan=`showq | awk -F" " '{ print $3" "$4 }'`
        Last="-"
        for t in ${Scan}; do 
            if ( [ "${Last}" == "Running" ] || [ "${Last}" == "Idle" ] || [ "${Last}" == "Deferred" ]); then 
                queuedjobs=`expr ${queuedjobs} + ${t}`; 
            fi; 
            Last=${t}; 
        done
    elif ([ ${queuemaster} -eq 1 ]); then
        Scan=`qstat | awk -F" " '{ print $9 }'`
        queuedjobs=-2
        for t in ${Scan}; do
            if (`echo "${t}" | grep -q -v "[^0-9]"`); then
                queuedjobs=`expr ${queuedjobs} + ${t}`; 
            fi
        done
    elif ([ ${queuemaster} -eq 2 ]); then
        Scan=`bstat | awk -F" " '{ print $5 }' | awk -F"*" '{ print $2 }'`
        queuedjobs=0
        for t in ${Scan}; do
            if (`echo "${t}" | grep -q -v "[^0-9]"`); then
                queuedjobs=`expr ${queuedjobs} + ${t}`; 
            fi
        done
    fi
}



NO_ARGS=0 
E_OPTERROR=65

if [ $# -eq "$NO_ARGS" ]  # Script invoked with no command-line args?
then
    Usage;
    exit 1;
fi  
# Usage: scriptname -options
# Note: dash (-) necessary

# General flags:
simversion=21 # Make sure it is identical in the pipeline_job script!

# Flags, which can be set via command line
rundir="."
actdirectory="actdirectory_none"
cosimasourcefile="NoCosimaSourceFileGiven"
name="name_none"
jobs=1
startjob=1
restart=0
maxjobs=32
maxqueued=100000
queuemaster=0
noout=0
retain=""
out=" "

submit=0
delay=1

sim=-1
norecompile=0
triggers=0

convert=0
geometry="geometry_none"

revan=0
revancfg="none"
zip=0

nustar=0
nustarbackgroundid=0

concat=0
response="-"
mimreccfg="none"

rename=0
renamefrom=""
renameto=""
renamesuffix=""

mergersps=0
transfer=0

temporary=0
temporarydir="acz"

cleantemp=0

special=0
specialprogram="acz"

irradiationtime=1.0
usecooldown=0
cooldowntime=-1.0

nicelevel=19
deletelowlevel=0

echo "Selected pipeline options:"
while getopts "hd:F:u:n:j:Jq:t:g:crs:aA:zp:ke:v:mf:x:oib:l:y:w:C:N:S:D" flag
do
case $flag in
h)
    Usage;
    exit 0;;
d)
    actdirectory="$OPTARG";
    echo "   * Using directory \"$actdirectory\"";;
F)
    cosimasourcefile="$OPTARG";
    echo "   * Using Cosima source file \"${cosimasourcefile}\"";;
u)
    rundir="$OPTARG";
    echo "   * Running job in directory \"$rundir"\";;
n)
    name="$OPTARG";
    echo "   * Using job name \"$name\"";;
j)
    startjob=1;
    jobs=$OPTARG;
    echo "   * Using $jobs jobs";;
J)
    restart=1;
    echo "   * Restarting (= increasing the number of jobs) the run";;
l)
    maxjobs=$OPTARG;
    echo "   * Maximum number of running jobs for user ${USER}: $maxjobs";;
q)
    submit=1;
    delay=$OPTARG;
    echo "   * Submitting jobs via qsub with ${delay} s delay";;
s)
    sim=$OPTARG;
    echo "   * Doing simulation with option $sim";;
t)
    triggers=$OPTARG;
    echo "   * Using number of triggers \"$triggers\"";;
g)
    geometry="$OPTARG";
    echo "   * Using geometry \"$geometry\"";;
c)
    convert=1;
    echo "   * Running ConvertMGGPOD";;
z)
    zip=1;
    echo "   * Gzip'ing output files";;
k)
    norecompile=1;
    echo "   * Do *NOT* recompile MGGPOD";;
m)
    mergersps=1;
    echo "   * Merging rsp's";;
a)
    concat=1;
    echo "   * Concatenting sim and tra files";;
A)
    transfer=1;
    machine=$OPTARG;
    echo "   * Packing & transfering data via rsync...";;
y)
    queuemaster=$OPTARG;
    echo "   * Using queue master ${queuemaster}";;
r)
    revan=1;
    echo "   * Running revan";;
o)
    noout=1;
    echo "   * Keeping no output";;
S)
    nustar=1;
    nustarbackgroundid=$OPTARG;
    echo "   * Running NuSTAR analysis";;
i)
    cleantemp=1;
    echo "   * Cleaning temporary directory";;
f)
    revancfg="$OPTARG";
    echo "   * Using this revan configuration file $revancfg";;
x)
    mimreccfg="$OPTARG";
    echo "   * Using this mimrec configuration file $mimreccfg";;
w)
    irradiationtime=$OPTARG;
    echo "   * Assuming an irradiation time of ${irradiationtime} years";;
C)
    usecooldown=1
    cooldowntime=$OPTARG;
    echo "   * Assuming a cool-down time of ${cooldowntime} years";;
v)
    temporary=1;
    temporarydir="$OPTARG";
    echo "   * Analyzing data in temporary directory";;
p)
    response="$OPTARG";
    echo "   * Running response generator";;
N)
    nicelevel=$OPTARG;
    echo "   * Using nicelevel ${nicelvel}";;
D)
    deletelowlevel=1;
    echo "   * Deleting low-level files";;
b)
    special=1
    specialprogram="$OPTARG";
    echo "   * Special program to be executed: $specialprogram";;
e)
    rename=1;
    renamefrom=`echo "$OPTARG" | awk -F ' ' '{ print $1 }'`;
    renameto=`echo "$OPTARG" | awk -F ' ' '{ print $2 }'`;
    renamesuffix=`echo "$OPTARG" | awk -F ' ' '{ print $3 }'`;
    echo "   * Running renaming task";;
esac
done



#### Input upgrades:

# Make absolute path names
here=`pwd`
case "$revancfg" in
    /*)
    ;;
    *) # relative path name
    revancfg="$here/$revancfg";;
esac
case "$mimreccfg" in
    /*)
    ;;
    *) # relative path name
    mimreccfg="$here/$mimreccfg";;
esac
case "$geometry" in
    /*)
    ;;
    *) # relative path name
    geometry="$here/$geometry";;
esac
case "$temporarydir" in
    /*)
    ;;
    *) # relative path name
    temporarydir="$here/$temporarydir";;
esac
case "$rundir" in
    /*)
    ;;
    *) # relative path name
    rundir="$here/$rundir";;
esac
case "$actdirectory" in
    /*)
    ;;
    *) # relative path name
    actdirectory="$here/$actdirectory";;
esac

# The delay is not allowed to be lower than 5 sec in order for showq to work...
if [ $delay -le 5 ]; then
    delay=5
fi


#### Input verification:

# The name is not allowed to end with a /
name=${name%/*}


# Test rename
if [ $rename -gt 0 ]; then
    if ([ "$renamefrom" == "" ] || [ "$renameto" == "" ] || [ "$renamesuffix" == "" ] ); then
        echo "ERROR: Rename needs three arguments enclosed by quotation marks!";
        echo "       The arguments itself are not allowed to contain blanks";
        Usage;
        exit 1;
    fi
fi

# Test temporary directory
if [ $temporary -gt 0 ]; then
    if [ $submit -eq 0 ]; then
        if (`test ! -d $temporarydir`); then
            echo "ERROR: The temporary directory \"$temporarydir\" does not exist"
            Usage;
            exit 1;
        fi
    fi
fi

# Test for geometry
if ([ ${mergersps} == 0 ]); then
    if [ "$geometry" == "geometry_none" ]; then
        echo "ERROR: You need to give a geometry file name";
        Usage;
        exit 1;
    fi        
    if (`test ! -f $geometry`); then
        echo "ERROR: The geometry file does not exist"
        Usage;
        exit 1;
    fi
fi


# Test for revan configuration file
if ([ $revan -gt 0 ] || [ $response != "-" ] ); then
    if [ "$revancfg" == "none" ]; then
        echo "ERROR: Revan/ResponseGenerator need all a revan configuration file name";
        Usage;
        exit 1;
    fi        
    if (`test ! -f $revancfg`); then
        echo "ERROR: The revan configuration file does not exist"
        Usage;
        exit 1;
    fi
fi

# Test for mimrec configuration file
if ([ $response != "-" ] ); then
    if [ "$mimreccfg" == "none" ]; then
        echo "ERROR: ResponseGenerator needs a mimrec configuration file name";
        Usage;
        exit 1;
    fi        
    if (`test ! -f $mimreccfg`); then
        echo "ERROR: The mimrec configuration file does not exist"
        Usage;
        exit 1;
    fi
fi


# ResponseGenerator Option:
if ([ "$response" != "-" ] && [ "$response" != "c" ] && [ "$response" != "t" ]  && [ "$response" != "e" ]  && [ "$response" != "il" ]  && [ "$response" != "ib" ]  && [ "$response" != "ic" ] ); then
    echo "ERROR: Invalid response parameter option";
    Usage;
    exit 1;
fi


# Exist the source directories?
if (([ $sim -gt -1 ] && [ $sim -lt 2 ]) || [ $sim -eq 3 ] || [ $sim -eq 3 ] ); then
    if (`test ! -d $actdirectory`); then
        echo "ERROR: The ACTexamp directory \"$actdirectory\" does not exist"
        Usage;
        exit 1;
    fi

    if (`test ! -d $rundir`); then
        echo "ERROR: The directory \"$rundir\" does not exist"
        Usage;
        exit 1;
    fi
fi


# Check if the main directory already exists, if yes quit!
if ((([ $sim -gt -1 ] && [ $sim -lt 2 ]) || [ $sim -eq 3 ]) && [ $restart -eq 0 ] ); then
    dirnamemain=$rundir"/"$name
    if (`test -d $dirnamemain`); then
        echo "ERROR: Main directory $dirnamemain already exists!"
        Usage;
        exit 1;
    fi
    # Generate: 
    mkdir $dirnamemain;
    # Copy the geometry model
    if [ "$geometry" != "none" ]; then
        echo cp -Lr `dirname $geometry` $dirnamemain
        cp -Lr `dirname $geometry` $dirnamemain
    fi
elif [ $cleantemp -eq 0 ]; then
    dirnamemain=$rundir"/"$name
    if ([ $special -eq 0 ]); then
        if (`test ! -d $dirnamemain`); then
            echo "ERROR: Main directory $dirnamemain does not exist!"
            Usage;
            exit 1;
        else
            # Verify the job number...
            files=0
            for file in `find ${dirnamemain}/ -type d -name "${name}_*"`; do
                files=`expr $files + 1`
            done
            echo "Start job: ${startjob}, existing jobs: ${files}, to be simulated jobs: ${jobs}"
            if ([ $restart -eq 0 ]); then
                if ([ "$files" -ne "$jobs" ] && [ $cleantemp -eq 0 ]); then
                    echo "WARNING: The directory contains a different amount of jobs. Using the calculated number: $files"
                    jobs=$files
                fi
            else
                startjob=`expr $files + 1`
                echo "Start job: ${startjob}, existing jobs: ${files}, to be simulated jobs: ${jobs}"
                if ([ $startjob -ge $jobs ]); then
                    echo "WARNING: Already all jobs have been simulated, i.e. no more jobs necessay via the -J option..."
                    exit 0;
                fi
            fi
        fi
    else
        if (`test ! -d $dirnamemain`); then
            mkdir $dirnamemain;
        fi
    fi
fi

# Test the queuemaster
if ([ ${queuemaster} -gt 2 ] || [ ${queuemaster} -lt 0 ]); then
    echo "ERROR: Unknown queue master ${queuemaster}"
    Usage;
    exit 1;
fi


#### Pre-Process the data

# Create geometry and compile mggpod
if ([ $sim -gt -1 ] && [ $sim -lt 2 ] && [ $norecompile -eq 0 ]); then
    cd $actdirectory
    rm -f materials.mat media.med setup.geo
    rm -f megalib.ini detector.det

    if (`test ! -f $MEGALIB/bin/geomega`); then
        echo "ERROR: geomega not found, aborting."
        Usage;
        exit 1;
    fi

    # Do not store IA infos in normal mode but do when creating a response!
    if ( [ "$response" != "-" ] ); then
        geomega --create-mggpod-default -f ${geometry}
    else
        geomega --create-mggpod-default -f ${geometry}
    fi

    if (`test -d FILES_GGOD2`); then
      rm -f FILES_GGOD2/materials.mat FILES_GGOD2/media.med FILES_GGOD2/setup.geo
      rm -f FILES_GGOD2/megalib.ini FILES_GGOD2/detector.det
      cp materials.mat media.med setup.geo FILES_GGOD2
      cp megalib.ini detector.det FILES_GGOD2
    fi
    make clean
    make
    cd $here
fi 


### The cosima activation stage:
if ([ $sim -eq 4 ]); then
# The following is not a parallel stage at the moment...
    cd ${rundir}"/"$name

    # Copy the Cosima source file into this directory
    cp ${actdirectory}/${cosimasourcefile} .

    # Modify it:
    
    # Replace the geometry file name (mainly to find it)
    geo=`grep "^Geometry" ${cosimasourcefile}`
    if [ "$geo" != "" ]; then 
        mv ${cosimasourcefile} ${cosimasourcefile}~
        sed "s:${geo}:Geometry ${geometry}:g" ${cosimasourcefile}~ > ${cosimasourcefile}
    fi

    # Comment out the original activator - if there is any
    # We have a problem if this line appears multiple times but this wouldn't be a good cosima file...
    act=`grep "^Activator" ${cosimasourcefile}`
    actname=`echo ${act} | awk -F" " '{ print $2 }'`;
    if [ "$act" != "" ]; then 
        mv ${cosimasourcefile} ${cosimasourcefile}~
        sed "s:${act}:#${act}:g" ${cosimasourcefile}~ > ${cosimasourcefile}
    fi
    if [ "$actname" != "" ]; then 
        mv ${cosimasourcefile} ${cosimasourcefile}~
        sed "s:${actname}\.:#${actname}.:g" ${cosimasourcefile}~ > ${cosimasourcefile}
    fi

    # Add the new activator:
    echo "Activator Activ" >> ${cosimasourcefile}
    # Gather all existing activation files
    for inc in `seq $startjob $jobs`; do
        if (`test ! -d ${name}_${inc}`); then
            echo "ERROR: Directory ${name}_${inc} not found!"
            exit 1;
        fi
        cd ${name}_${inc}
        #if (`test -f ${name}.isotopes.dat`); then
        #    echo "Activ.IsotopesFile ${name}_${inc}/${name}.isotopes.dat"
        #fi
        if (`test -f IsotopesLarge.dat`); then
            echo "Activ.IsotopeProductionFile ${name}_${inc}/IsotopesLarge.dat" >> ../${cosimasourcefile}
        fi
        cd ..
    done
    echo "Activ.ActivationFile ${name}.activation.dat" >> ${cosimasourcefile}
    irradiationtime=`echo ${irradiationtime}*365.25*24*60*60 | bc -l`
    echo "Activ.ActivationTime ${irradiationtime}" >> ${cosimasourcefile}
    echo " " >> ${cosimasourcefile}

    cosima ${cosimasourcefile}
fi



### Launch the jobs:
if (([ $sim -gt -1 ] && [ $sim -lt 4 ])  || [ $convert -gt 0 ] || [ $revan -gt 0 ] || [ $nustar -gt 0 ] || [ "$response" != "-" ] ); then
    pipel="pipeline_job"
    if [ $submit -gt 0 ]; then
        pipel=$pipel" -q"
    fi
    pipel=$pipel" -d $actdirectory -F $cosimasourcefile -u $rundir -n $name -t $triggers -z $zip -a $concat -s $sim -g $geometry -o $noout -f $revancfg -x $mimreccfg -N $nicelevel -D $deletelowlevel "

    if [ $convert -gt 0 ]; then
        pipel=$pipel" -c $convert"
    fi
    if [ $revan -gt 0 ]; then
        pipel=$pipel" -r $revan"        
    fi
    if [ $nustar -gt 0 ]; then
        pipel=$pipel" -S $nustarbackgroundid"        
    fi
    if [ $temporary -gt 0 ]; then
        pipel=$pipel"  -v $temporarydir"        
    fi
    if [ "$response" != "-" ]; then
        pipel=$pipel" -p $response"        
    fi 
    
    # Prepare GGOD2 run
    if [ $sim -eq 2 ]; then
        echo "Merging mggpod rate files..."
        cd ${rundir}"/"$name
        cp $MGGPOD/Scripts/meta.lib .
        rm -f intori_mult.log
        rm -f *.rate
        rm -f *.radi
        rm -f fort.*
        
        itime=0.0
        time=0.0
        
      # Verify that all files are present:
        echo "Starting verfication of files (checking for crashed jobs) ..."
        for inc in `seq $startjob $jobs`; do
            if (`test ! -d ${name}_${inc}`); then
                echo "ERROR: Directory ${name}_${inc} not found!"
                exit 1;
            fi
            cd ${name}_${inc}
            if (`test -f ${name}.fits.init`); then
                echo "ERROR: Init file found! Simulation crashed!"
                exit 1;
            fi
            
            if ((`test ! -f ${name}.mggpod_ggod1.log.gz` ) && (`test ! -f ${name}.mggpod_ggod1.log`)); then
                echo "ERROR: In directory ${name}_${inc}: ${name}.mggpod_ggod1.log file is missing!"
                exit 1;
            fi
            if ((`test ! -f ${name}.ggod1.fits.gz`) && (`test ! -f ${name}.ggod1.fits`)); then
                echo "ERROR: In directory ${name}_${inc}: ${name}.ggod1.fits file is missing!"
                exit 1;
            fi
            
            if (`test ! -f ${name}.volumes_list`); then
                echo "ERROR: In directory ${name}_${inc}: ${name}.volumes_list is missing!"
                exit 1;
            fi
            cd ..
        done
        echo " ... verfication succeeded!"
        
        echo "Determining simulation time and *.count files..."
        for inc in `seq $startjob $jobs`; do
            cd ${name}_${inc}
            
            # Calculate time:
            if (`test -f ${name}.mggpod_ggod1.log.gz`); then
                gunzip ${name}.mggpod_ggod1.log.gz
            fi
            if (`test ! -f ${name}.mggpod_ggod1.log`); then
                echo "ERROR: In directory ${name}_${inc}: ${name}.mggpod_ggod1.log could not be unzipped!"
                exit 1;
            fi        
            stime=`grep --text "uglast" ${name}.mggpod_ggod1.log`
            itime=`echo $stime | awk -F ' ' '{ print $5 }'`;
            time=`awk -v a=${itime} -v b=${time} 'BEGIN {print a + b}'`;
            gzip ${name}.mggpod_ggod1.log
            
            # Determine counts:
            rm -f ${name}.ggod1.counts
            rm -f ${name}.prodcountfits.log
            if (`test -f ${name}.ggod1.fits.gz`); then
                gunzip ${name}.ggod1.fits.gz
            fi
            if (`test ! -f ${name}.ggod1.fits`); then
                echo "ERROR: In directory ${name}_${inc}: ${name}.ggod1.fits could not be unzipped!"
                exit 1;
            fi        
            jexec="expect $MEGALIB/bin/pipeline_job_prodcountfits ${name}.volumes_list ${name}.ggod1.fits > ${name}.prodcountfits.log"
            jecho "Starting (${inc} ${jexec}"
            /bin/bash -c "${jexec}"
            gzip ${name}.ggod1.fits
            
            cd ..
        done
        echo " ... total simulation time: ${time} sec"
        
        list=" "
        for inc in `seq $startjob $jobs`; do
            list="${list} ${name}_${inc}/${name}.ggod1.counts"
        done
        jexec="expect $MEGALIB/bin/pipeline_prodrate ${name}_1/${name}.volumes_list ${time} ${jobs} ${list} > $name.prodrate.log"    
        jecho "Starting ${jexec}"
        /bin/bash -c "${jexec}"
        
        if [ $usecooldown -gt 0 ]; then
            jexec="intorid_mult ${name}_1/${name}.volumes_list ${irradiationtime} ${cooldowntime}"
        else
            jexec="intori_mult ${name}_1/${name}.volumes_list ${irradiationtime}"
        fi
        
        jecho "Starting ${jexec}"
        /bin/bash -c "${jexec}"
        
        for inc in `seq $startjob $jobs`; do
            cp *.radi ${name}_${inc}/FILES_GGOD2
        done
        
        cd ${here}
    fi
    
    # Write the max jobs to the directory, so that it can be modified later...
    rm -f ${rundir}"/"${name}"/"${name}_maxjobs.txt
    rm -f ${rundir}"/"${name}"/"${name}_maxqueued.txt
    echo ${maxjobs} >> ${rundir}"/"${name}"/"${name}_maxjobs.txt
    echo ${maxqueued} >> ${rundir}"/"${name}"/"${name}_maxqueued.txt
    
    # Remove job-info from temporary directory
    rm -f /tmp/${name}.*.jobruns

    echo $pipel;
    for inc in `seq $startjob $jobs`; do
        locpipel=$pipel" -i $inc";
        if [ $inc -eq $jobs ]; then
            locpipel=$locpipel" -a 0"
        fi
        if [ $submit -gt 0 ]; then
  	        # Check if we have reached the maximum number of allowed jobs, if yes wait!
            submitted=0
            while [ $submitted -eq 0 ]; do 
                runningjobs=0
                getrunningjobs
                currentmaxjobs=`cat ${rundir}"/"${name}"/"${name}_maxjobs.txt`
                echo "Running jobs for ${USER}: ${runningjobs} of ${currentmaxjobs} allowed"
                
                queuedjobs=0
                getqueuedjobs
                currentmaxqueued=`cat ${rundir}"/"${name}"/"${name}_maxqueued.txt`
                echo "Running or queued jobs for all users: ${queuedjobs} of ${currentmaxqueued}"
                
                if ( [ ${runningjobs} -lt ${currentmaxjobs} ] && [ ${queuedjobs} -lt ${currentmaxqueued} ] ); then 
		                # Unfortunately we have to generate a script here:
                    echo "Submitting job to queue .... "
                    script="Id"$inc"_"$name".sh"
                    rm -f $script
                    echo "#!/bin/bash" >> $script
                    if ([ ${queuemaster} -eq 0 ]); then
                        echo "# Torque/Maui pipeline job script" >> $script 
                        echo "#"  >> $script 
                        echo "#PBS -N Id${inc}_${name}" >> $script              
                    elif ([ ${queuemaster} -eq 1 ]); then
                        echo "# Sun grid engine pipeline job script" >> $script 
                        echo "#"  >> $script 
                        echo "# Change to the current working directory upon starting of the job" >> $script 
                        echo "#$ -cwd" >> $script 
                        echo "# join the error and standard output streams" >> file$script 
                        echo "#$ -j y" >> $script 
                        echo "#" >> $script 
                        echo "# set the required cpu time of your job" >> $script 
                        echo "#$ -l h_cpu=60:00:00" >> $script 
                        echo "#" >> $script 
                        echo "# don't flood myself with e-mail" >> $script 
                        echo "#$ -m n" >> $script 
                        echo "#" >> $script 
                        echo "# name of the job" >> $script  
                        echo "#$ -N Id${inc}_${name}">> $script  
                    elif ([ ${queuemaster} -eq 2 ]); then
                        echo "# LSF pipeline job script" >> $script 
                        echo "#BSUB -J ${name}_${inc}">> $script  
                    fi
                    echo "date" >> $script
                    echo "source ${HOME}/.bashrc" >> $script
                    echo "export" >> $script
                    echo "cd ${here}" >> $script
                    echo "${locpipel}" >> $script
                    echo "exit 0;" >> $script
                    if ([ ${queuemaster} -eq 0 ]); then
                        qsub -V ${retain} ${script}
                    elif ([ ${queuemaster} -eq 1 ]); then
                        qsub -V ${script}
                    elif ([ ${queuemaster} -eq 2 ]); then
                        bsub -n1 -W720 -o ${name}_${inc}_out.txt < ${script}
                    fi
                    submitted=1
                    sleep ${delay}
                else
                    echo "Too many running jobs... sleeping for 30 seconds"
                    sleep 30
                fi
            done
        else
  	    # Check if we have reached the maximum number of allowed jobs, if yes wait!
            submitted=0
            while [ $submitted -eq 0 ]; do 
                runningjobs=0
                lines=`find /tmp -mindepth 1 -maxdepth 1 -name "*.jobruns" -printf "%f "`
                for i in ${lines}; do
                    runningjobs=`expr ${runningjobs} + 1`
                done
                
                currentmaxjobs=`cat ${rundir}"/"${name}"/"${name}_maxjobs.txt`
                
                echo "Running jobs for ${USER}: ${runningjobs} of ${currentmaxjobs} allowed"
                
                if [ ${runningjobs} -lt ${currentmaxjobs} ]; then
                    if [ ${jobs} -gt 1 ]; then
                        /bin/bash ${locpipel} &
                    else
                        /bin/bash ${locpipel} &
                    fi
                    submitted=1
                    sleep 10
                else
                    echo "Too many running jobs... sleeping for 30 seconds"
                    sleep 30
                fi
            done
        fi
    done 
fi

#### Rename
if [ $rename -gt 0 ]; then
    echo "Start to rename files..."
    cd "${rundir}/${name}"
    for inc in `seq $startjob $jobs`; do
        cd $name"_$inc"
        rename ${renamefrom} ${renameto} ${renamesuffix}
        cd ..
    done
    rename ${renamefrom} ${renameto} ${renamesuffix}
    cd "${rundir}"
    rename ${renamefrom} ${renameto} ${renamesuffix}
    cd $here
fi

#### Merge rsp's
Merge() {
    file="$1"
    
    manip="responsemanipulator "
    first=1
    # Generate list of Response files
    cd ${rundir}/${name}
    if (`test -f ${name}_*/${file}.gz`); then
        gunzip ${name}_*/${file}.gz
    fi
    for inc in `ls ${name}_*/${file}`; do
        if [ $first -eq 1 ]; then
            first=0
            cp ${inc} ${file}
            manip="${manip} -f ${file}"
        else
            manip="${manip} -a ${inc}"
        fi
    done
    
    if [ $first -eq 0 ]; then
        echo "Launching ${manip}..."
        $manip
        mv ${file}.new ${file}
    else
        echo "No files ${file} found..."
    fi
    cd $here
}


if [ ${mergersps} -gt 0 ]; then

    # Generate list of Response files
    cd ${rundir}/${name}/${name}_1
    RSPs=""
    pwd
    for inc in `find . -name "${name}*.rsp*" -printf "%f\n"`; do
        RSPs="${RSPs} ${inc}"
    done
    cd ${here}

    for r in ${RSPs}; do
        Merge ${r}
    done
fi



#### Run special program
if ([ $special -gt 0 ]); then
    cd $rundir"/"$name
    pwd
    for inc in `seq $startjob $jobs`; do
        # Unfortunately we have to generate a script here:
        subdir=${name}_${inc}
        if (`test ! -d $subdir`); then
            mkdir $subdir
        fi
        echo "Submitting job to queue .... "
        script="Id"$inc"_"$name".sh"
        rm -f $script
        echo "#!/bin/bash" >> $script
        if ([ ${queuemaster} -eq 0 ]); then
            echo "#PBS -N Id${inc}_${name}" >> $script
        elif ([ ${queuemaster} -eq 1 ]); then
            echo "# Sun grid engine pipeline job script" >> $script 
            echo "#"  >> $script 
            echo "# join the error and standard output streams" >> $script 
            echo "#$ -j y" >> $script 
            echo "#" >> $script 
            echo "# set the required cpu time of your job" >> $script 
            echo "#$ -l h_cpu=60:00:00" >> $script 
            echo "#" >> $script 
            echo "# don't flood myself with e-mail" >> $script 
            echo "#$ -m n" >> $script 
            echo "#" >> $script 
            echo "# name of the job" >> $script  
            echo "#$ -N Id${inc}_${name}">> $script  
        elif ([ ${queuemaster} -eq 2 ]); then
            echo "# LSF pipeline job script" >> $script 
            echo "#"  >> $script 
            echo "# Number of CPUs" >> $script 
            echo "#BSUB -n1" >> $script 
            echo "#" >> $script 
            echo "# Set the required cpu time of your job" >> $script 
            echo "#BSUB -W 720.0" >> $script 
            echo "#" >> $script 
            echo "# Name of the job" >> $script  
            echo "#BSUB -J Id${inc}_${name}">> $script  
        fi
        echo "source ~/.bashrc" >> $script
        date=`date +'%y%m%d%H%M%S'`
        tempjobdir=${temporarydir}/${name}_${id}_${date}
        if ([ $submit -gt 0 ] && [ $temporary -gt 0 ]); then
            echo "mkdir ${tempjobdir}" >> $script
            echo "cd ${tempjobdir}" >> $script
        else
            echo "cd ${rundir}/${name}/${subdir}" >> $script
        fi
        echo "${specialprogram}" >> $script
        if ([ $submit -gt 0 ] && [ $temporary -gt 0 ]); then
            echo "cp * ${rundir}/${name}/${subdir}" >> $script
            echo "cd ..; rm -rf ${tempjobdir}" >> $script
        fi
        echo "exit 0;" >> $script
        if [ $submit -gt 0 ]; then
            if ([ ${queuemaster} -eq 0 ]); then
                qsub -p 500 -V $script
            elif ([ ${queuemaster} -eq 1 ]); then
                qsub -V ${script}
            elif ([ ${queuemaster} -eq 2 ]); then
                bsub < ${script}
            fi
            sleep $delay
        else
            /bin/bash $script
        fi
        rm -f $script
    done
fi

#### Clean temporary directory
if ([ $temporary -gt 0 ] && [ $cleantemp -gt 0 ]); then
    if [ $submit -gt 0 ]; then
        for inc in `seq $startjob $jobs`; do
            echo "Submitting cleaning job to qsub queue .... "
            script="Id"$inc"_"$name".sh"
            rm -f $script
            echo "#!/bin/bash" >> $script
	          echo "#PBS -N Id${inc}_CleanTempDirAndSleep4Min" >> $script
            echo "source ~/.bashrc" >> $script
            echo "cd $temporarydir" >> $script
            echo "sleep 240" >> $script
            echo "pwd" >> $script
            echo "ls -la" >> $script
            echo "echo $HOSTNAME" >> $script
            echo "rm -rf ${name}_*" >> $script
            echo "exit 0;" >> $script
            qsub -V $retain $script
        done
    else
        cd $temporarydir
        rm -r ${name}_*
    fi
fi


#### Pack and transfer data
if ([ $transfer -gt 0 ]); then
    cd ${rundir}/${name}

    usermachine=`echo "$machine" | gawk -F'|' '{ print $1 }'`
    password=`echo "$machine" | gawk -F'|' '{ print $2 }'`

    # Exclude files:
    exclude="Excludes_"${name}".txt"
    rm -f ../$exclude
    include="Includes_"${name}".txt"
    rm -f ../$include

    # Pack all files:
    allfiles=`find . -maxdepth 1 -mindepth 1 -type f -printf "%f "`
    tar cvfz Files.tgz ${allfiles} 
    removelist="${removelist} Files.tgz"

    # Pack all directories
    for subdir in `find . -maxdepth 1 -mindepth 1 -type d -printf "%f\n"`; do
        tar cvfz ${subdir}.tgz ${subdir}
        removelist="${removelist} ${subdir}.tgz"
    done
  
    echo "*" >> ../$exclude
    echo "${name}/" >> ../$include
    echo "*.tgz" >> ../$include

    #Transfer
    cd ${rundir}
    expect ${MEGALIB}/bin/pipeline_rsync "${name}" "${usermachine}" "${password}" "${exclude}" "${include}"

    cd ${rundir}/${name}
    rm -r ${removelist}

    cd $here
fi


#### Postanalysis
simfile="nothing"
filename="tbd"

if [ $concat -gt 0 ]; then
    cd $rundir"/"$name
    if [ $sim -gt 0 ]; then
        # Create dummy sim file:
        simfile=$name".sim"
        rm -rf $simfile
        echo "# Dummy sim file for concatenation" >> $simfile
        echo "Type SIM" >> $simfile
        echo "Version ${simversion}" >> $simfile
        echo "Geometry ${geometry}"  >> $simfile
        echo " "  >> $simfile
        for inc in `seq 1 ${jobs}`; do
            filename="${name}_${inc}/${name}.sim"
            echo "IN $filename" >> $simfile
        done

        if [ $sim -eq 2 ]; then
            # activation component
            simfile=$name".activation.sim"
            rm -rf $simfile
            echo "# Dummy sim file for concatenation" >> $simfile
            echo "Type SIM" >> $simfile
            echo "Version ${simversion}" >> $simfile
            echo "Geometry ${geometry}"  >> $simfile
            echo " "  >> $simfile
            for inc in `seq 1 ${jobs}`; do
                filename=$name"_$inc/"$name".activation.sim"
                echo "IN $filename" >> $simfile
            done

	          # prompt component
            simfile=$name".prompt.sim"
            rm -rf $simfile
            echo "# Dummy sim file for concatenation" >> $simfile
            echo "Type SIM" >> $simfile
            echo "Version ${simversion}" >> $simfile
            echo "Geometry $geometry"  >> $simfile
            echo " "  >> $simfile
            for inc in `seq 1 ${jobs}`; do
                filename=$name"_$inc/"$name".prompt.sim"
                echo "IN $filename" >> $simfile
            done
        fi
    fi

    if ([ $revan -gt 0 ] || [ $nustar -gt 0 ]); then
        # Create dummy tra file:
        trafile=$name".tra"
        rm -rf $trafile
        echo "# Dummy tra file for concatenation" >> $trafile
        echo "Type TRA" >> $trafile
        echo " "  >> $trafile
        for inc in `seq 1 ${jobs}`; do
            filename="${name}_${inc}/${name}.tra"
            echo "IN $filename" >> $trafile
        done

        if [ `find . -name "*.activation.sim*" | wc -m` -gt 0 ]; then
	          # activation component
            trafile=$name".activation.tra"
            rm -rf $trafile
            echo "# Dummy tra file for concatenation" >> $trafile
            echo "Type TRA" >> $trafile
            echo " "  >> $trafile
            for inc in `seq 1 ${jobs}`; do
                filename=$name"_$inc/"$name".activation.tra"
                echo "IN $filename" >> $trafile
            done
	          # prompt component
            trafile=$name".prompt.tra"
            rm -rf $trafile
            echo "# Dummy tra file for concatenation" >> $trafile
            echo "Type TRA" >> $trafile
            echo " "  >> $trafile
            for inc in `seq ${jobs}`; do
                filename=$name"_$inc/"$name".prompt.tra"
                echo "IN $filename" >> $trafile
            done
        fi
    fi
fi

exit 0

