#!/bin/bash

EASY_VERSION="1.1"

bb=`tput bold`
nn=`tput sgr0`

# help
if [ $# -eq 0 ]; then
    echo "${bb}-------------------------------------------------------------------${nn}"
    echo "${bb}Run the KMOS pipeline on a whole set of calibration files."
    echo " "
    echo "This includes kmo_dark for all filters and angles"
    echo "              kmo_flat"
    echo "              kmo_wave_cal${nn}"
    echo " "
    echo "${bb}IMPORTANT:${nn} Files can be searched as well in folders from +/- 1 day!"
    echo "           (if toplevel directories are e.g. named YYYY-MM-DD)"
    echo " "
    echo "- The environment variable ${bb}KMOS_CALIB${nn} pointing to a path has to be"
    echo "  set in order to find the needed static calibration frames."
    echo "  Otherwise the execution of this script will fail!"
    echo "- As argument an arbitrary exposure of an executed"
    echo "  ${bb}KMOS_spec_cal_dark${nn}, ${bb}KMOS_spec_cal_calunitflat${nn} or"
    echo "  ${bb}KMOS_spec_cal_wave${nn} template must be provided."
    echo "  All matching exposures are collected automatically."
    echo "- Sometimes the provided ${bb}KMOS_spec_cal_dark${nn} frames are a bit"
    echo "  arbitrary. If a badpixel_dark*.fits is already present in the working"
    echo "  directory, it can be chosen as well, depending on user input."
    echo "- (Optional) If as additional argument ${bb}sof${nn} is provided, then only"
    echo "  the sof-file is generated but not executed."
    echo "- (Optional) If as additional argument ${bb}par${nn} is provided, then"
    echo "  all flats and all wave_cals for all bands are processed in parallel."
    echo "  If not set, only 3 tasks are started in parallel and afterwards the"
    echo "  remaining bands are processed."
    echo "- All processing output is logged into log_xxx.txt files"
    echo " "
    echo "${bb}Usage:${nn}"
    echo "  easySPARK_calibration.sh <filename with full path>"
    echo "    or"
    echo "  easySPARK_calibration.sh <filename with full path> sof"
    echo " "
    echo "Version: "$EASY_VERSION
    echo "${bb}-------------------------------------------------------------------${nn}"
    exit
fi

spinner()
{
    local delay=0.75
    local spinstr='|/-\'
    while [ "$(ps a | awk '{print $1}' | grep $pid)" ]; do
        local temp=${spinstr#?}
        printf " [%c]  " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b\b\b"
    done
    printf "    \b\b\b\b"
}

# from a list of fits-files get all TPL.START IDs, in order to parse them separately afterwards
getTplStartIds()
{
    local tplStart
    ind=$[-1]
    for file in $@; do
        if [ $ind = -1 ]; then
            # first tpl.start, just store it
            ind=$[$ind+1]
            tplStart[$ind]=$(dfits $file | \
                             fitsort -d tpl.start |\
                             gawk '{print $2}' |\
                             tr "\n" " ")
        else
            # subsequent tpl.start
            tmpTplStart=$(dfits $file | \
                          fitsort -d tpl.start |\
                          gawk '{print $2}' |\
                          tr "\n" " ")

            # compare if it is already stored
            found=$[0]
            for id in ${tplStart[*]}; do
                if [ $tmpTplStart = $id ]; then
                    # new tpl.start
                    found=$[1]
                    break
                fi
            done

            # it hasn't been found, so add it
            if [ $found = 0 ] ; then
                ind=$[$ind+1]
                tplStart[$ind]=$tmpTplStart
            fi
        fi
    done
    echo "${tplStart[@]}"
}

vers_dark=( $(easySPARK_dark.sh vers) )
vers_flat=( $(easySPARK_flat.sh vers) )
vers_wave=( $(easySPARK_wave_cal.sh vers) )
vers_test=( $(easySPARK_test.sh vers) )

if [[ $EASY_VERSION != $vers_dark || $EASY_VERSION != $vers_flat || $EASY_VERSION != $vers_wave  || $EASY_VERSION != $vers_test ]]; then
    echo "${bb}ERROR:${nn} Version mismatch!"
    echo "  easySPARK_calibration.sh: v"$EASY_VERSION
    echo "    easySPARK_dark.sh:      v"$vers_dark
    echo "    easySPARK_flat.sh:      v"$vers_flat
    echo "    easySPARK_wave_cal.sh:  v"$vers_wave
    echo "    easySPARK_test.sh:      v"$vers_test
    echo ""
    echo "Here the paths of the identified easySPARK-scripts:"
    echo "  "$(which easySPARK_calibration.sh)
    echo "  "$(which easySPARK_dark.sh)
    echo "  "$(which easySPARK_flat.sh)
    echo "  "$(which easySPARK_wave_cal.sh)
    echo "  "$(which easySPARK_test.sh)
    exit
fi

# init
sofOnly=0
parallelMode=0
printVers=0

# extract all arguments beginning with KMOS, eventually imethod and sof
while [ "$1" ] ; do
   tmp=$(basename $1)
   if [[ $tmp == KMOS* ]]; then
       # file-check
       if [ ! -e $1 ]; then
          echo "${bb}ERROR:${nn} File doesn't exist! ($1)"
          exit
       fi

       fileName=$1

       if [ ${fileName:0:1} != "/" ]; then
          echo "${bb}ERROR:${nn} Filepath should be absolute!"
          exit
       fi
   else
      if [[ $tmp == sof ]]; then
         sofOnly=1
      elif [[ $tmp == par ]]; then
         parallelMode=1
      elif [[ $tmp == vers ]]; then
         printVers=1
      fi
   fi
   shift
done

if [ $printVers = 1 ]; then
    echo $EASY_VERSION
    exit
fi

# check if gawk is existing
if command -v gawk >/dev/null; then
   :
else
    echo "'gawk' has not been found on your system!"
    OS=$(uname -s)
    if [ $OS != "Linux" ]; then
        echo "You seem using a MacOS, please use e.g. MacPorts to install gawk."
    fi
    echo "Aborting now.!"
fi

cntBadPixDark=0
fn_badpix=
if ls badpixel_dark* &> /dev/null; then
    fn_badpix=$(ls badpixel_dark*)
    for i in $fn_badpix; do
       cntBadPixDark=$[$cntBadPixDark+1]
    done
fi

#check if file is of correct template
tplId=$(dfits $fileName |\
        fitsort -d tpl.id |\
        gawk '{print $2}')
if [[ $tplId != KMOS_spec_cal_dark && $tplId != KMOS_spec_cal_calunitflat && $tplId != KMOS_spec_cal_wave ]]; then
    echo "${bb}ERROR${nn}: File is of wrong template: $tplId instead of KMOS_spec_cal_dark, KMOS_spec_cal_calunitflat or KMOS_spec_cal_wave !"
    exit
fi

if [ -z "${KMOS_CALIB}" ]; then
    echo "${bb}ERROR${nn}: environment variable KMOS_CALIB is not defined! (see help)"
    exit
fi

if [ $parallelMode = 1 ]; then
    echo "${bb}PARALLEL MODE activated: all flats and wave_cals for all bands are processed in parallel.${nn}"
fi

##########################################################
# identifying observation run
##########################################################

# extract OBS.START of ths file
obsStartTime=$(dfits $fileName |\
               fitsort -d obs.start |\
	       gawk '{print $2}')

# analyze path, extract dates for today and yesterday            fileName:     # /<bla>/<blo>/2013-01-20/KMOS<xxx>
dataDir_today=$(dirname "$fileName")                                           # /<bla>/<blo>/2013-01-20
dataDir=${dataDir_today%/*}                                                    # /<bla>/<blo>/
today=${dataDir_today##*/}                                                     # 2013-01-20
matchingFiles=

date -d $today > /dev/null 2>&1
if [ $? = 0 ]; then
    # we have a directory like 2013-06-10
    yesterday=$(date --date=@$(( $(date --date=$today +%s) - 86400 )) +'%Y-%m-%d') # 2013-01-19
    dataDir_yesterday=$dataDir"/"$yesterday                                        # /<bla>/<blo>/2013-01-19
    tomorrow=$(date --date=@$(( $(date --date=$today +%s) + 86400 )) +'%Y-%m-%d')  # 2013-01-21
    dataDir_tomorrow=$dataDir"/"$tomorrow                                          # /<bla>/<blo>/2013-01-21

    # get all files with matching obsStartTime from yesterday
    matchingFiles_yesterday=
    if [ -e $dataDir_yesterday ]; then
        matchingFiles_yesterday=$(dfits $dataDir_yesterday/KMOS*.fits | \
                                  fitsort -d obs.start |\
                                  grep $obsStartTime |\
                                  gawk '{print $1}' |\
                                  tr "\n" " ")
    fi

    # get all files with matching obsStartTime from tomorrow
    matchingFiles_tomorrow=
    if [ -e $dataDir_tomorrow ]; then
        matchingFiles_tomorrow=$(dfits $dataDir_tomorrow/KMOS*.fits | \
                                 fitsort -d obs.start |\
                                 grep $obsStartTime |\
                                 gawk '{print $1}' |\
                                 tr "\n" " ")
    fi

    matchingFiles=$matchingFiles_yesterday$matchingFiles_tomorrow
fi

# get all files with matching obsStartTime from today
matchingFiles_today=$(dfits $dataDir_today/KMOS*.fits | \
                      fitsort -d obs.start |\
                      grep $obsStartTime |\
                      gawk '{print $1}' |\
                      tr "\n" " ")

matchingFiles=$matchingFiles$matchingFiles_today

##########################################################
# get all files for DARK, FLAT, ARC
##########################################################
# get files with matching TPL.ID for dark
matchingDarks=$(dfits $matchingFiles | \
                fitsort -d tpl.id |\
                grep KMOS_spec_cal_dark |\
                gawk '{print $1}' |\
                tr "\n" " ")

# get files with matching TPL.ID for flat
matchingFlats=$(dfits $matchingFiles | \
                fitsort -d tpl.id |\
                grep KMOS_spec_cal_calunitflat |\
                gawk '{print $1}' |\
                tr "\n" " ")

# get files with matching TPL.ID for wave_cal
matchingArcs=$(dfits $matchingFiles | \
               fitsort -d tpl.id |\
               grep KMOS_spec_cal_wave |\
               gawk '{print $1}' |\
               tr "\n" " ")

##########################################################
# get number of TPL.START IDs for KMOS_spec_cal_calunitflat
##########################################################
tplStartIdsFlat=( $(getTplStartIds $matchingFlats) )
cntFlatSets=$[${#tplStartIdsFlat[@]}]

if [ $cntFlatSets -gt 5 ]; then
    echo "${bb}ERROR:${nn} There exist" $cntFlatSets "different template IDs in the specified OB"
    echo "       for flat calibration, but just one for each band is expected!"
    echo "       Please run ${bb}easySPARK_flat.sh${nn} and ${bb}easySPARK_wave_cal.sh${nn} individually on the desired data sets!"
    exit
elif [ $cntFlatSets -gt 0 ]; then
    setNr=0
    #check if
    for id in ${tplStartIdsFlat[*]}; do
        matchingFlatBand[$setNr]=$(dfits $matchingFlats | \
                                   fitsort -d tpl.start |\
                                   grep $id |\
                                   gawk '{print $1}')

        flatGratings[$setNr]=$(dfits ${matchingFlatBand[$setNr]} |\
                               fitsort -d ins.grat1.id |\
                               tail -n1 |\
                               gawk '{print $2}')
        setNr=$[$setNr+1]
    done
elif [ $cntFlatSets -gt 5 ]; then
    exit
fi

##########################################################
# get number of TPL.START IDs for KMOS_spec_cal_wave
##########################################################
tplStartIdsArc=( $(getTplStartIds $matchingArcs) )
cntArcSets=$[${#tplStartIdsArc[@]}]

if [ $cntArcSets -gt 5 ]; then
    echo "${bb}ERROR:${nn} There exist" $cntArcSets "different template IDs in the specified OB"
    echo "       for wave calibration, but just one for each band is expected!"
    echo "       Please run ${bb}easySPARK_wave_cal.sh${nn} individually on the desired data sets!"
    exit
elif [ $cntArcSets -gt 0 ]; then
    setNr=0
    #check if
    for id in ${tplStartIdsArc[*]}; do
        matchingArcBand[$setNr]=$(dfits $matchingArcs | \
                                  fitsort -d tpl.start |\
                                  grep $id |\
                                  gawk '{print $1}')

        arcGratings[$setNr]=$(dfits ${matchingArcBand[$setNr]} |\
                              fitsort -d ins.grat1.id |\
                              tail -n1 |\
                              gawk '{print $2}')
        setNr=$[$setNr+1]
    done
fi

##########################################################
# get number of TPL.START IDs for KMOS_spec_cal_dark
# if there is more than one TPL.START ID, let the user select which one to process
##########################################################
tplStartIdsDark=( $(getTplStartIds $matchingDarks) )
cntDarkSets=$[${#tplStartIdsDark[@]}]

if [[ $cntDarkSets -eq 0 && $cntBadPixDark -eq 0 ]]; then
    echo "${bb}ERROR:${nn} No matching dark exposures and no badpixel_dark*.fits found!"
    exit
elif [[ $cntBadPixDark -eq 1 && $cntDarkSets -eq 0 ]]; then
    echo "${bb}ATTENTION:${nn}"
    echo "No ${bb}KMOS_spec_cal_dark${nn} template has been executed in this OB,"
    echo "but "$fn_badpix" has been found in this directory."
    echo "It will be used to calculate flats and wave_cals."
    echo ""
    cntBadPixDark=1
    cntDarkSets=0
elif [[ $cntBadPixDark -eq 0 && $cntDarkSets -eq 1 ]]; then
    # do nothing
    cntBadPixDark=0
    cntDarkSets=1
elif [[ $cntBadPixDark -eq 0 && $cntDarkSets -gt 1 ]]; then
    echo "${bb}ATTENTION:${nn}"
    echo "Several ${bb}KMOS_spec_cal_dark${nn} templates have been executed in this OB!"
    echo "Select the set you want to use for calibration (normally EXPTIME = 10s and"
    echo "at least 3 exposures per set)"
    echo ""
    echo -e "ID\tEXPTIME\tTPL.START\t\tFILES"
    echo ""

    # print available TPL.START sets
    setNr=1
    for id in ${tplStartIdsDark[*]}; do
        tmpDarks=$(dfits $matchingDarks | \
                   fitsort -d tpl.start |\
                   grep $id|\
                   gawk '{print $1}')
        tmpExptime=$(dfits ${tmpDarks} | \
                     fitsort -d tpl.start exptime |\
                     grep $id |\
                     gawk '{print $3}')

        echo -e "$setNr\t${tmpExptime%%.*}\t$id"
        for file in $tmpDarks; do
            echo -e "\t\t\t\t\t"$(basename $file)
        done
        setNr=$[$setNr+1]
    done
    setNr=$[$setNr-1]

    # ask user which one to use
    read -p "Enter ID (1 to $setNr): " input
    while [[ $input -lt 1 || $input -gt $setNr ]]; do
        read -p "Enter ID (1 to $setNr): " input
    done

    myTplStartIdDark=${tplStartIdsDark[$[$input-1]]}

    # update matchingDarks
    for id in ${tplStartIdsDark[*]}; do
        if [ $id = $myTplStartIdDark ]; then
            matchingDarks=$(dfits $matchingDarks | \
                            fitsort -d tpl.start |\
                            grep $id|\
                            gawk '{print $1}')
        fi
    done
    cntDarkSets=1
    cntBadPixDark=0
else
    echo "${bb}ATTENTION:${nn}"
    echo "Several ${bb}KMOS_spec_cal_dark${nn} templates have been executed in this OB!"
    echo "as well "$fn_badpix" has been found in this directory!"
    echo "Select the set you want to use for calibration (normally EXPTIME = 10s and"
    echo "at least 3 exposures per set)"
    echo ""
    echo -e "ID\tEXPTIME\tTPL.START\t\tFILES"
    echo ""

    # print available badpixel_dark*.fits
    setNr=1
    for i in $fn_badpix; do
       echo -e "$setNr\t"$i
       echo ""
       setNr=$[$setNr+1]
    done

    # print available TPL.START sets
    for id in ${tplStartIdsDark[*]}; do
        tmpDarks=$(dfits $matchingDarks | \
                   fitsort -d tpl.start |\
                   grep $id|\
                   gawk '{print $1}')
        tmpExptime=$(dfits ${tmpDarks} | \
                     fitsort -d tpl.start exptime |\
                     grep $id |\
                     gawk '{print $3}')

        echo -e "$setNr\t${tmpExptime%%.*}\t$id"
        for file in $tmpDarks; do
            echo -e "\t\t\t\t\t"$(basename $file)
        done
        setNr=$[$setNr+1]
    done
    setNr=$[$setNr-1]

    # ask user which one to use
    read -p "Enter ID (1 to $setNr): " input
    while [[ $input -lt 1 || $input -gt $setNr ]]; do
        read -p "Enter ID (1 to $setNr): " input
    done

    if [[ $input -ge 1 && $input -le $cntBadPixDark ]]; then
        cntDarkSets=0
        cntBadPixDark=$input
    else
        myTplStartIdDark=${tplStartIdsDark[$[$input-2]]}

        # update matchingDarks
        for id in ${tplStartIdsDark[*]}; do
            if [ $id = $myTplStartIdDark ]; then
                matchingDarks=$(dfits $matchingDarks | \
                                fitsort -d tpl.start |\
                                grep $id|\
                                gawk '{print $1}')
            fi
        done
        cntDarkSets=1
        cntBadPixDark=0
    fi
fi

# check if all bands in wave_cal exist as well in flats.
# If not, the band is ignored in arcs as well
setNrArc=0
for idArc in ${arcGratings[*]}; do
    foundBand=0
    setNrFlat=0
    for idFlat in ${flatGratings[*]}; do
        if [[ foundBand -eq 0 && $idArc = $idFlat ]]; then
            foundBand=1
            break
        fi
        setNrFlat=$[$setNrFlat+1]
    done

    if [ $foundBand -eq 0 ]; then
        echo "${bb}ATTENTION:${nn} Band $idArc is present in the wave_cals but not in the flats. Omitting it from now on."
        tmpSetNr=$setNrArc
        while [ $tmpSetNr -le $cntArcSets ]; do
            arcGratings[$tmpSetNr]=${arcGratings[$tmpSetNr+1]}
            matchingArcBand[$tmpSetNr]=${matchingArcBand[$tmpSetNr+1]}
            tplStartIdsArc[$tmpSetNr]=${tplStartIdsArc[$tmpSetNr+1]}
            tmpSetNr=$[$tmpSetNr+1]
        done
        cntArcSets=$[${#tplStartIdsArc[@]}]
    fi
    setNrArc=$[$setNrArc+1]
done

##########################################################
# print overview of extracted frames
##########################################################
echo ""
echo "------------------------- easySPARK_calibration -------------------------"
echo "INPUT"
echo "filename:  "$fileName
echo "with "
echo "OBS.START: "$obsStartTime
echo ""
echo "OUTPUT"
echo ""

# print darks
echo "    darks:"
i=1
if [ $cntDarkSets -eq 1 ]; then
    for file in $matchingDarks; do
        echo -e "`printf "%02d" $i`  $(basename $file)"
        i=$[i+1]
    done
else
    for file in $fn_badpix; do
        if [ $i -eq $cntBadPixDark ]; then
            echo "01  $file"
            myDarkFilename=$file
        fi
        i=$[$i+1]
    done
fi
echo ""

# print flats
echo "    flats:"
if [ $cntFlatSets -eq 0 ]; then
    echo "No matching flat exposures found."
else
    setNr=0
    out_str="    "
    for id in ${tplStartIdsFlat[*]}; do
        out_str=$out_str${flatGratings[$setNr]}"\t\t\t\t"
        setNr=$[$setNr+1]
    done
    echo -e "$out_str"

    lineNr=1
    while [ $lineNr -gt 0 ]; do
        setNr=0
        out_str="`printf "%02d" $lineNr`  "
        maxNr=-1
        for id in ${tplStartIdsFlat[*]}; do
            tmp_arr=(${matchingFlatBand[$setNr]})
            tmp_arr_size=${#tmp_arr[*]}
            if [ $tmp_arr_size -gt $maxNr ]; then
                maxNr=$tmp_arr_size
            fi

            if [ $tmp_arr_size -ge $lineNr ]; then
                out_str=$out_str$(basename $(echo ${matchingFlatBand[$setNr]}|gawk -v r=$lineNr '{print $r}'))"\t"
            else
                out_str=$out_str"\t\t\t\t"
            fi
            setNr=$[$setNr+1]
        done
        echo -e "$out_str"
        lineNr=$[$lineNr+1]
        if [ $lineNr -gt $maxNr ]; then
            lineNr=-1
        fi
    done
fi
echo ""

# print arcs
echo "    wave_cals:"
if [ $cntArcSets = 0 ]; then
    echo "No matching wave_cal exposures found."
else
setNr=0
out_str="    "
for id in ${tplStartIdsArc[*]}; do
    out_str=$out_str${arcGratings[$setNr]}"\t\t\t\t"
    setNr=$[$setNr+1]
done
echo -e "$out_str"

lineNr=1
while [ $lineNr -gt 0 ]; do
    setNr=0
    out_str="`printf "%02d" $lineNr`  "
    maxNr=-1
    for id in ${tplStartIdsArc[*]}; do
        tmp_arr=(${matchingArcBand[$setNr]})
        tmp_arr_size=${#tmp_arr[*]}
        if [ $tmp_arr_size -gt $maxNr ]; then
            maxNr=$tmp_arr_size
        fi

        if [ $tmp_arr_size -ge $lineNr ]; then
            out_str=$out_str$(basename $(echo ${matchingArcBand[$setNr]}|gawk -v r=$lineNr '{print $r}'))"\t"
        else
            out_str=$out_str"\t\t\t\t"
        fi
        setNr=$[$setNr+1]
    done
    echo -e "$out_str"
    lineNr=$[$lineNr+1]
    if [ $lineNr -gt $maxNr ]; then
        lineNr=-1
    fi
done
fi

##########################################################
# ask for user confirmation before processing
##########################################################
echo "--------------------------------------------------------------------"
if [ $sofOnly = 0 ]; then
    # execute
    if [[ $cntDarkSets -eq 1 || $cntBadPixDark -ge 1 ]]; then
        read -p "Should the calibrations be executed like this now? (Y/n)" input
        input=${input:-y}
        if [[ "$input" != "y" && "$input" != "Y" ]]; then
            echo "Quit now."
            exit 0
        fi
    else
        exit
    fi
else
    # create sof only
    read -p "Should the sof-files be written like this now? (Y/n)" input
    input=${input:-y}
    if [[ "$input" != "y" && "$input" != "Y" ]]; then
        echo "Quit now."
        exit 0
    fi
fi

if [ $sofOnly = 0 ]; then
    echo "--------------------------------------------------------------------"
    read -p "Should a directory '$obsStartTime' be created ? (Y/n)" createDir
    createDir=${createDir:-y}
    if [[ "$createDir" != "n" && "$createDir" != "N" ]]; then
        mkdir $obsStartTime
        cd $obsStartTime
        if [ $cntBadPixDark -ge 1 ]; then
            cp -s ../$myDarkFilename .
        fi
    fi

    echo "This takes a while. Go get a coffee. And a sandwich afterwards..."
    echo ""

    startTime=`date +%s`

    ##########################################################
    # process dark
    ##########################################################
    if [ $cntDarkSets -eq 1 ]; then
        echo "Start processing dark... (log_dark.txt)..."
        echo "easySPARK_dark.sh "$(echo $matchingDarks|gawk '{print $1}') > log_dark.txt
        (easySPARK_dark.sh $(echo $matchingDarks|gawk '{print $1}') 2>&1 >> log_dark.txt) &
        pid=$!
        spinner &
        #wait for dark to finish
        wait
        echo "Finished processing dark."
    fi

    ##########################################################
    # process all flats in parallel for 5 bands
    # the bands are started with 90 seconds delay in order not to slow down
    # the computer too much when saving files to disk
    ##########################################################
    if [ $parallelMode = 0 ]; then
        # serial processing
        echo "The first 3 flats are now started in parallel with an offset of 90 seconds."
        setNr=0
        for i in ${tplStartIdsFlat[*]}; do
            if [ $setNr -le 2 ]; then
                mkdir -p ${flatGratings[$setNr]}"tmp"
                cd ${flatGratings[$setNr]}"tmp"
                ln -s ../$myDarkFilename $myDarkFilename
                echo "Start processing flat in ${flatGratings[$setNr]}-band... (log_flat_${flatGratings[$setNr]}.txt)"
                echo "easySPARK_flat.sh "$(echo ${matchingFlatBand[$setNr]}|gawk '{print $(NF)}') auto > ../log_flat_${flatGratings[$setNr]}.txt
                (easySPARK_flat.sh $(echo ${matchingFlatBand[$setNr]}|gawk '{print $(NF)}') auto 2>&1 >> ../log_flat_${flatGratings[$setNr]}.txt) &
                pid=$!		# save pid from task above
                spinner &
                cd ..
            fi
            setNr=$[$setNr+1]
        done
        # wait for all flats beeing finished
        wait

        echo "The last flats are now started now in parallel with an offset of 90 seconds."
        setNr=0
        for i in ${tplStartIdsFlat[*]}; do
            if [ $setNr -ge 3 ]; then
                mkdir -p ${flatGratings[$setNr]}"tmp"
                cd ${flatGratings[$setNr]}"tmp"
                ln -s ../myDarkFilename $myDarkFilename
                echo "Start processing flat in ${flatGratings[$setNr]}-band... (log_flat_${flatGratings[$setNr]}.txt)"
                echo "easySPARK_flat.sh "$(echo ${matchingFlatBand[$setNr]}|gawk '{print $(NF)}') auto > ../log_flat_${flatGratings[$setNr]}.txt
                (easySPARK_flat.sh $(echo ${matchingFlatBand[$setNr]}|gawk '{print $(NF)}') auto 2>&1 >> ../log_flat_${flatGratings[$setNr]}.txt) &
                pid=$!		# save pid from task above
                spinner &
                cd ..
                if [ $i -ne 4 ]; then
                    sleep 90
                fi
            fi
            setNr=$[$setNr+1]
        done
    else
        # parallel processing
        echo "All flats are now started in parallel with an offset of 90 seconds."
        setNr=0
        for i in ${tplStartIdsFlat[*]}; do
            mkdir -p ${flatGratings[$setNr]}"tmp"
            cd ${flatGratings[$setNr]}"tmp"
            ln -s ../$myDarkFilename $myDarkFilename
            echo "Start processing flat in ${flatGratings[$setNr]}-band... (log_flat_${flatGratings[$setNr]}.txt)"
            echo "easySPARK_flat.sh "$(echo ${matchingFlatBand[$setNr]}|gawk '{print $(NF)}') auto > ../log_flat_${flatGratings[$setNr]}.txt
            (easySPARK_flat.sh $(echo ${matchingFlatBand[$setNr]}|gawk '{print $(NF)}') auto 2>&1 >> ../log_flat_${flatGratings[$setNr]}.txt) &
            pid=$!		# save pid from task above
            spinner &
            cd ..
            if [ $setNr -ne 4 ]; then
                sleep 90
            fi
            setNr=$[$setNr+1]
        done
    fi
    # wait for all flats beeing finished
    myPwd=$(pwd)
    wait
    setNr=0
    for i in ${tplStartIdsFlat[*]}; do
        cd ${flatGratings[$setNr]}"tmp"
        rm $myDarkFilename
        cd ..
        mv ${flatGratings[$setNr]}"tmp"/* .
        rm -r ${flatGratings[$setNr]}"tmp"
        setNr=$[$setNr+1]
    done
    cd $myPwd
    echo "Finished processing flats."

    ##########################################################
    # process all arcs in parallel for 5 bands
    # the bands are started with 90 seconds delay in order not to slow down
    # the computer too much when saving files to disk
    ##########################################################
    if [ $parallelMode = 0 ]; then
        # serial processing
        echo "The first 3 wave_cals are now started in parallel with an offset of 90 seconds."
        setNr=0
        for i in ${tplStartIdsArc[*]}; do
            if [ $setNr -le 2 ]; then
                echo "Start processing arc in ${arcGratings[$setNr]}-band... (log_wave_cal_${arcGratings[$setNr]}.txt)"
                echo "easySPARK_wave_cal.sh "$(echo ${matchingArcBand[$setNr]}|gawk '{print $(NF)}') > log_wave_cal_${arcGratings[$setNr]}.txt
                (easySPARK_wave_cal.sh $(echo ${matchingArcBand[$setNr]}|gawk '{print $(NF)}') 2>&1 >> log_wave_cal_${arcGratings[$setNr]}.txt) &
                pid=$!
                spinner &
            fi
            setNr=$[$setNr+1]
        done
        # wait for all arcs beeing finished
        wait

        echo "The last 2 wave_cals are now started now in parallel with an offset of 90 seconds."
        setNr=0
        for i in ${tplStartIdsArc[*]}; do
            if [ $setNr -ge 3 ]; then
                echo "Start processing arc in ${arcGratings[$setNr]}-band... (log_wave_cal_${arcGratings[$setNr]}.txt)"
                echo "easySPARK_wave_cal.sh "$(echo ${matchingArcBand[$setNr]}|gawk '{print $(NF)}') > log_wave_cal_${flatGratings[$setNr]}.txt
                (easySPARK_wave_cal.sh $(echo ${matchingArcBand[$setNr]}|gawk '{print $(NF)}') 2>&1 >> log_wave_cal_${flatGratings[$setNr]}.txt) &
                pid=$!
                spinner &
                if [ $setNr -ne 4 ]; then
                    sleep 90
                fi
            fi
            setNr=$[$setNr+1]
        done
    else
        # parallel processing
        echo "The 5 wave_cals are now started in parallel with an offset of 90 seconds."
        setNr=0
        for i in ${tplStartIdsArc[*]}; do
            echo "Start processing arc in ${arcGratings[$setNr]}-band... (log_wave_cal_${arcGratings[$setNr]}.txt)"
            echo "easySPARK_wave_cal.sh "$(echo ${matchingArcBand[$setNr]}|gawk '{print $(NF)}') > log_wave_cal_${flatGratings[$setNr]}.txt
            (easySPARK_wave_cal.sh $(echo ${matchingArcBand[$setNr]}|gawk '{print $(NF)}') 2>&1 >> log_wave_cal_${flatGratings[$setNr]}.txt) &
            pid=$!
            spinner &
            if [ $setNr -ne 4 ]; then
                sleep 90
            fi
            setNr=$[$setNr+1]
        done
    fi
    # wait for all arcs beeing finished
    wait
    echo "Finished processing arcs."

    echo "${bb}--------------------------------------------------------------------${nn}"
    echo "${bb}Finished processing easySPARK_calibration.sh${nn}"
    echo ""
    echo "The calibrations are to be found in:"
    echo $(pwd)
    echo ""
    endTime=`date +%s`
    echo "Overall execution time was "`expr $endTime - $startTime`" s."
    echo "${bb}--------------------------------------------------------------------${nn}"

    # print status report of created files
    easySPARK_test.sh .
else
    # just create sof files
    if [ $cntDarkSets -eq 1 ]; then
        easySPARK_dark.sh $(echo $matchingDarks|gawk '{print $1}') sof
    fi

    setNr=0
    for i in ${tplStartIdsFlat[*]}; do
        easySPARK_flat.sh $(echo ${matchingFlatBand[$setNr]}|gawk '{print $(NF)}') sof auto
        setNr=$[$setNr+1]
    done

    setNr=0
    for i in ${tplStartIdsArc[*]}; do
        easySPARK_wave_cal.sh $(echo ${matchingArcBand[$setNr]}|gawk '{print $(NF)}') sof
        setNr=$[$setNr+1]
    done
fi
