#!/bin/bash
# set -x
# User is forced to use either SGE or SLURM options, except
# -p and -q can both be used for either batch system.
#
# For Stampede the system startup file /etc/profile.d/z95_idev.<csh|sh>
# takes care of sourcing /tmp/idev_<csh|sh>_env.<jobid> if the file
# exists, and exports the env. vars: IDEV_SETUP_BYPASS="1.0",
# idev_ip=cyxz-abc, idev_env_status=[0|1].
# If the sourcing file  exists in /tmp on the compute node 
# idev_env_status is set to 0, the file is sourced, shell cd's
# to $IDEV_PWD, and DDTMPIRUN=ibrun is exported.  This is called
# System Sourcing.

# If System Sourcing exists for the machine, IDEV_ENV_REPLICATION
# is exported as SYSTEM, and no user startup file (.profile, etc.)
# is instrumented to source an environment file.  This can be 
# bypassed by including:
#       IDEV_ENV_REPLICATION=USER
# in the .idevrc file.
# Note: For SYSTEM /tmp/idev_<csh|sh>_env.<jobid> is the sourcing file
#       For USER     /tmp/my_<csh|sh>_env.<jobid> is the sourcing file

# SLURM
# If user account becomes invalid, or single-project mode 
# is not longer valid (user acquires an additional project), 
# the idev_project line (default project) is removed from ~/.idevrc.
#                                                06/28/2013 KentM
# SLURM 
# Reservations can be requested. Users are queried
# about any active reservation-- do they want to use it. 
# Use "idev -r none" to avoid questions.
#                                                09/20/2013 KentM
# Remove SSH_xxx from environment
# Allow users to remove environment variables
# by adding idev_rm_env=<space-separated list of env vars>
#
# Eliminates PERL5LIB from idev environment when
# executing idev perl commands.
#                                                03/30/2015 KentM 
# Adjustments for ls5
#   On nidxxxx service node, system type will be ls5
#   Remove returns BASH_FUNC_ functions. See env_replication routine.
#   Now uses $HOME/.slurm to hold job scripts
#   (use IDEV_KEEP_MY_JOB_DIR variable to save job scipt to directory)
#   zzz85_idev.sh and zzz85_idev.csh updated in /etc/profile.d
#Set up CpN for vis and gpu queue on ls5 -- change for stampede later
#Remove SHELL_STARTUP_DEBUG_T0 from environment variables on compute node.
#Use $CpN when determining Project from test batch launch (R.McLay req).
#Changed ls5 default queue to development (was normal)
#Inserted -A in ssh command (forwarding of authentication agent connection).
#        
#                                                12/09/2015 KentM 
#
#Hidden grace-serial queue 
#   Added -p <queue_name> option to squeue (to see queue)
#   Allow -n 1 for default for grace-serial queue
#   Changed format from 9 to 15 for queue name (grace-serial is 12 chars)
#   grace-serial queue users are allowed only 1 interactive job
#                                                01/29/2016 KentM 
# TODO
# Update version number.
#------------------------------------------------------------------------------
#
#               Versions Number
   IDEV_VER=1.00
   IDEV_SETUP_VER=1.00

#NOTE:  use hostname -f
#Determine the system
  myhostname=`hostname -f`
  if [[ $myhostname =~ nid ]] ; then
     MY_SYS=ls5  #I'm on a MOM node (login0, etc.)     
  else
     MY_SYS=`echo $myhostname | awk 'BEGIN { FS = "." } ; { print $2 }'`
  fi

#800 comment her 800_Wfix
  srun=off        # Don't use srun on Stampede
# trap ctrl-c and call ctrl_c()
trap ctrl_c INT EXIT
did_exit=no
exit_msg=""

#800  comment for 900_Wfix
#                         Trap Function.  Deletes job on exit of this script
function ctrl_c() {
                         #Job submitted, and job id available.
        if [[ "x$job_submitted" = "xyes" && "x$job_id" != "xnone" ]]; then
#                                        Keeps from calling qdel twice. (why twice?)
           if [[ $did_exit == no ]]; then
             echo "Cleaning up: submitted job ($job_submitted) removing job $job_id."
 
             if [[ $MY_SYS == "stampede" || $MY_SYS == "ls5" ]]; then
                scancel $job_id
             else
                qdel    $job_id
             fi
           fi
           did_exit=yes
           exit;
        fi
                         #Job submitted, but no job id yet.
        if [ "x$job_submitted" = "xyes" ]; then
           echo "Please control-c again, SGE/SLURM is responding slowly and I need a job ID for cleanup."
           echo "job_id:$job_id job_submitted:$job_submitted"
        else
                         #User control-c'd out before job launched.
           echo -e "$exit_msg"
           echo " --  idev: no session created, gracefully exiting..."
           exit
        fi
}

#               Do the non-installation sensitive options here.


#               Intel port and server -- for using intel products on comp. nodes.
   my_intel_port_server=27000@troy.tacc.utexas.edu

###               @TACC copy intel license to .sge/intel
###               Intel license info:
##intel_license=intel.current.20120301.lic
## my_intel_dir=$HOME/.sge/intel
##    intel_dir=/opt/intel/licenses

#                 idev puts the default project in .idevrc
#                 pick it up here for the help option

  my_pid=$$

#                 Defaults
  MY_MINUTES=30
  MY_hhmmss_TIME=00:${MY_MINUTES}:00
  MY_H_RT="h_rt=$MY_hhmmss_TIME"
  MY_QUEUE=development

  RESERVATION_CONTROL="NO_OPTION"
  RESERVATION_NAME=""
  MY_RESERVATION=""
  RESERVATION_USE=no

JOB_NAME=idv$$
monitor_sleep=4
sleep_after_run_factor=4
job_sleep=$(( MY_MINUTES * 60 ))


#                                          SSH to master node by default

     use_node_type=master
     insert_option=none
      IDEV_ACCOUNT="-A  willpromptyou"
        MY_ACCOUNT=none
     check_project=yes
         nodes_set=no
   total_tasks_set=no
        queue_arch=standard
  
                                      #Make squeue look identical for ls5/stampede
   squeue_opts='-o %.18i %.15P %.9j %.8u %.2t %.10M %.6D %R' 

  if [ -e $HOME/.idevrc ]; then
#                                                 Get Project Name
     grep -q idev_project $HOME/.idevrc
     if [[ $? == 0 ]]; then                      #Pick up last occurence
        MY_ACCOUNT=`grep idev_project $HOME/.idevrc | tail -1 |sed 's/idev_project//'`
        MY_ACCOUNT="${MY_ACCOUNT%\\n}"               # get rid of return
        MY_ACCOUNT=`echo $MY_ACCOUNT | sed 's/ //g'` # get rid of spaces
        if [[ $MY_ACCOUNT =~ ^$ ]]; then
            echo "Please "
            echo "       remove idev_project line from $HOME/.idevrc"
            echo "and execute idev again."
            exit 1
        fi
           echo ""
           echo "Defaults file    : ~/.idevrc"
           check_project=no
        if [[ $MY_ACCOUNT =~ use_default ]]; then
           IDEV_ACCOUNT=" "
           echo "Project          : $MY_ACCOUNT "
        else
           IDEV_ACCOUNT="-A  $MY_ACCOUNT"
           echo "Default  project : $MY_ACCOUNT"
        fi
     fi

#                                                 Get time default
     grep -q idev_time $HOME/.idevrc
     if [[ $? == 0 ]]; then
        MY_TIME=`grep idev_time $HOME/.idevrc | sed 's/idev_time//'`
        MY_TIME="${MY_TIME%\\n}"               # get rid of return
        MY_TIME=`echo $MY_TIME | sed 's/ //g'` # get rid of spaces
          if [[ $MY_TIME =~ ":" ]]; then
                   MY_H_RT="h_rt=$MY_TIME"
                   regex='0*([[:digit:]]+)[:::]0*([[:digit:]]+)[:::]0*([[:digit:]]+)'
                   if [[ "$MY_H_RT" =~ $regex ]]; then
                     secs=$((${BASH_REMATCH[1]}*3600+${BASH_REMATCH[2]}*60+${BASH_REMATCH[3]}))
                   else
                     echo "ERROR:  default time syntax: time <#>."
                     echo "        Value found: \"$MY_TIME\""
                     exit
                   fi
                   MY_MINUTES=$(( secs / 60 ))
                   job_sleep=$secs
             #     if [[ $BATCH_SYS == "SGE" ]] ; then
             #        echo "default time     : -l $MY_H_RT"
             #     else
             #        MY_hhmmss_TIME=$MY_TIME
             #        echo "default time     : -t $MY_hhmmss_TIME"
             #     fi
          else
                   MY_MINUTES=$MY_TIME
                            MY_H_RT="h_rt=00:${MY_MINUTES}:00 "  #SGE/SLURM accept more than 2 digits in this field.
                          MY_hhmmss_TIME="00:${MY_MINUTES}:00 "  #SGE/SLURM accept more than 2 digits in this field.
             #     if [[ $BATCH_SYS == "SGE" ]] ; then
             #       echo "time (minutes)   : -l $MY_H_RT"
             #       echo "default time     : -l $MY_H_RT"
             #     else
             #       echo "default time     : -t $MY_hhmmss_TIME"
             #     fi
                   job_sleep=$(( $MY_MINUTES * 60 ))
          fi
          echo "Default  time    : $MY_MINUTES min."
     else
          echo "Default  time    : $MY_MINUTES min."

     fi

     grep -q idev_queue $HOME/.idevrc
     if [[ $? == 0 ]]; then
        MY_QUEUE=`grep idev_queue $HOME/.idevrc | sed 's/idev_queue//'`
        MY_QUEUE="${MY_QUEUE%\\n}"               # get rid of return
        MY_QUEUE=`echo $MY_QUEUE | sed 's/ //g'` # get rid of spaces
        [[ $MY_QUEUE =~ gpu    ]] && queue_arch=gpu
        [[ $MY_QUEUE =~ vis    ]] && queue_arch=gpu
        [[ $MY_QUEUE =~ serial ]] && queue_arch=serial
        echo "Default  queue   : $MY_QUEUE"
     else
        echo "Default  queue   : $MY_QUEUE"
     fi


     grep -q idev_rm_env $HOME/.idevrc
     if [[ $? == 0 ]]; then
        IDEV_RM_ENV=`grep idev_rm_env $HOME/.idevrc | sed 's/idev_rm_env//'`
        IDEV_RM_ENV="${IDEV_RM_ENV%\\n}"               # get rid of return
        IDEV_RM_ENV=`echo "$IDEV_RM_ENV" | sed 's/^ *//g' | sed 's/ *$//g'` # get rid of spaces
        echo "Removed env vars : $IDEV_RM_ENV"
        export IDEV_RM_ENV
     fi

  fi

# Determine my present working shell
 idev_ppid=`ps -l |  awk '{if ($4 == '$$'        ) print $5 }'`
 idev_cmd=` ps -l |  awk '{if ($4 == '$idev_ppid') print $14}'`

 if [[ $idev_cmd =~ (tcsh|csh) ]] ; then
    idev_pwshell=tcsh
 else
    idev_pwshell=bash
 fi


# Determine my LOGIN shell
 my_gecos=`grep $USER /etc/passwd`
 if [[ $my_gecos =~ (tcsh$|csh$) ]] ; then
    idev_login_shell=tcsh
    idev_pwd_cmd='setenv IDEV_PWD `pwd`'
    idev_env_set=setenv
    idev_env_equal=" "

                   # rm PERL5LIB for idev env_replication, but put in /tmp/env
    idev_rm_perl5="       
       setenv idev_has_user_PERL5LIB no
       if (\$?PERL5LIB && { eval 'test ! -z \$PERL5LIB' } ) then
          setenv idev_user_PERL5LIB \${PERL5LIB} && setenv idev_has_user_PERL5LIB yes
          unsetenv PERL5LIB
       endif
    "
 else
    idev_login_shell=bash
    idev_pwd_cmd='export IDEV_PWD=`pwd`'
    idev_env_set=export
    idev_env_equal="="
                   # rm PERL5LIB for idev env_replication, but put in /tmp/env
    idev_rm_perl5='        
       export idev_has_user_PERL5LIB=no;
       if [[ ! -z "$PERL5LIB" ]]; then
          export     idev_user_PERL5LIB=$PERL5LIB
          export idev_has_user_PERL5LIB=yes
          unset PERL5LIB
       fi
    '
 fi

if [[ $idev_pwshell -ne $idev_login_shell ]]; then
   echo " Warning:  Your present working shell is $idev_pwshell."
   echo "           Your login           shell is $idev_login_shell."
   echo "           Idev will place you in your login shell on the master node."
fi


MY_RESOURCE=""
CREATE_NODE_LIST_CMD=""

#OPERATION  Set batch system parameters.

JOB_NAME=idv$$
monitor_sleep=4
sleep_after_run_factor=4
job_sleep=$(( MY_MINUTES * 60 ))

login_sourcing=yes   #use .login/.profile sourcing code for creating batch env

case $MY_SYS in

   maverick) echo "System           : Maverick"
             MY_WAY=20;         MY_CORES=20
             MY_TOTAL_TASKS=20; MY_NODES=1;  CpN=20 # Cores/Node
             MY_PROJECT=""
             sleep_after_run=7
             ## -P  vis, data, gpgpu, hpc
             BATCH_SYS=SLURM
             Q="-p"
             MY_QUEUE=gpu
             IDEV_NODES=""    #Usually not set
             export IDEV_QDEL=scancel
             login_sourcing=by_tacc_etc_profile.d #Let TACC do this
             ;;
        ls4) echo "System           : Lonestar"
             MY_WAY=12; MY_CORES=12; CpN=12 # Cores/Node
             MY_PROJECT=""
             sleep_after_run=4
             BATCH_SYS=SGE
             Q="-q"
             export IDEV_QDEL=qdel
             ;;
   stampede) echo "System           : Stampede"
             MY_WAY=16;         MY_CORES=16; CpN=16 # Cores/Node
             MY_TOTAL_TASKS=16; MY_NODES=1;  CpN=16 # Cores/Node
             MY_PROJECT=""
             sleep_after_run=7
             BATCH_SYS=SLURM
             Q="-p"
             IDEV_NODES=""    #Usually not set
             export IDEV_QDEL=scancel
             login_sourcing=by_tacc_etc_profile.d #Let TACC do this
             ;;
        ls5) echo "System           : Lonestar"

             if [[ $queue_arch == standard ]]; then
                MY_WAY=24;         MY_CORES=24; CpN=24 # Cores/Node
                MY_TOTAL_TASKS=24; MY_NODES=1;  CpN=24 # Cores/Node
             fi

             if [[ $queue_arch == gpu ]]; then
                MY_WAY=10;         MY_CORES=10; CpN=10 # Cores/Node
                MY_TOTAL_TASKS=10; MY_NODES=1;  CpN=10 # Cores/Node
             fi

             if [[ $queue_arch == serial ]]; then
                MY_WAY=1;          MY_CORES=1;  CpN=24 # Cores/Node
                MY_TOTAL_TASKS=1;  MY_NODES=1;  CpN=24 # Cores/Node
             fi

             MY_PROJECT=""
             sleep_after_run=7
             BATCH_SYS=SLURM
             Q="-p"
             IDEV_NODES=""    #Usually not set
             export IDEV_QDEL=scancel
             login_sourcing=by_tacc_etc_profile.d #Let TACC do this

             MY_QUEUE=development

             ;;
          *) echo " Don't know about this system ($MY_SYS)"
             exit 1
             ;;
esac

######

do_exit=no
check_project=yes
cmd_args=$@

#     Get command line arguments -- they override everything
OPT_ERR=0

while (( "$#" )); do

#                Non-installation sensitive options here

   known_option=no

   if [[ x$1 == x-help ||  x$1 == x--help  || x$1 == x-h || x$1 == xhelp ]]; then
                  known_option=yes
                  echo ""
                  echo ""
                  echo "Usage:  idev [OPTIONS]"
                  echo " "
                  echo "idev creates an interactive session on a compute node"
                  echo "for executing serial, openmp-parallel, or mpi-parallel"
                  echo "code as you would in a batch job."
                  echo "Supported systems: Lonestar, Longhorn and Stampede."
                  echo ""
                  echo "Idev uses a single node by default and charges the project in \$HOME/.idevrc."
                  echo "Options can be in any order."
                  echo ""
                  echo "For Lonestar:>>"
                  echo "Use a login node window for MPI compiling, and an idev window for running."
                  echo "The mpicc/mpif90 compiler does not work correctly on Lonestar compute nodes."
                  echo "(You can compile openmp and serial code with intel compilers, though).<<"
                  echo " "
                  echo "OPTIONS:"
                  echo " OPTION  ARGUMENTS          DESCRIPTION"
                  echo ""
                  echo "  -A     account_name       sets account name (default: -A $MY_ACCOUNT)"
                  echo "  -m     minutes            sets time in minutes (default: $MY_MINUTES)"
if [[ $BATCH_SYS == SLURM ]]; then
                  echo "  -n     total_tasks        Total number of tasks"
                  echo "  -N     nodes              Number of nodes"
fi
if [[ $BATCH_SYS == SGE ]]; then
                  echo "  -pe    way cores          sets wayness & cores (default 1 node: -pe $MY_WAY $MY_CORES)"
fi
if [[ $MY_SYS == longhorn ]]; then
                  echo "  -P     project_name       project type for Longhorn {vis, data, gpgpu, or hpc}"
fi
                  echo "  $Q     queue_name         sets queue to named queue (default: $Q $MY_QUEUE)"
if [[ $BATCH_SYS == SGE ]]; then
                  echo "  -r     resource_name      selects hardware "
                  echo "                            (e.g. -r mem_total=250G or -r h=<specific nodes>)"
                  echo "                            added to -l option as h=<name> (no default)"
fi

if [[ $BATCH_SYS == SLURM ]]; then
                  echo "  -r                        (no res. name) find and use my ACTIVE reservation (no questions)"
                  echo "  -r     reservation_name   requests use of a specific reservation"
                  echo "  -r     none               idev always automatically looks for reserations"
                  echo "                            and asks you to use an ACTIVE res. none==don't look/ask."
                  echo "                            (use    scontrol show reserations to see reservation details)"
fi

                  echo "  -t     hh:mm:ss           sets time to hh:mm:ss (default: 00:${MY_MINUTES}:00, 30 min.)"
                  echo ""
if [[ $BATCH_SYS == SGE ]]; then
                  echo "  -rank0     [--rank0    ]  login to rank 0 node (may not be the master node)"
                  echo "                            TACC users: no need to use this since NO_HOSTSORT "
                  echo "                            env. var is set and task0 will be on master node."
                  echo "  -uninstall [--uninstall]  removes idev login setup and \$HOME/.idevrc file."
fi
                  echo "  -help [help, --help, -h]  displays (this) help message"
                  echo "  -v         [ --version ]  output version information and exit"
                  echo ""
                  echo "  Syntax and Examples:"
if [[ $BATCH_SYS == SGE ]]; then
                  echo "       Syn: idev     (defaults: 1 node, full wayness, $MY_MINUTES min., default account.)"
                  echo "       Syn: idev  [-help | -v | -uninstall] "
                  echo "       Syn  idev  [-q que.] [-m min.] [-t time ] [-pe way cores] \ "
                  echo "                  [-A acct] [-r res.] [-P proj.]"

                  echo "       ex1  idev -pe 12 12  -q development -A A-ccvis  -m 90 "
                  echo "       ex2  idev -pe 12 96  -q normal                  -t 00:90:00 "
                  echo "       ex3  idev -P vis                                   (a Longhorn Project type)"
                  echo "       ex4  idev"
                  echo "       ex5  idev -help"
fi
if [[ $BATCH_SYS == SLURM ]]; then
                  echo "       Syn: idev     (defaults: 1 node, 16 tasks, $MY_MINUTES min., default account.)"
                  echo "       Syn: idev  [-help | -v | -uninstall] "
                  echo "       Syn  idev  [-p queue]  [-m min.] [-t time ]  [-A acct] \ "
                  echo "                  [-n tasks]  [-N nodes  -n tasks]  "
                  echo ""
                  echo "       ex4  idev"
                  echo "       ex5  idev -help"
                  echo "       ex1  idev -p development -m 90       -A A-ccvis        -n 16"
                  echo "       ex2  idev -p normal      -t 00:90:00 -A TG-STA123 -N 2 -n 16 "
fi

                  echo ""
                  echo "Idev ssh's to compute nodes with the -Y option to allow X forwarding for debugging."
                  echo ""
                  echo "Insert  \"idev_time MINUTES\",  \"idev_queue QUEUE\" or  \"idev_project PROJECT\" "
                  echo "       without quotes, one item per line in ~/.idevrc to set your own default "
                  echo "       time, queue and project.  (Substitute values for MINUTES, QUEUE and PROJECT.)"
                  echo "       E.g.  idev_time 60  <new line> idev_queue normal " 
                  echo ""
                  echo "Insert  \"idev_rm_env VAR_LIST\" "
                  echo "        in ~/.idevrc, where VAR_LIST is a space separated list of environment variables "
                  echo "        not to be replicated from the job environment initialized by idev.  "
                  echo "        Note: all SSH_xx variables are removed by default so that all interactive "
                  echo "              ssh terminal sessions have their correct SSH_xx values for a new session."
#                 echo ""
                  do_exit=yes; OPT_ERR=0
   fi

#                                           Login to rank 0 node, may not be the master node.
   if [[ x$1 == x-version || x$1 == x--version || x$1 == x-v ]]; then
                  known_option=yes
                  echo "Version          : $IDEV_VER"
                  do_exit=yes; OPT_ERR=0
   fi
#                                           Remove login setup from bash/tcsh files, remove $HOME/.idevrc
   if [[ x$1 == x-uninstall || x$1 == x--uninstall ]]; then
                  known_option=yes
                  insert_option=uninstall
   fi

   if [[ x$1 == x-n ]]; then
                  known_option=yes
                   if [[ $BATCH_SYS == "SLURM" ]] ; then
                     shift; MY_TOTAL_TASKS=$1;  total_tasks_set=yes
                     echo "Using total tasks: -n $MY_TOTAL_TASKS"
                   else
                     echo -e "\n ERROR: **The -N option is not allowed on $MY_SYS."
                     echo -e "     It is not a valid $BATCH_SYS option."
                     echo -e "     See options, execute: idev -help"
                     do_exit=yes; OPT_ERR=1
                   fi
   fi

   if [[ x$1 == x-N ]]; then
                  known_option=yes
                   if [[ $BATCH_SYS == "SLURM" ]] ; then
                     shift; MY_NODES=$1; IDEV_NODES="-N $1"; nodes_set=yes
                     echo "Using nodes      : -N $MY_NODES"
                   else
                     echo -e "\n ERROR: **The -N option is not allowed on $MY_SYS."
                     echo -e "     It is not a valid $BATCH_SYS option."
                     echo -e "     See options, execute: idev -help"
                     do_exit=yes; OPT_ERR=1
                   fi
   fi


   if [[ x$1 == x-pe ]]; then
                  known_option=yes
                   if [[ $BATCH_SYS == "SGE" ]] ; then
                     shift; myway=$1
                     shift; MY_CORES=$1
                     MY_WAY=`echo $myway | sed s/way//`
                     echo "Using Prog Env   : -pe ${MY_WAY}way $MY_CORES"
                     MY_NODES=$(( MY_CORES / CpN ))
                     EXTRA_TIME=`echo "scale=0;  l($MY_NODES)/l(10)" | bc -l`
                     EXTRA_TIME=$(( MY_NODES * EXTRA_TIME / CpN / sleep_after_run_factor  ))
  
                     sleep_after_run=$(( sleep_after_run + EXTRA_TIME ))
                   else
                     echo -e "\n ERROR: **The -pe option is not allowed on $MY_SYS."
                     echo -e "     It is not a valid $BATCH_SYS option."
                     echo -e "     See options, execute: idev -help"
                     do_exit=yes; OPT_ERR=1
                   fi

   fi

   if [[ x$1 == x-q ]]; then
                  known_option=yes
                   shift; MY_QUEUE=$1
                   if [[ $BATCH_SYS == "SGE" ]] ; then
                      echo "Using queue      : -q $MY_QUEUE"
                   else
                      echo "Using queue      : -p $MY_QUEUE"
                      CMDLINE_QUEUE=$MY_QUEUE
                      queue_from=cmdline
                   fi

   fi

   if [[ x$1 == x-p ]]; then
                  known_option=yes
                   shift; MY_QUEUE=$1
                   if [[ $BATCH_SYS == "SGE" ]] ; then
                      echo "Using queue      : -q $MY_QUEUE"
                   else
                      echo "Using queue      : -p $MY_QUEUE"
                      CMDLINE_QUEUE=$MY_QUEUE
                      queue_from=cmdline
                   fi

   fi

   if [[ x$1 == x-A ]]; then
                  known_option=yes
                   shift; IDEV_ACCOUNT="-A  $1"
                   check_project=no
                   echo "Using Project    : $IDEV_ACCOUNT"

   fi

   if [[ x$1 == x-t ]]; then
                  known_option=yes
                   shift; MY_H_RT="h_rt=$1"
                   regex='0*([[:digit:]]+)[:::]0*([[:digit:]]+)[:::]0*([[:digit:]]+)'
                   if [[ "$MY_H_RT" =~ $regex ]]; then
                     secs=$((${BASH_REMATCH[1]}*3600+${BASH_REMATCH[2]}*60+${BASH_REMATCH[3]}))
                   else
                     echo "ERROR:  h_rt (syntax hr:min:sec) requires value for hr, min and sec entry."
                     echo "        Value found: \"$MY_H_RT\""
                     exit
                   fi
                   job_sleep=$secs
                   if [[ $BATCH_SYS == "SGE" ]] ; then
                      echo "time (h_rt)      : -l $MY_H_RT"
                   else
                      MY_hhmmss_TIME=$1
                      echo "time (hh:mm:ss)  : -t $MY_hhmmss_TIME"
                   fi
   fi

   if [[ x$1 == x-m ]]; then
                  known_option=yes
                   shift; MY_MINUTES=$1
                          MY_H_RT="h_rt=00:${MY_MINUTES}:00 "  #SGE/SLURM accept more than 2 digits in this field.
                          MY_hhmmss_TIME="00:${MY_MINUTES}:00 "  #SGE/SLURM accept more than 2 digits in this field.
                   if [[ $BATCH_SYS == "SGE" ]] ; then
                     echo "time (minutes)   : -l $MY_H_RT"
                   else
                     echo "time (minutes)   : -t $MY_hhmmss_TIME"
                   fi
                   job_sleep=$(( $MY_MINUTES * 60 ))
   fi

   if [[ x$1 == x-rank0 || x$1 == x--rank0 ]]; then
                  known_option=yes
                   if [[ $BATCH_SYS == "SGE" ]] ; then
                     use_node_type=rank0
                     CREATE_NODE_LIST_CMD="ibrun date >& /dev/null  #creates $HOME/.sge/*job*hostlist.* file"
                     echo "SSHing to        :  rank 0 node"
                   else
                     echo -e "\n **The -rank0 option is not allowed on $MY_SYS."
                     echo -e "     It is not a valid $BATCH_SYS option."
                     echo -e "     See options, execute: idev -help"
                     do_exit=yes; OPT_ERR=1
                   fi
   fi

   if [[ x$1 == x-r ]]; then
                  known_option=yes

                   if [[ $BATCH_SYS == "SGE" ]] ; then
                      shift; MY_RESOURCE="-l  $1"
                      echo "hardware request : $MY_RESOURCE"
                   fi

                   if [[ $BATCH_SYS == "SLURM" ]] ; then


                      if [[ x$2 = x ]]; then
                         # We are at the end of the line without a 
                         # reservation name-- meaning we will look for any reservation.
                         RESERVATION_CONTROL="FIND_IT"
                         echo "Reservation      : Will search and request it for you."
                      else
                         if [[ x$2 =~ "x-" ]]; then
                            # We have found the next option.  No reservation name
                            # means we look for the reservation.
                            RESERVATION_CONTROL="FIND_IT"
                            echo "Reservation      : Will find it for you."
                         else
                            # Found reservation request with name
                            shift
                            if [[ $1 =~ none ]]; then
                               RESERVATION_CONTROL="NONE"
                               MY_RESERVATION=""
                               echo "Reservation      : $1"
                            else
                               RESERVATION_CONTROL="USER_SET"
                               RESERVATION_NAME=$1
                               MY_RESERVATION="--reservation=$1"
                               echo "Reservation      : $1"
                               MY_RESERVATION="#SBATCH $MY_RESERVATION"
                            fi
                         fi

                      fi

                   # echo -e "\n ERROR: **The -r option is not allowed on $MY_SYS."
                   # echo -e "     It is not a valid $BATCH_SYS option."
                   # echo -e "     See options, execute: idev -help"
                   # do_exit=yes; OPT_ERR=1
                   fi
   fi

   if [[ x$1 == x-P ]]; then
                  known_option=yes
                   if [[ $MY_SYS == "longhorn" ]] ; then
                     shift; MY_PROJECT="-P  $1"
                     echo "Longhorn project type : $MY_PROJECT"
                   else
                     echo -e "\n ERROR: **The -P option is only allowed on Longhorn."
                     echo -e "          To see options, execute: idev -help."
                     do_exit=yes; OPT_ERR=1
                   fi
   fi

   if [[ $known_option == "no" ]]; then
                     echo -e "\n ERROR: **The $1 option is not known."
                     echo -e "          To see options, execute: idev -help."
                     do_exit=yes; OPT_ERR=2
            
   fi


   shift

done


############################################beg  Dependent options

 [[ $MY_QUEUE =~ gpu    ]] && queue_arch=gpu
 [[ $MY_QUEUE =~ vis    ]] && queue_arch=gpu
 [[ $MY_QUEUE =~ serial ]] && queue_arch=serial


if [ x$MY_SYS = "xls5" ] &&  
   [ x$queue_arch = "xgpu" ] && 
   ( [ x$total_tasks_set = "xno" -a x$nodes_set = "xno" ]); then
   MY_WAY=10;         MY_CORES=10; CpN=10 # Cores/Node
   MY_TOTAL_TASKS=10; MY_NODES=1;  CpN=10 # Cores/Node
fi

if [ x$MY_SYS = "xls5" ] &&  
   [ x$queue_arch = "xserial" ] && 
   ( [ x$total_tasks_set = "xno" -a x$nodes_set = "xno" ]); then
   MY_WAY=1;          MY_CORES=1;  CpN=24 # Cores/Node
   MY_TOTAL_TASKS=1;  MY_NODES=1;  CpN=24 # Cores/Node
fi

if [ x$MY_SYS = "xls5" ] &&  
   [ x$queue_arch = "xserial" ] && 
   [ x$nodes_set  = "xyes"    ]; then
      echo " "
      echo " ERROR: -N resource option MUST NOT BE USED in $MY_QUEUE queue."
      echo "        This is a serial queue, only -n 1 should be specified."
      do_exit=yes; OPT_ERR=1
fi

# SLURM on Stampede & LS5.  If -N is provided, but -n is not, disallow

if [[ $total_tasks_set == no && $nodes_set == yes ]] ; then

   echo -e "\n ERROR: ** -n option (total tasks) is not set. **"
   echo -e "           When using the -N option, the -n option is required."
   echo -e "           (Same as batch requirements--See Stampede User Guide.)"
   do_exit=yes; OPT_ERR=1

fi

#      if grace-serial queue     
if [[ $MY_QUEUE =~ grace-serial ]]; then

  squeue -p $MY_QUEUE "$squeue_opts" -u $USER | \
            awk '$3 ~ /idv[0-9][0-9][0-9][0-9][0-9]/' | \
            awk '{print $5}' | grep R >/dev/null
  if [ $? = 0 ]; then
    using_node=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | \
            awk '$3 ~ /idv[0-9][0-9][0-9][0-9][0-9]/' | \
            awk '{print $8}' `
    echo ""
    echo " ERROR:  Only one ONE idev interactive session allowed in $MY_QUEUE queue."
    echo ""
    echo " You are executing idev again in the $MY_QUEUE queue."
    echo "    In this queue only 1 idev interactive job is allowed."
    echo "    You may have multiple windows open to the job (node) for an active"
    echo "    IDEV session by opening a new window on a login node, and SSHing"
    echo "    to the node of the job with the command:"
    echo ""
    echo "                 ssh -Y $using_node   "
    echo ""
    echo " In a new window you can customize your environment if"
    echo " you want to work interactively on different projects."  
    echo ""
    do_exit=yes; OPT_ERR=2
  fi
fi


############################################end  Dependent options


#                         Bail out if not launching or arg error
#                         Only last error value is reported in trap.
#                         But all error messages reported (above).
#------------------------------------------------------------------------------
[[ $do_exit == "yes" ]] && exit $OPT_ERR
#------------------------------------------------------------------------------

  DIR=`dirname $0`       #Need directory where idev & utils are
  [ x$? != x0 ] && exit 1;

  idev_dir=$DIR         #Need full path for PATH in login scripts
  [[ ! $idev_dir =~ ^/ ]] && idev_dir=$PWD/$idev_dir

############################################beg  RESERVATION

# Values of RESERVATION_CONTROL
# -r      --> FIND_IT
# -r none --> NONE
# -r xyz  --> USER_SET
#         --> NO_OPTION
#   tot tot_active  status         name queue
#    1   0/1        act/nonactive               1 doit, 0 advise
#    2+  1          ---                         doit 1 active
#    3+  0,2...     ---                         Advise

if [[ $BATCH_SYS == SLURM ]]; then

RESERVATION_NAME_FOUND=no

 if [[ $RESERVATION_CONTROL == NONE ]]; then
    # do nothing-- If the user set it, good luck.
    echo doing_nothing>/dev/null
 else
    reservation_string=`$DIR/idev_utils get_reservation $USER`;
    reservation_array=( $reservation_string )
    res_totl=${reservation_array[0]}
    res_actv=${reservation_array[1]}
    res_stat=${reservation_array[2]}
    res_name=${reservation_array[3]}
    res_queue=${reservation_array[4]}
 #                          (If we see 1 reservation all is done.)
 
 #                          Get ACTIVE reservation  information here.
   #if [[ $res_totl -ge 2 && $res_actv -le 1 ]]; then
    if [[ $res_totl -ge 2 ]]; then
       for((i=0;i<${#reservation_array[@]};i++))
       do
 #                         Find 1 ACTIVE reservation.
 #                         For res_actv=0, nothing changes since ACTIVE not found.
 #                         We avoid the case for 2 or more ACTIVE reservations (later).
           #echo "RES_NAME $i ${reservation_array[$i]}"
           if [[ ${reservation_array[$i]} == ACTIVE ]]; then
             res_stat=${reservation_array[$i]}
             res_name=${reservation_array[  (($i+1)) ]}
             res_queue=${reservation_array[ (($i+2)) ]}
             RESERVATION_USE=yes
           fi
       done
 
 #                         We validate what the user has set a valid name.
 #                         This many not be an (the) active reservation!
       if [[ $RESERVATION_CONTROL == USER_SET ]]; then
       for((i=0;i<${#reservation_array[@]};i++))
       do
          if [[ ${reservation_array[$i]} == $RESERVATION_NAME ]]; then
             RESERVATION_NAME_FOUND=yes 
             RESERVATION_STATUS=${reservation_array[ (($i-1)) ]}
             RESERVATION_QUEUE=${reservation_array[  (($i+1)) ]}
             if [[ $RESERVATION_STATUS == INACTIVE && $res_actv == 1 ]]; then
                echo " NOTE: you have requested a INACTIVE reservation ($RESERVATION_NAME)."
                echo " NOTE: $res_name is your ACTIVE reservation." 
 
                echo " Do you really want to wait on a INACTIVE reservation?  (suggested answer: no)"
                echo -n " Enter y/n [default y]: "
                read -s -n 1 yorn
                echo ""
                if [[ $yorn != y && $yorn != n ]]; then  yorn=y; fi
  
                if [[ $yorn == y ]]; then
                   res_stat=INACTIVE
                   res_name=$RESERVATION_NAME
                   res_queue=$RESERVATION_QUEUE
                   RESERVATION_USE=yes
                else 
                   echo " Adios.  Use \"scontrol show reservations\" to view reservations."
                   exit 1
                fi
             fi
          fi
       done
       fi
 
    fi
 
 
    if [[ $res_totl -eq 0 ]]; then
       if [[ $RESERVATION_CONTROL == USER_SET ]]; then
         echo " No reservation named $RESERVATION for $USER was found."
         echo " Execute: \"scontrol show reservations\"  to see reservations."
         exit 1;
       fi
 
       if [[ $RESERVATION_CONTROL == FIND_IT ]]; then
         echo " No reservation for $USER was found."
         echo " Execute: \"scontrol show reservations\"  to see reservations."
         exit 1;
       fi
 
      # not here
      #if [[ $res_name != $RESERVATION_NAME ]]; then
      #echo " Please correct and try again."
      #else #slide on by
    fi
 
    if [[ $res_totl -eq 1 || (( $res_totl -ge 2 && $res_actv -eq 1 )) ]]; then
 
       if [[ $res_actv -eq 1  ]]; then

          RESERVATION_USE=yes
 
          # ACTIVE Reservation
             if [[ $RESERVATION_CONTROL == USER_SET ]]; then
                if [[ $res_name != $RESERVATION_NAME ]]; then
                   echo " ERROR:"
                   echo " idev reservation name (-r $RESERVATION_NAME) "
                   echo " does not match slurm reservation name ($res_name)."
                   echo " Aborting idev.  Execute: \"idev -r\" and idev will find the name for you."
                   echo " Execute: \"scontrol show reservations\"  to see reservations."
                   exit 1;
                else           #good to go
 
                   echo "Reservation      : --reservation=$res_name ($res_stat)"
                   RESERVATION_NAME=$res_name
                   MY_RESERVATION="#SBATCH --reservation=$res_name"
                   MY_QUEUE=$res_queue
                   RESERVATION_USE=yes

                fi
             fi
 
            
             if [[ $RESERVATION_CONTROL == FIND_IT ]]; then
                   echo "Reservation      : --reservation=$res_name (ACTIVE)"
                   RESERVATION_NAME=$res_name
                   MY_RESERVATION="#SBATCH --reservation=$res_name"
                   MY_QUEUE=$res_queue
                   RESERVATION_USE=yes
#CHECK
#                  echo "Using res. queue : -p $MY_QUEUE"
             fi
             if [[ $RESERVATION_CONTROL == NO_OPTION ]]; then
                   echo " We found an ACTIVE reservation request for you, named $res_name."
                   echo " Do you want to use it for your interactive session?"
                   echo -n " Enter y/n [default y]: "
                   read -s -n 1 yorn
                   echo ""
                   if [[ $yorn != y && $yorn != n ]]; then  yorn=y; fi
 
                   if [[ $yorn == y ]]; then 
                      echo "Reservation      : --reservation=$res_name (ACTIVE)"
                      RESERVATION_NAME=$res_name
                      MY_RESERVATION="#SBATCH --reservation=$res_name"
                      MY_QUEUE=$res_queue
                      RESERVATION_USE=yes
#CHECK
#                     echo "Using res. queue : -p $MY_QUEUE"
                    else  #Continue on as a non-reservation request.
                       echo " Continuing without using your reservation."
                       echo " Use \"scontrol show reservations\" to view your reservation(s)."
                       RESERVATION_USE=no 
                   fi
             fi
       else        ##INACTIVE RESERVATION
          
          if [[ $RESERVATION_CONTROL == USER_SET || $RESERVATION_CONTROL == FIND_IT ]]; then
 
             echo "Reservation      : --reservation=$res_name (INACTIVE)"
             RESERVATION_NAME=$res_name
             MY_RESERVATION="#SBATCH --reservation=$res_name"
             MY_QUEUE=$res_queue
             RESERVATION_USE=yes
#CHECK
#            echo "Using res. queue : -p $MY_QUEUE"
 
             echo ""
             echo "  ****** YOU HAVE AN INACTIVE RESERVATION ******"
             echo ""
             if [[ $RESERVATION_CONTROL == FIND_IT ]]; then
             echo "  Meaning your reservation period has not started yet."
             echo "  You might want to control-c out of this session,"
             echo "  and execute idev without the -r option to access nodes that are available."
             fi
             if [[ $RESERVATION_CONTROL == USER_SET ]]; then
             echo "  You explicitly requested this, so idev will proceed, as you wish."
             echo "  Execute control-c if you get tired of waiting."
             fi
             echo "  To see when the reservation begins, execute:  scontrol show reservations"
          fi
 
          if [[ $RESERVATION_CONTROL == NO_OPTION ]]; then
             echo " "
             echo " Just a Note: Your reservation $res_name is INACTIVE."
             echo " Use idev -r to use it when it becomes ACTIVE."
             echo " To see when it begins, execute:  scontrol show reservations"
             echo " "
             RESERVATION_USE=no
          fi
 
       fi
    fi

    if [[ $res_totl -ge 2 &&  $res_actv -ne 1 ]]; then
 
       # Don't process: $RESERVATION_CONTROL == NO_OPTION
 
        if [[ $RESERVATION_CONTROL == USER_SET ]]; then
           if [[ $RESERVATION_NAME_FOUND != yes ]]; then
              echo " idev didn't find a reservation ($RESERVATION_NAME)."
              echo " Execute: \"scontrol show reservations\"  to see reservations."
              exit 1
           else           #good to go
              if [[ $RESERVATION_STATUS == INACTIVE ]]; then
 
                 echo " We found your reservation to be INACTIVE."
                 echo " idev will not run until it becomes ACTIVE."
                 echo " Do you want to wait on it?  We suggest no."
                 echo -n " Enter y/n [default y]: "
                 read -s -n 1 yorn
                 echo ""
                 if [[ $yorn != y && $yorn != n ]]; then  yorn=y; fi
    
                 if [[ $yorn == y ]]; then 
                    echo "Reservation      : --reservation=$RESERVATION_NAME (INACTIVE)"
                    MY_RESERVATION="#SBATCH --reservation=$RESERVATION_NAME"
                    MY_QUEUE=$RESERVATION_QUEUE
#CHECK
#                   echo "Using res. queue : -p $MY_QUEUE"
                    RESERVATION_USE=yes
                 else 
                    echo " Adios.  Use \"scontrol show reservations\" to view reservations."
                    exit 1
                 fi
              else
          
                 #But USER_SET assures us one particular reservation is requested.
                 echo "Reservation      : --reservation=$RESERVATION_NAME (ACTIVE)"
                 MY_RESERVATION="#SBATCH --reservation=$RESERVATION_NAME"
                 MY_QUEUE=$RESERVATION_QUEUE
#CHECK
#                echo "Using res. queue : -p $MY_QUEUE"
                 RESERVATION_USE=yes
              fi
           fi
        fi
 
 
       if [[  $RESERVATION_CONTROL == FIND_IT ]]; then
          if [[ $res_actv -gt 1  ]]; then
             echo -e "\n You have $res_actv ACTIVE reservations."
             echo -e " I'm at a loss as to which one idev should use."
             echo -e " Please execute idev with the -r <reservation> -p <queue> options."
             echo -e " To see your ACTIVE reservations, execute:  scontrol show reservations."
             exit 1
          fi
          if [[ $res_actv -eq 0  ]]; then
             echo -e "\n You have NO ACTIVE reservations and $res_totl INACTIVE reservations."
             echo -e " I'm at a loss as to which one should be used."
             echo -e " Please execution idev with the -r <reservation> -p <queue> options."
             echo -e " To see your reservations, execute:  scontrol show reservations."
             exit 1
          fi
       fi

       if [[  $RESERVATION_CONTROL == NO_OPTION  ]]; then
          if [[ $res_actv -gt 1  ]]; then
             echo -e "\n NOTE: **** You have $res_actv ACTIVE reservations.        ****"
             echo -e "         **** Idev expects that you want to use one of them. ****"
             echo -e " If you want to use one of these, execute idev -r <reservation_name>"
             echo -e " To see your reservations, execute:  scontrol show reservations."
             echo -e " Use \"idev -r none\" to avoid this message and NOT exit idev."
             exit 1
          else 
             echo -e "\n FYI: You have $res_totl INACTIVE reservations."
             echo -e "      Idev is not asking to use them because they are INACTIVE."
             echo -e "      If you want to wait on one of them, re-execute idev with the -r option."
             echo -e "\n"
          fi
       fi

    fi
 
 fi
fi 
 # Array info, etc. 
 #http://stackoverflow.com/questions/6723426/looping-over-arrays-printing-both-index-and-value
 #http://tldp.org/HOWTO/Bash-Prog-Intro-HOWTO-10.html
 #http://tldp.org/LDP/abs/html/comparison-ops.html
 #http://tldp.org/HOWTO/Bash-Prog-Intro-HOWTO.html#toc10

############################################end  RESERVATION






#################beg POST RESERVATION
################
#                                       Make adjustment if reservation doesn't have a queue.
##                                       -- when queue is '(null)'
                   if [[ $res_queue == '(null)' && $RESERVATION_USE == yes ]]; then
                      if [[ $queue_from == cmdline ]]; then

                         MY_QUEUE=$CMDLINE_QUEUE
                         echo "Cmdline    queue : -p $MY_QUEUE"

                      else

                         echo "Reser.     queue : $res_queue"
                         echo " "
                         echo " Error: ** You must specify a queue for this reservation. **"
                         echo "           NO queue was assigned when the reservation was made "
                         echo "           and NO queue is specified on the idev command line."
                         echo "           Specify a queue (-p option) that contains the nodes"
                         echo "           of your reservation, or ask the person who made the"
                         echo "           reservation to include a queue for the nodes."

                         exit 1
                      fi
                  #else
##                #                     Else, use the queue of the reservation
                  #   MY_QUEUE=$CMDLINE_QUEUE
                  #   MY_QUEUE=$res_queue
                  #   echo "Using res. queue : -p $MY_QUEUE"

                   fi

#  If$MY_QUEUE='(null)' and does not have -p abort.
#

#################beg POST RESERVATION

if [[ $login_sourcing == "yes" ]] ; then     #login sourcing
  #SETUP  Make sure user has the idev login commands.

  if [[  -n $IDEV_SETUP_BYPASS ]]; then
    if [   "$IDEV_SETUP_BYPASS" != "$IDEV_SETUP_VER" ]; then
      echo "The idev shell setup has changed to version $IDEV_SETUP_VER."
      echo "Please read about the changes in the idev script, make appropriate"
      echo "adjustments, and set your IDEV_SETUP_BYPASS var to $IDEV_SETUP_VER."
      exit 1
    fi
  else
    $DIR/idev_utils ${idev_login_shell}_login_insertion $MY_SYS $idev_dir
  fi

  if [[ $insert_option == uninstall ]]; then
                  echo " Uninstalling..."
        $DIR/idev_utils ${idev_login_shell}_login_insertion $MY_SYS $idev_dir $insert_option
                  echo " Uninstall Complete."
                 #do_exit=yes
        if [[ ! -z $idev_login_ver ]] ; then
           echo "Remember to LOGOUT!"
           echo "Remember to LOGOUT!"
           echo "Remember to LOGOUT!"
           echo "Remember to LOGOUT!"
           echo "Remember to LOGOUT!"
        fi
                  exit 0
  fi
fi # end of login sourcing

###################################  Project INFO Begin

if [[ $check_project == yes ]]; then

  #                             If no project in HOME/.idevrc
  #                             or -A option not used on cmd line
  #                             Submit a bogus job to get projects
  #                             This is the best I can do as a user!!
  #                             Hmmm.  I cannot redirect error on this.

 if [[ $MY_ACCOUNT == none || -z $MY_ACCOUNT ]]; then
     echo "We need a project to charge for interactive use.";
     echo "We will be using a dummy job submission to determine your project(s).";
     echo "We will store your (selected) project \$HOME/.idevrc file.";
     echo ""
  #                             Submit bogus job
if [[ $BATCH_SYS == SGE ]]; then
qsub <<EOF 2>&1 >/tmp/idev_acct_job_output_$my_pid
#$ -V
#$ -pe 1way $MY_CORES
#$ -q $MY_QUEUE
#$ -l h_rt=00:01:00
#$ $MY_RESOURCE
#$ $MY_PROJECT
EOF

     echo "Please disregard above 2 lines from the qsub utility."
     echo ""

fi

if [[ $BATCH_SYS == SLURM ]]; then
   test_queue=normal
   if [[ $MY_SYS == maverick ]]; then
      test_queue=gpu
   fi
cat <<EOF >/tmp/idev_acct_job_$my_pid
#!/bin/bash
#SBATCH -J acct_tes
#SBATCH -o /dev/null
#SBATCH -p $test_queue
#SBATCH -n $CpN
#SBATCH -t 00:01:00
EOF

sbatch   /tmp/idev_acct_job_$my_pid >& /tmp/idev_acct_job_output_$my_pid
#rm -rf  /tmp/idev_acct_job_$my_pid

fi

  #                            Extract Projects from file in /tmp
  #                            (Get_projects.pl script removes it.)
  #  echo "HERE   idev_utils get_projects $my_pid ";
     project_string=`$DIR/idev_utils get_projects $my_pid`;
     project_array=( $project_string )

  #                            Now ask user which one he wants to use if count>1.
     if [[ ${#project_array[@]} > 1 ]]; then

       echo "Please select the NUMBER of the project you want to charge.\n";
       count=1;
       count=1;
       for i in "${project_array[@]}"
       do
          echo $count $i
          count=$(( $count + 1 ))
       done

       echo "Please type the NUMBER(default=1) and hit return: ";
       read project_no
       if [[ "$project_no" == "" ]]; then project_no=1; fi

       if [ $project_no -eq $project_no 2> /dev/null ]; then
         echo ""  #keep shell happy
       else
         echo "You didn't type a number.  You typed $project_no. Try again."
         exit 1
       fi

       echo "$project_no is the project number you selected."

       if [[ $project_no -gt 0 && $project_no -le ${#project_array[@]} ]]; then
  #    if [[ $project_no > 0 && $project_no <= ${project_array[@]} ]]; then
         array_no=$(( $project_no - 1));
         echo " OK I'll use ${project_array[$array_no]}. Thanks. "
         echo "idev_project ${project_array[$array_no]}" >> $HOME/.idevrc
         MY_ACCOUNT=${project_array[$array_no]}
       else
         echo "$project_no is not a valid number, please try again."
         exit 1
       fi

       IDEV_ACCOUNT="-A  $MY_ACCOUNT"

     else
     #  echo " You have a single project named ${project_array[0]}."
     #  MY_ACCOUNT=${project_array[0]}
     #  echo "idev_project ${project_array[$array_no]}" >> $HOME/.idevrc
        echo " You have a single project which we will use as the default ."
        MY_ACCOUNT=${project_array[0]}
        echo "idev_project ${project_array[$array_no]}" >> $HOME/.idevrc

        IDEV_ACCOUNT=" "
     fi

     rm -rf /tmp/idev_acct_job_output_$my_pid
 fi
fi


###################################  Project INFO End
#################################### Latest Setup Check Begin

#               Make sure user is using the latest setup.
   if [[ ! -z $idev_login_ver ]] ; then                        #if defined-> using earlier version
      echo " We have improved idev! But it will be necessary"  #using IDEV_SETUP_VER now
      echo " to uninstall your initial idev setup. Please execute:"
      echo ""
      echo "              idev -uninstall"
      echo ""
      echo " AND THEN LOGOUT. Login and then execute \"idev\" to reinstall."
      echo " "
      exit
   fi
#################################### Setup Check End

if [[ $BATCH_SYS == SGE ]]; then

mult_proj_err="ERROR: You have multiple projects we can charge."

cat >/tmp/myjob_$USER.$$ <<EOF
#!/bin/${idev_login_shell}
#$ -V
#$ -cwd
#$ -N  ${JOB_NAME}
#$ -j  y
#$ -o  \$HOME/.slurm/\$JOB_NAME.o\$JOB_ID
#$ -pe ${MY_WAY}way ${MY_CORES}
#$ -q  $MY_QUEUE
#$ $IDEV_ACCOUNT
#$ -t  $MY_hhmmss_TIME
#$ $MY_RESOURCE
#$ $MY_PROJECT

$idev_env_set INTEL_LICENSE_FILE$idev_env_equal$my_intel_port_server

$idev_pwd_cmd
$DIR/idev_utils env_replication ${idev_login_shell} \$JOB_ID
$CREATE_NODE_LIST_CMD
sleep $job_sleep
 
EOF
 
 #cp    /tmp/myjob_$USER.$$ /tmp/$USER_idev_job_$$
 qsub < /tmp/myjob_$USER.$$
 if [[ $? != 0 ]]; then
    exit_msg=" idev detected an error in your resource request (see details above):\n Check your (this) command:\n $0 $cmd_args"
 #                                     Try this again, and analyze error report.
    qsub_log=idev_qsub_err_$USER.$$
    qsub <      /tmp/myjob_$USER.$$ 1>&2  >/tmp/$qsub_log
    grep  "$mult_proj_err"  /tmp/$qsub_log >& /dev/null
    if [[ $? == 0 && $MY_ACCOUNT =~ use_default ]]; then
        exit_msg=" "
        echo ""
        echo "*** IDEV found that you have a new account   ***"
        echo "*** You must reselect a default account.     ***"
        echo "*** Execute   idev  to select a new default, ***"
        echo "*** and begin your next idev session.        ***"
        echo ""
        rm -rf $HOME/.idevrc
    fi
    rm -rf /tmp/$qsub_log
    exit
 fi
 
 rm -rf /tmp/myjob_$USER.$$
 
 job_id=none
 job_submitted=yes
 
 ###                        I should ask to use the .sge
 ###                        maybe later. I'll just assume
 ###                        that I have "right" just like
 ###                        the the SGE batch system:)
 ##
 ###               Make dir. and copy file if necesary
 ##if [[ ! -d $my_intel_dir ]]; then
 ##   mkdir   $my_intel_dir
 ##   cp      $intel_dir/$intel_license $my_intel_dir
 ##fi
 ##
 ###               If no reasonable license file, get one
 ##if [ ! -f $my_intel_dir/intel.current.*.lic ]; then
 ##   cp        $intel_dir/$intel_license $my_intel_dir
 ##fi
 ##
 ###               If it doesn't match the System version,
 ###               get the new one; but remove old one
 ##present_intel_license=`basename    $intel_dir/*current*`
 ##     my_intel_license=`basename $my_intel_dir/*current*`
 ##
 ##if [[ $present_intel_license != $my_intel_license ]]; then
 ##   rm      $my_intel_dir/intel.current.*.lic
 ##   cp         $intel_dir/$intel_license $my_intel_dir
 ##fi
 
 
 
 echo -e "\n After your idev job begins to run, a command prompt will appear,"
 echo -e   " and you can begin your interactive development session. "
  
 echo -e   " We will report the job status every $monitor_sleep seconds: (qw=queue wait, r=running).\n"
 torf=1
 while [ $torf == 1 ]
 do
 
    qstat | grep "$JOB_NAME" >& /dev/null
    if [ $? ]; then
       mystatus=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $5}'`
       echo "Job status:" $mystatus   in $MY_QUEUE queue.; echo ""
       job_id=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'`
       if [ x"$mystatus" == "xr" ]; then
          master_node=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $8}'`
                JOBID=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'`
          nodestring=`echo $master_node | sed 's/.*@//'`
 
          echo "--> Job is now running (masternode =  $master_node) ."
          echo "--> But, sleeping for $sleep_after_run seconds."
          sleep $sleep_after_run
 #                                            Get master and rank 0 nodes
          master_node=`echo $nodestring | sed 's/.'"$MY_SYS"'.*//'`
 
 #                                            Set ssh to the node of choice
 #                                            Might be more general later.
          [[ $use_node_type == master ]] && node=$master_node
          if [[ $use_node_type == rank0  ]] ; then 
             rank0_node=`head -n 1 $HOME/.sge/job.$JOBID.hostlist.*`
             rank0_node=${rank0_node%%.*}
             node=$rank0_node
          fi
 
          echo "--> Launching interactive session on $node, the $use_node_type node...OK"
 #                                            Finally!
          ssh -Y -A -o "StrictHostKeyChecking no" $node
          exit
       fi
    fi
 
 sleep $monitor_sleep
 done
fi

##############  -------------------------------SLURM
if [[ $BATCH_SYS == SLURM ]]; then

mult_proj_err="ERROR: You have multiple projects to charge to"

#cat >/tmp/myjob_$USER.$$ <<EOF
cat >$HOME/.slurm/myjob_$USER.$$ <<EOF
#!/bin/${idev_login_shell}
#SBATCH -J  ${JOB_NAME}
#SBATCH -o  $HOME/.slurm/${JOB_NAME}.o%j
#SBATCH -p  $MY_QUEUE
#SBATCH -t  $MY_hhmmss_TIME
#SBATCH -n  ${MY_TOTAL_TASKS}
#SBATCH $IDEV_NODES
#SBATCH $IDEV_ACCOUNT
$MY_RESERVATION

$idev_pwd_cmd
$idev_rm_perl5
#                         Remove old files (needed for grace-serial queue).
rm -rf /tmp/idev_${USER}_${idev_login_shell}_env.[0-9]*
$DIR/idev_utils env_replication ${idev_login_shell} \$SLURM_JOB_ID
$CREATE_NODE_LIST_CMD
sleep $job_sleep

EOF

#sbatch /tmp/myjob_$USER.$$
 sbatch $HOME/.slurm/myjob_$USER.$$

 if [[ $? != 0 ]]; then
    exit_msg=" idev detected an error in your resource request (see details above):\n Check your (this) command:\n   $0 $cmd_args \n"
 #                                     Try this again, and analyze error report.
    qsub_log=idev_qsub_err_$USER.$$
    sbatch   /tmp/myjob_$USER.$$ >&  /tmp/$qsub_log
    grep  "$mult_proj_err"  /tmp/$qsub_log >& /dev/null
    if [[ $? == 0 && $MY_ACCOUNT =~ use_default ]]; then
        exit_msg=" "
        echo ""
        echo "*** IDEV found that you have a new project               ***"
        echo "*** OR your default project (account) is not valid.      ***"
        echo "*** The default project is being removed from ~/.idevrc. ***"
        echo "*** Execute idev again to select a new default--         ***"
        echo "*** and begin your next idev session.                    ***"
        echo ""
        sed  '/idev_project/d' $HOME/.idevrc > $HOME/.idevrc_$$
        mv   $HOME/.idevrc     $HOME/.idevrc_$$_safety
        mv   $HOME/.idevrc_$$  $HOME/.idevrc
 
    fi
    rm -rf /tmp/$qsub_log
    exit
 fi
 
#rm -rf /tmp/myjob_$USER.$$
if  [ ! -d "$IDEV_KEEP_MY_JOB_DIR" ] ;then
  rm -rf $HOME/.slurm/myjob_$USER.$$
else
  cp $HOME/.slurm/myjob_$USER.$$ $IDEV_KEEP_MY_JOB_DIR/myjob_$USER.$$
  rm -rf $HOME/.slurm/myjob_$USER.$$
fi
 
 job_id=none
 job_submitted=yes
 
 echo -e "\n After your idev job begins to run, a command prompt will appear,"
 echo -e   " and you can begin your interactive development session. "
  
#echo -e   " We will report the job status every $monitor_sleep seconds: (qw=queue wait, r=running).\n"
 echo -e   " We will report the job status every $monitor_sleep seconds: (PD=pending, R=running).\n"
 torf=1
#                             Environment file check: parameters.
 found_env_file=1
 time_waited_env_file=0
 sleep_time_env_file=5

 while [ $torf == 1 ]
 do

    squeue -p $MY_QUEUE "$squeue_opts" -u $USER | grep "$JOB_NAME" >& /dev/null
    if [ $? ]; then
       mystatus=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $5}'`
       echo "job status:  $mystatus"
       job_id=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'`
       if [ x"$mystatus" == "xR" ]; then
          master_node=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $8}'`
                JOBID=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'`

          master_node=`echo $master_node | sed 's/,.*//'`; 
          if [[ $master_node =~ \[ ]]; then    #Has form c557-[100-104]
            master_node=`echo $master_node | sed 's/\[//'`
            master_node=`echo $master_node | sed -e 's/\(^........\).*/\1/'`
          fi
          nodestring=$master_node

 
          echo "--> Job is now running on masternode= $master_node...OK"
          echo -ne "--> Sleeping for $sleep_after_run seconds."
          sleep $sleep_after_run
          echo "..OK"
 #                                            Get master and rank 0 nodes
          master_node=`echo $nodestring | sed 's/.'"$MY_SYS"'.*//'`
 
 #                                            Set ssh to the node of choice
 #                                            Might be more general later.
 #                                            May need this SGE trick later
          [[ $use_node_type == master ]] && node=$master_node
          if [[ $use_node_type == rank0  ]] ; then 
             rank0_node=`head -n 1 $HOME/.slurm/job.$JOBID.hostlist.*`
             rank0_node=${rank0_node%%.*}
             node=$rank0_node
          fi
#                                             Check to make sure environment file
#                                             is in /tmp on the compute node.
#                                             Multi-node jobs may take up to 25 sec.
#                                             before the script is started.
          echo -ne "--> Checking to make sure your job has initialized an env for you."
#####
          while [ $found_env_file == 1 ]; do
          
              ssh -o "StrictHostKeyChecking no" $node "ls /tmp/idev_${USER}_${idev_login_shell}_env.${JOBID}" >& /dev/null
              if [[ $? == 0 ]]; then
                 found_env_file=0
                 if [[ $time_waited_env_file > 0 ]]; then
                    echo ""
                    echo "--> Oh joy! The environment is now in place...OK"
                 else
                    echo "...OK"
                 fi
              else
          
                 if [[ $time_waited_env_file == 0 ]]; then
                   echo ""
                   echo "Hmm, for multi-node jobs, (or when $BATCH_SYS is sleepy) we need to"
                   echo "     give $BATCH_SYS more time to kick things off (maybe 20 sec.). "
                   echo "     I'll check every $sleep_time_env_file secs."
                   echo ""
                 fi
                 echo   "     Time waited (sec.) : $time_waited_env_file"
                 sleep $sleep_time_env_file
                 time_waited_env_file=$(( $time_waited_env_file + 5 ))
              fi
          done

#####

          echo "--> Creating interactive terminal session (login) on $use_node_type node $node."
 #                                            Finally!
          echo -en "\007"
          ssh -Y -A  -o "StrictHostKeyChecking no" $node
          exit
       fi
    fi
 
 sleep $monitor_sleep
 done
fi

fi 
#http://github.com/milfeld2/idivx
#trapping
#http://rimuhosting.com/knowledgebase/linux/misc/trapping-ctrl-c-in-bash
#http://www.linuxjournal.com/content/use-bash-trap-statement-cleanup-temporary-files
#http://github.com/milfeld/idivx