From 53224fb1b15417c35aeb260f433269bfa66ccedc Mon Sep 17 00:00:00 2001 From: Kent Milfeld Date: Mon, 25 Jan 2016 08:21:43 -0600 Subject: [PATCH] INITIAL COMMIT --- idev | 1538 ++++++++++++++++++++++++++++++++++++++++++++++++++++ idev_utils | 634 ++++++++++++++++++++++ 2 files changed, 2172 insertions(+) create mode 100755 idev create mode 100755 idev_utils diff --git a/idev b/idev new file mode 100755 index 0000000..f47ee6a --- /dev/null +++ b/idev @@ -0,0 +1,1538 @@ +#!/bin/bash +#set -x +# User is forces to use either SGE or SLURM options, except +# -p and -q can both be used for either batch system +# +# For Stampede the system startup file /etc/profile.d/z95_idev. +# takes care of sourcing /tmp/idev__env. if the file +# exists, and exports the env. vars: IDEV_SETUP_BYPASS="1.0", +# idev_ip=cyxz-abc, idev_env_status=[0|1]. +# If the sourcing file exists in /tmp on the compute node +# idev_env_status is set to 0, the file is sourced, shell cd's +# to $IDEV_PWD, and DDTMPIRUN=ibrun is exported. This is called +# System Sourcing. + +# If System Sourcing exists for the machine, IDEV_ENV_REPLICATION +# is exported as SYSTEM, and no user startup file (.profile, etc.) +# is instrumented to source an environment file. This can be +# bypassed by including: +# IDEV_ENV_REPLICATION=USER +# in the .idevrc file. +# Note: For SYSTEM /tmp/idev__env. is the sourcing file +# For USER /tmp/my__env. is the sourcing file + +# SLURM +# If user account becomes invalid, or single-project mode +# is not longer valid (user acquires an additional project), +# the idev_project line (default project) is removed from ~/.idevrc. +# 06/28/2013 KentM +# SLURM +# Reservations can be requested. Users are queried +# about any active reservation-- do they want to use it. +# Use "idev -r none" to avoid questions. +# 09/20/2013 KentM +# Remove SSH_xxx from environment +# Allow users to remove environment variables +# by adding idev_rm_env= +# +# Eliminates PERL5LIB from idev environment when +# executing idev perl commands. +# 03/30/2015 KentM +# Adjustments for ls5 +# On nidxxxx service node, system type will be ls5 +# Remove returns BASH_FUNC_ functions. See env_replication routine. +# Now uses $HOME/.slurm to hold job scripts +# (use IDEV_KEEP_MY_JOB_DIR variable to save job scipt to directory) +# zzz85_idev.sh and zzz85_idev.csh updated in /etc/profile.d +#Set up CpN for vis and gpu queue on ls5 -- change for stampede later +#Remove SHELL_STARTUP_DEBUG_T0 from environment variables on compute node. +#Use $CpN when determining Project from test batch launch (R.McLay req). +#Changed ls5 default queue to development (was normal) +#Inserted -A in ssh command (forwarding of authentication agent connection). +# +# 12/09/2015 KentM +# +#Hidden grace-serial queue +# Added -p option to squeue (to see queue) +# Allow -n 1 for default for grace-serial queue +# Changed format from 9 to 15 chars for queue name (grace-serial is 12 chars) +# Abort if -N is used in grace-serial queue. +# 01/21/2016 KentM +#------------------------------------------------------------------------------ +# +# Versions Number + IDEV_VER=1.00 + IDEV_SETUP_VER=1.00 + +#Determine the system + myhostname=`hostname -f` + if [[ $myhostname =~ nid ]] ; then + MY_SYS=ls5 #I'm on a MOM node (login0, etc.) + else + MY_SYS=`echo $myhostname | awk 'BEGIN { FS = "." } ; { print $2 }'` + fi + + + srun=off # Don't use srun on Stampede +# trap ctrl-c and call ctrl_c() +trap ctrl_c INT EXIT +did_exit=no +exit_msg="" + +# Trap Function. Deletes job on exit of this script +function ctrl_c() { + #Job submitted, and job id available. + if [[ "x$job_submitted" = "xyes" && "x$job_id" != "xnone" ]]; then +# Keeps from calling qdel twice. (why twice?) + if [[ $did_exit == no ]]; then + echo "Cleaning up: submitted job ($job_submitted) removing job $job_id." + + if [[ $MY_SYS == "stampede" || $MY_SYS == "ls5" ]]; then + scancel $job_id + else + qdel $job_id + fi + fi + did_exit=yes + exit; + fi + #Job submitted, but no job id yet. + if [ "x$job_submitted" = "xyes" ]; then + echo "Please control-c again, SGE/SLURM is responding slowly and I need a job ID for cleanup." + echo "job_id:$job_id job_submitted:$job_submitted" + else + #User control-c'd out before job launched. + echo -e "$exit_msg" + echo " -- idev: no session created, gracefully exiting..." + exit + fi +} + +# Do the non-installation sensitive options here. + + +# Intel port and server -- for using intel products on comp. nodes. + my_intel_port_server=27000@troy.tacc.utexas.edu + +### @TACC copy intel license to .sge/intel +### Intel license info: +##intel_license=intel.current.20120301.lic +## my_intel_dir=$HOME/.sge/intel +## intel_dir=/opt/intel/licenses + +# idev puts the default project in .idevrc +# pick it up here for the help option + + my_pid=$$ + +# Defaults + MY_MINUTES=30 + MY_hhmmss_TIME=00:${MY_MINUTES}:00 + MY_H_RT="h_rt=$MY_hhmmss_TIME" + MY_QUEUE=development + + RESERVATION_CONTROL="NO_OPTION" + RESERVATION_NAME="" + MY_RESERVATION="" + RESERVATION_USE=no + +JOB_NAME=idv$$ +monitor_sleep=4 +sleep_after_run_factor=4 +job_sleep=$(( MY_MINUTES * 60 )) + + +# SSH to master node by default + + use_node_type=master + insert_option=none + IDEV_ACCOUNT="-A willpromptyou" + MY_ACCOUNT=none + check_project=yes + nodes_set=no + total_tasks_set=no + queue_arch=standard + + #Make squeue look identical for ls5/stampede + squeue_opts='-o %.18i %.15P %.9j %.8u %.2t %.10M %.6D %R' + + if [ -e $HOME/.idevrc ]; then +# Get Project Name + grep -q idev_project $HOME/.idevrc + if [[ $? == 0 ]]; then #Pick up last occurence + MY_ACCOUNT=`grep idev_project $HOME/.idevrc | tail -1 |sed 's/idev_project//'` + MY_ACCOUNT="${MY_ACCOUNT%\\n}" # get rid of return + MY_ACCOUNT=`echo $MY_ACCOUNT | sed 's/ //g'` # get rid of spaces + if [[ $MY_ACCOUNT =~ ^$ ]]; then + echo "Please " + echo " remove idev_project line from $HOME/.idevrc" + echo "and execute idev again." + exit 1 + fi + echo "" + echo "Defaults file : ~/.idevrc" + check_project=no + if [[ $MY_ACCOUNT =~ use_default ]]; then + IDEV_ACCOUNT=" " + echo "Project : $MY_ACCOUNT " + else + IDEV_ACCOUNT="-A $MY_ACCOUNT" + echo "Default project : $MY_ACCOUNT" + fi + fi + +# Get time default + grep -q idev_time $HOME/.idevrc + if [[ $? == 0 ]]; then + MY_TIME=`grep idev_time $HOME/.idevrc | sed 's/idev_time//'` + MY_TIME="${MY_TIME%\\n}" # get rid of return + MY_TIME=`echo $MY_TIME | sed 's/ //g'` # get rid of spaces + if [[ $MY_TIME =~ ":" ]]; then + MY_H_RT="h_rt=$MY_TIME" + regex='0*([[:digit:]]+)[:::]0*([[:digit:]]+)[:::]0*([[:digit:]]+)' + if [[ "$MY_H_RT" =~ $regex ]]; then + secs=$((${BASH_REMATCH[1]}*3600+${BASH_REMATCH[2]}*60+${BASH_REMATCH[3]})) + else + echo "ERROR: default time syntax: time <#>." + echo " Value found: \"$MY_TIME\"" + exit + fi + MY_MINUTES=$(( secs / 60 )) + job_sleep=$secs + # if [[ $BATCH_SYS == "SGE" ]] ; then + # echo "default time : -l $MY_H_RT" + # else + # MY_hhmmss_TIME=$MY_TIME + # echo "default time : -t $MY_hhmmss_TIME" + # fi + else + MY_MINUTES=$MY_TIME + MY_H_RT="h_rt=00:${MY_MINUTES}:00 " #SGE/SLURM accept more than 2 digits in this field. + MY_hhmmss_TIME="00:${MY_MINUTES}:00 " #SGE/SLURM accept more than 2 digits in this field. + # if [[ $BATCH_SYS == "SGE" ]] ; then + # echo "time (minutes) : -l $MY_H_RT" + # echo "default time : -l $MY_H_RT" + # else + # echo "default time : -t $MY_hhmmss_TIME" + # fi + job_sleep=$(( $MY_MINUTES * 60 )) + fi + echo "Default time : $MY_MINUTES min." + else + echo "Default time : $MY_MINUTES min." + + fi + + grep -q idev_queue $HOME/.idevrc + if [[ $? == 0 ]]; then + MY_QUEUE=`grep idev_queue $HOME/.idevrc | sed 's/idev_queue//'` + MY_QUEUE="${MY_QUEUE%\\n}" # get rid of return + MY_QUEUE=`echo $MY_QUEUE | sed 's/ //g'` # get rid of spaces + [[ $MY_QUEUE =~ gpu ]] && queue_arch=gpu + [[ $MY_QUEUE =~ vis ]] && queue_arch=gpu + [[ $MY_QUEUE =~ serial ]] && queue_arch=serial + echo "Default queue : $MY_QUEUE" + else + echo "Default queue : $MY_QUEUE" + fi + + + grep -q idev_rm_env $HOME/.idevrc + if [[ $? == 0 ]]; then + IDEV_RM_ENV=`grep idev_rm_env $HOME/.idevrc | sed 's/idev_rm_env//'` + IDEV_RM_ENV="${IDEV_RM_ENV%\\n}" # get rid of return + IDEV_RM_ENV=`echo "$IDEV_RM_ENV" | sed 's/^ *//g' | sed 's/ *$//g'` # get rid of spaces + echo "Removed env vars : $IDEV_RM_ENV" + export IDEV_RM_ENV + fi + + fi + +# Determine my present working shell + idev_ppid=`ps -l | awk '{if ($4 == '$$' ) print $5 }'` + idev_cmd=` ps -l | awk '{if ($4 == '$idev_ppid') print $14}'` + + if [[ $idev_cmd =~ (tcsh|csh) ]] ; then + idev_pwshell=tcsh + else + idev_pwshell=bash + fi + + +# Determine my LOGIN shell + my_gecos=`grep $USER /etc/passwd` + if [[ $my_gecos =~ (tcsh$|csh$) ]] ; then + idev_login_shell=tcsh + idev_pwd_cmd='setenv IDEV_PWD `pwd`' + idev_env_set=setenv + idev_env_equal=" " + + # rm PERL5LIB for idev env_replication, but put in /tmp/env + idev_rm_perl5=" + setenv idev_has_user_PERL5LIB no + if (\$?PERL5LIB && { eval 'test ! -z \$PERL5LIB' } ) then + setenv idev_user_PERL5LIB \${PERL5LIB} && setenv idev_has_user_PERL5LIB yes + unsetenv PERL5LIB + endif + " + else + idev_login_shell=bash + idev_pwd_cmd='export IDEV_PWD=`pwd`' + idev_env_set=export + idev_env_equal="=" + # rm PERL5LIB for idev env_replication, but put in /tmp/env + idev_rm_perl5=' + export idev_has_user_PERL5LIB=no; + if [[ ! -z "$PERL5LIB" ]]; then + export idev_user_PERL5LIB=$PERL5LIB + export idev_has_user_PERL5LIB=yes + unset PERL5LIB + fi + ' + fi + +if [[ $idev_pwshell -ne $idev_login_shell ]]; then + echo " Warning: Your present working shell is $idev_pwshell." + echo " Your login shell is $idev_login_shell." + echo " Idev will place you in your login shell on the master node." +fi + + +MY_RESOURCE="" +CREATE_NODE_LIST_CMD="" + +#OPERATION Set batch system parameters. + +JOB_NAME=idv$$ +monitor_sleep=4 +sleep_after_run_factor=4 +job_sleep=$(( MY_MINUTES * 60 )) + +login_sourcing=yes #use .login/.profile sourcing code for creating batch env + +case $MY_SYS in + + maverick) echo "System : Maverick" + MY_WAY=20; MY_CORES=20 + MY_TOTAL_TASKS=20; MY_NODES=1; CpN=20 # Cores/Node + MY_PROJECT="" + sleep_after_run=7 + ## -P vis, data, gpgpu, hpc + BATCH_SYS=SLURM + Q="-p" + MY_QUEUE=gpu + IDEV_NODES="" #Usually not set + export IDEV_QDEL=scancel + login_sourcing=by_tacc_etc_profile.d #Let TACC do this + ;; + ls4) echo "System : Lonestar" + MY_WAY=12; MY_CORES=12; CpN=12 # Cores/Node + MY_PROJECT="" + sleep_after_run=4 + BATCH_SYS=SGE + Q="-q" + export IDEV_QDEL=qdel + ;; + stampede) echo "System : Stampede" + MY_WAY=16; MY_CORES=16; CpN=16 # Cores/Node + MY_TOTAL_TASKS=16; MY_NODES=1; CpN=16 # Cores/Node + MY_PROJECT="" + sleep_after_run=7 + BATCH_SYS=SLURM + Q="-p" + IDEV_NODES="" #Usually not set + export IDEV_QDEL=scancel + login_sourcing=by_tacc_etc_profile.d #Let TACC do this + ;; + ls5) echo "System : Lonestar" + + if [[ $queue_arch == standard ]]; then + MY_WAY=24; MY_CORES=24; CpN=24 # Cores/Node + MY_TOTAL_TASKS=24; MY_NODES=1; CpN=24 # Cores/Node + fi + + if [[ $queue_arch == gpu ]]; then + MY_WAY=10; MY_CORES=10; CpN=10 # Cores/Node + MY_TOTAL_TASKS=10; MY_NODES=1; CpN=10 # Cores/Node + fi + + if [[ $queue_arch == serial ]]; then + MY_WAY=1; MY_CORES=1; CpN=24 # Cores/Node + MY_TOTAL_TASKS=1; MY_NODES=1; CpN=24 # Cores/Node + fi + + MY_PROJECT="" + sleep_after_run=7 + BATCH_SYS=SLURM + Q="-p" + IDEV_NODES="" #Usually not set + export IDEV_QDEL=scancel + login_sourcing=by_tacc_etc_profile.d #Let TACC do this + + MY_QUEUE=development + + ;; + *) echo " Don't know about this system ($MY_SYS)" + exit 1 + ;; +esac + +###### + +do_exit=no +check_project=yes +cmd_args=$@ + +# Get command line arguments -- they override everything +OPT_ERR=0 + +while (( "$#" )); do + +# Non-installation sensitive options here + + known_option=no + + if [[ x$1 == x-help || x$1 == x--help || x$1 == x-h || x$1 == xhelp ]]; then + known_option=yes + echo "" + echo "" + echo "Usage: idev [OPTIONS]" + echo " " + echo "idev creates an interactive session on a compute node" + echo "for executing serial, openmp-parallel, or mpi-parallel" + echo "code as you would in a batch job." + echo "Supported systems: Lonestar, Longhorn and Stampede." + echo "" + echo "Idev uses a single node by default and charges the project in \$HOME/.idevrc." + echo "Options can be in any order." + echo "" + echo "For Lonestar:>>" + echo "Use a login node window for MPI compiling, and an idev window for running." + echo "The mpicc/mpif90 compiler does not work correctly on Lonestar compute nodes." + echo "(You can compile openmp and serial code with intel compilers, though).<<" + echo " " + echo "OPTIONS:" + echo " OPTION ARGUMENTS DESCRIPTION" + echo "" + echo " -A account_name sets account name (default: -A $MY_ACCOUNT)" + echo " -m minutes sets time in minutes (default: $MY_MINUTES)" +if [[ $BATCH_SYS == SLURM ]]; then + echo " -n total_tasks Total number of tasks" + echo " -N nodes Number of nodes" +fi +if [[ $BATCH_SYS == SGE ]]; then + echo " -pe way cores sets wayness & cores (default 1 node: -pe $MY_WAY $MY_CORES)" +fi +if [[ $MY_SYS == longhorn ]]; then + echo " -P project_name project type for Longhorn {vis, data, gpgpu, or hpc}" +fi + echo " $Q queue_name sets queue to named queue (default: $Q $MY_QUEUE)" +if [[ $BATCH_SYS == SGE ]]; then + echo " -r resource_name selects hardware " + echo " (e.g. -r mem_total=250G or -r h=)" + echo " added to -l option as h= (no default)" +fi + +if [[ $BATCH_SYS == SLURM ]]; then + echo " -r (no res. name) find and use my ACTIVE reservation (no questions)" + echo " -r reservation_name requests use of a specific reservation" + echo " -r none idev always automatically looks for reserations" + echo " and asks you to use an ACTIVE res. none==don't look/ask." + echo " (use scontrol show reserations to see reservation details)" +fi + + echo " -t hh:mm:ss sets time to hh:mm:ss (default: 00:${MY_MINUTES}:00, 30 min.)" + echo "" +if [[ $BATCH_SYS == SGE ]]; then + echo " -rank0 [--rank0 ] login to rank 0 node (may not be the master node)" + echo " TACC users: no need to use this since NO_HOSTSORT " + echo " env. var is set and task0 will be on master node." + echo " -uninstall [--uninstall] removes idev login setup and \$HOME/.idevrc file." +fi + echo " -help [help, --help, -h] displays (this) help message" + echo " -v [ --version ] output version information and exit" + echo "" + echo " Syntax and Examples:" +if [[ $BATCH_SYS == SGE ]]; then + echo " Syn: idev (defaults: 1 node, full wayness, $MY_MINUTES min., default account.)" + echo " Syn: idev [-help | -v | -uninstall] " + echo " Syn idev [-q que.] [-m min.] [-t time ] [-pe way cores] \ " + echo " [-A acct] [-r res.] [-P proj.]" + + echo " ex1 idev -pe 12 12 -q development -A A-ccvis -m 90 " + echo " ex2 idev -pe 12 96 -q normal -t 00:90:00 " + echo " ex3 idev -P vis (a Longhorn Project type)" + echo " ex4 idev" + echo " ex5 idev -help" +fi +if [[ $BATCH_SYS == SLURM ]]; then + echo " Syn: idev (defaults: 1 node, 16 tasks, $MY_MINUTES min., default account.)" + echo " Syn: idev [-help | -v | -uninstall] " + echo " Syn idev [-p queue] [-m min.] [-t time ] [-A acct] \ " + echo " [-n tasks] [-N nodes -n tasks] " + echo "" + echo " ex4 idev" + echo " ex5 idev -help" + echo " ex1 idev -p development -m 90 -A A-ccvis -n 16" + echo " ex2 idev -p normal -t 00:90:00 -A TG-STA123 -N 2 -n 16 " +fi + + echo "" + echo "Idev ssh's to compute nodes with the -Y option to allow X forwarding for debugging." + echo "" + echo "Insert \"idev_time MINUTES\", \"idev_queue QUEUE\" or \"idev_project PROJECT\" " + echo " without quotes, one item per line in ~/.idevrc to set your own default " + echo " time, queue and project. (Substitute values for MINUTES, QUEUE and PROJECT.)" + echo " E.g. idev_time 60 idev_queue normal " + echo "" + echo "Insert \"idev_rm_env VAR_LIST\" " + echo " in ~/.idevrc, where VAR_LIST is a space separated list of environment variables " + echo " not to be replicated from the job environment initialized by idev. " + echo " Note: all SSH_xx variables are removed by default so that all interactive " + echo " ssh terminal sessions have their correct SSH_xx values for a new session." +# echo "" + do_exit=yes; OPT_ERR=0 + fi + +# Login to rank 0 node, may not be the master node. + if [[ x$1 == x-version || x$1 == x--version || x$1 == x-v ]]; then + known_option=yes + echo "Version : $IDEV_VER" + do_exit=yes; OPT_ERR=0 + fi +# Remove login setup from bash/tcsh files, remove $HOME/.idevrc + if [[ x$1 == x-uninstall || x$1 == x--uninstall ]]; then + known_option=yes + insert_option=uninstall + fi + + if [[ x$1 == x-n ]]; then + known_option=yes + if [[ $BATCH_SYS == "SLURM" ]] ; then + shift; MY_TOTAL_TASKS=$1; total_tasks_set=yes + echo "Using total tasks: -n $MY_TOTAL_TASKS" + else + echo -e "\n ERROR: **The -N option is not allowed on $MY_SYS." + echo -e " It is not a valid $BATCH_SYS option." + echo -e " See options, execute: idev -help" + do_exit=yes; OPT_ERR=1 + fi + fi + + if [[ x$1 == x-N ]]; then + known_option=yes + if [[ $BATCH_SYS == "SLURM" ]] ; then + shift; MY_NODES=$1; IDEV_NODES="-N $1"; nodes_set=yes + echo "Using nodes : -N $MY_NODES" + else + echo -e "\n ERROR: **The -N option is not allowed on $MY_SYS." + echo -e " It is not a valid $BATCH_SYS option." + echo -e " See options, execute: idev -help" + do_exit=yes; OPT_ERR=1 + fi + fi + + + if [[ x$1 == x-pe ]]; then + known_option=yes + if [[ $BATCH_SYS == "SGE" ]] ; then + shift; myway=$1 + shift; MY_CORES=$1 + MY_WAY=`echo $myway | sed s/way//` + echo "Using Prog Env : -pe ${MY_WAY}way $MY_CORES" + MY_NODES=$(( MY_CORES / CpN )) + EXTRA_TIME=`echo "scale=0; l($MY_NODES)/l(10)" | bc -l` + EXTRA_TIME=$(( MY_NODES * EXTRA_TIME / CpN / sleep_after_run_factor )) + + sleep_after_run=$(( sleep_after_run + EXTRA_TIME )) + else + echo -e "\n ERROR: **The -pe option is not allowed on $MY_SYS." + echo -e " It is not a valid $BATCH_SYS option." + echo -e " See options, execute: idev -help" + do_exit=yes; OPT_ERR=1 + fi + + fi + + if [[ x$1 == x-q ]]; then + known_option=yes + shift; MY_QUEUE=$1 + if [[ $BATCH_SYS == "SGE" ]] ; then + echo "Using queue : -q $MY_QUEUE" + else + echo "Using queue : -p $MY_QUEUE" + CMDLINE_QUEUE=$MY_QUEUE + queue_from=cmdline + fi + + fi + + if [[ x$1 == x-p ]]; then + known_option=yes + shift; MY_QUEUE=$1 + if [[ $BATCH_SYS == "SGE" ]] ; then + echo "Using queue : -q $MY_QUEUE" + else + echo "Using queue : -p $MY_QUEUE" + CMDLINE_QUEUE=$MY_QUEUE + queue_from=cmdline + fi + + fi + + if [[ x$1 == x-A ]]; then + known_option=yes + shift; IDEV_ACCOUNT="-A $1" + check_project=no + echo "Using Project : $IDEV_ACCOUNT" + + fi + + if [[ x$1 == x-t ]]; then + known_option=yes + shift; MY_H_RT="h_rt=$1" + regex='0*([[:digit:]]+)[:::]0*([[:digit:]]+)[:::]0*([[:digit:]]+)' + if [[ "$MY_H_RT" =~ $regex ]]; then + secs=$((${BASH_REMATCH[1]}*3600+${BASH_REMATCH[2]}*60+${BASH_REMATCH[3]})) + else + echo "ERROR: h_rt (syntax hr:min:sec) requires value for hr, min and sec entry." + echo " Value found: \"$MY_H_RT\"" + exit + fi + job_sleep=$secs + if [[ $BATCH_SYS == "SGE" ]] ; then + echo "time (h_rt) : -l $MY_H_RT" + else + MY_hhmmss_TIME=$1 + echo "time (hh:mm:ss) : -t $MY_hhmmss_TIME" + fi + fi + + if [[ x$1 == x-m ]]; then + known_option=yes + shift; MY_MINUTES=$1 + MY_H_RT="h_rt=00:${MY_MINUTES}:00 " #SGE/SLURM accept more than 2 digits in this field. + MY_hhmmss_TIME="00:${MY_MINUTES}:00 " #SGE/SLURM accept more than 2 digits in this field. + if [[ $BATCH_SYS == "SGE" ]] ; then + echo "time (minutes) : -l $MY_H_RT" + else + echo "time (minutes) : -t $MY_hhmmss_TIME" + fi + job_sleep=$(( $MY_MINUTES * 60 )) + fi + + if [[ x$1 == x-rank0 || x$1 == x--rank0 ]]; then + known_option=yes + if [[ $BATCH_SYS == "SGE" ]] ; then + use_node_type=rank0 + CREATE_NODE_LIST_CMD="ibrun date >& /dev/null #creates $HOME/.sge/*job*hostlist.* file" + echo "SSHing to : rank 0 node" + else + echo -e "\n **The -rank0 option is not allowed on $MY_SYS." + echo -e " It is not a valid $BATCH_SYS option." + echo -e " See options, execute: idev -help" + do_exit=yes; OPT_ERR=1 + fi + fi + + if [[ x$1 == x-r ]]; then + known_option=yes + + if [[ $BATCH_SYS == "SGE" ]] ; then + shift; MY_RESOURCE="-l $1" + echo "hardware request : $MY_RESOURCE" + fi + + if [[ $BATCH_SYS == "SLURM" ]] ; then + + + if [[ x$2 = x ]]; then + # We are at the end of the line without a + # reservation name-- meaning we will look for any reservation. + RESERVATION_CONTROL="FIND_IT" + echo "Reservation : Will search and request it for you." + else + if [[ x$2 =~ "x-" ]]; then + # We have found the next option. No reservation name + # means we look for the reservation. + RESERVATION_CONTROL="FIND_IT" + echo "Reservation : Will find it for you." + else + # Found reservation request with name + shift + if [[ $1 =~ none ]]; then + RESERVATION_CONTROL="NONE" + MY_RESERVATION="" + echo "Reservation : $1" + else + RESERVATION_CONTROL="USER_SET" + RESERVATION_NAME=$1 + MY_RESERVATION="--reservation=$1" + echo "Reservation : $1" + MY_RESERVATION="#SBATCH $MY_RESERVATION" + fi + fi + + fi + + # echo -e "\n ERROR: **The -r option is not allowed on $MY_SYS." + # echo -e " It is not a valid $BATCH_SYS option." + # echo -e " See options, execute: idev -help" + # do_exit=yes; OPT_ERR=1 + fi + fi + + if [[ x$1 == x-P ]]; then + known_option=yes + if [[ $MY_SYS == "longhorn" ]] ; then + shift; MY_PROJECT="-P $1" + echo "Longhorn project type : $MY_PROJECT" + else + echo -e "\n ERROR: **The -P option is only allowed on Longhorn." + echo -e " To see options, execute: idev -help." + do_exit=yes; OPT_ERR=1 + fi + fi + + if [[ $known_option == "no" ]]; then + echo -e "\n ERROR: **The $1 option is not know." + echo -e " To see options, execute: idev -help." + do_exit=yes; OPT_ERR=2 + + fi + + + shift + +done + + +############################################beg Dependent options + + [[ $MY_QUEUE =~ gpu ]] && queue_arch=gpu + [[ $MY_QUEUE =~ vis ]] && queue_arch=gpu + [[ $MY_QUEUE =~ serial ]] && queue_arch=serial + + +if [ x$MY_SYS = "xls5" ] && + [ x$queue_arch = "xgpu" ] && + ( [ x$total_tasks_set = "xno" -a x$nodes_set = "xno" ]); then + MY_WAY=10; MY_CORES=10; CpN=10 # Cores/Node + MY_TOTAL_TASKS=10; MY_NODES=1; CpN=10 # Cores/Node +fi + +if [ x$MY_SYS = "xls5" ] && + [ x$queue_arch = "xserial" ] && + ( [ x$total_tasks_set = "xno" -a x$nodes_set = "xno" ]); then + MY_WAY=1; MY_CORES=1; CpN=24 # Cores/Node + MY_TOTAL_TASKS=1; MY_NODES=1; CpN=24 # Cores/Node +fi + +if [ x$MY_SYS = "xls5" ] && + [ x$queue_arch = "xserial" ] && + [ x$nodes_set = "xyes" ]; then + echo " " + echo " ERROR: -N resource option MUST NOT BE USED in $MY_QUEUE queue." + echo " This is a serial queue, only -n 1 should be specified." + do_exit=yes; OPT_ERR=1 +fi + +# SLURM on Stampede & LS5. If -N is provided, but -n is not, disallow + +if [[ $total_tasks_set == no && $nodes_set == yes ]] ; then + + echo -e "\n ERROR: ** -n option (total tasks) is not set. **" + echo -e " When using the -N option, the -n option is required." + echo -e " (Same as batch requirements--See Stampede User Guide.)" + do_exit=yes; OPT_ERR=1 + +fi + +############################################end Dependent options + + +# Bail out if not launching or arg error +# Only last error value is reported in trap. +# But all error messages reported (above). +#------------------------------------------------------------------------------ +[[ $do_exit == "yes" ]] && exit $OPT_ERR +#------------------------------------------------------------------------------ + + DIR=`dirname $0` #Need directory where idev & utils are + [ x$? != x0 ] && exit 1; + + idev_dir=$DIR #Need full path for PATH in login scripts + [[ ! $idev_dir =~ ^/ ]] && idev_dir=$PWD/$idev_dir + +############################################beg RESERVATION + +# Values of RESERVATION_CONTROL +# -r --> FIND_IT +# -r none --> NONE +# -r xyz --> USER_SET +# --> NO_OPTION +# tot tot_active status name queue +# 1 0/1 act/nonactive 1 doit, 0 advise +# 2+ 1 --- doit 1 active +# 3+ 0,2... --- Advise + +if [[ $BATCH_SYS == SLURM ]]; then + +RESERVATION_NAME_FOUND=no + + if [[ $RESERVATION_CONTROL == NONE ]]; then + # do nothing-- If the user set it, good luck. + echo doing_nothing>/dev/null + else + reservation_string=`$DIR/idev_utils get_reservation $USER`; + reservation_array=( $reservation_string ) + res_totl=${reservation_array[0]} + res_actv=${reservation_array[1]} + res_stat=${reservation_array[2]} + res_name=${reservation_array[3]} + res_queue=${reservation_array[4]} + # (If we see 1 reservation all is done.) + + # Get ACTIVE reservation information here. + #if [[ $res_totl -ge 2 && $res_actv -le 1 ]]; then + if [[ $res_totl -ge 2 ]]; then + for((i=0;i<${#reservation_array[@]};i++)) + do + # Find 1 ACTIVE reservation. + # For res_actv=0, nothing changes since ACTIVE not found. + # We avoid the case for 2 or more ACTIVE reservations (later). + #echo "RES_NAME $i ${reservation_array[$i]}" + if [[ ${reservation_array[$i]} == ACTIVE ]]; then + res_stat=${reservation_array[$i]} + res_name=${reservation_array[ (($i+1)) ]} + res_queue=${reservation_array[ (($i+2)) ]} + RESERVATION_USE=yes + fi + done + + # We validate what the user has set a valid name. + # This many not be an (the) active reservation! + if [[ $RESERVATION_CONTROL == USER_SET ]]; then + for((i=0;i<${#reservation_array[@]};i++)) + do + if [[ ${reservation_array[$i]} == $RESERVATION_NAME ]]; then + RESERVATION_NAME_FOUND=yes + RESERVATION_STATUS=${reservation_array[ (($i-1)) ]} + RESERVATION_QUEUE=${reservation_array[ (($i+1)) ]} + if [[ $RESERVATION_STATUS == INACTIVE && $res_actv == 1 ]]; then + echo " NOTE: you have requested a INACTIVE reservation ($RESERVATION_NAME)." + echo " NOTE: $res_name is your ACTIVE reservation." + + echo " Do you really want to wait on a INACTIVE reservation? (suggested answer: no)" + echo -n " Enter y/n [default y]: " + read -s -n 1 yorn + echo "" + if [[ $yorn != y && $yorn != n ]]; then yorn=y; fi + + if [[ $yorn == y ]]; then + res_stat=INACTIVE + res_name=$RESERVATION_NAME + res_queue=$RESERVATION_QUEUE + RESERVATION_USE=yes + else + echo " Adios. Use \"scontrol show reservations\" to view reservations." + exit 1 + fi + fi + fi + done + fi + + fi + + + if [[ $res_totl -eq 0 ]]; then + if [[ $RESERVATION_CONTROL == USER_SET ]]; then + echo " No reservation named $RESERVATION for $USER was found." + echo " Execute: \"scontrol show reservations\" to see reservations." + exit 1; + fi + + if [[ $RESERVATION_CONTROL == FIND_IT ]]; then + echo " No reservation for $USER was found." + echo " Execute: \"scontrol show reservations\" to see reservations." + exit 1; + fi + + # not here + #if [[ $res_name != $RESERVATION_NAME ]]; then + #echo " Please correct and try again." + #else #slide on by + fi + + if [[ $res_totl -eq 1 || (( $res_totl -ge 2 && $res_actv -eq 1 )) ]]; then + + if [[ $res_actv -eq 1 ]]; then + + RESERVATION_USE=yes + + # ACTIVE Reservation + if [[ $RESERVATION_CONTROL == USER_SET ]]; then + if [[ $res_name != $RESERVATION_NAME ]]; then + echo " ERROR:" + echo " idev reservation name (-r $RESERVATION_NAME) " + echo " does not match slurm reservation name ($res_name)." + echo " Aborting idev. Execute: \"idev -r\" and idev will find the name for you." + echo " Execute: \"scontrol show reservations\" to see reservations." + exit 1; + else #good to go + + echo "Reservation : --reservation=$res_name ($res_stat)" + RESERVATION_NAME=$res_name + MY_RESERVATION="#SBATCH --reservation=$res_name" + MY_QUEUE=$res_queue + RESERVATION_USE=yes + + fi + fi + + + if [[ $RESERVATION_CONTROL == FIND_IT ]]; then + echo "Reservation : --reservation=$res_name (ACTIVE)" + RESERVATION_NAME=$res_name + MY_RESERVATION="#SBATCH --reservation=$res_name" + MY_QUEUE=$res_queue + RESERVATION_USE=yes +#CHECK +# echo "Using res. queue : -p $MY_QUEUE" + fi + if [[ $RESERVATION_CONTROL == NO_OPTION ]]; then + echo " We found an ACTIVE reservation request for you, named $res_name." + echo " Do you want to use it for your interactive session?" + echo -n " Enter y/n [default y]: " + read -s -n 1 yorn + echo "" + if [[ $yorn != y && $yorn != n ]]; then yorn=y; fi + + if [[ $yorn == y ]]; then + echo "Reservation : --reservation=$res_name (ACTIVE)" + RESERVATION_NAME=$res_name + MY_RESERVATION="#SBATCH --reservation=$res_name" + MY_QUEUE=$res_queue + RESERVATION_USE=yes +#CHECK +# echo "Using res. queue : -p $MY_QUEUE" + else #Continue on as a non-reservation request. + echo " Continuing without using your reservation." + echo " Use \"scontrol show reservations\" to view your reservation(s)." + RESERVATION_USE=no + fi + fi + else ##INACTIVE RESERVATION + + if [[ $RESERVATION_CONTROL == USER_SET || $RESERVATION_CONTROL == FIND_IT ]]; then + + echo "Reservation : --reservation=$res_name (INACTIVE)" + RESERVATION_NAME=$res_name + MY_RESERVATION="#SBATCH --reservation=$res_name" + MY_QUEUE=$res_queue + RESERVATION_USE=yes +#CHECK +# echo "Using res. queue : -p $MY_QUEUE" + + echo "" + echo " ****** YOU HAVE AN INACTIVE RESERVATION ******" + echo "" + if [[ $RESERVATION_CONTROL == FIND_IT ]]; then + echo " Meaning your reservation period has not started yet." + echo " You might want to control-c out of this session," + echo " and execute idev without the -r option to access nodes that are available." + fi + if [[ $RESERVATION_CONTROL == USER_SET ]]; then + echo " You explicitly requested this, so idev will proceed, as you wish." + echo " Execute control-c if you get tired of waiting." + fi + echo " To see when the reservation begins, execute: scontrol show reservations" + fi + + if [[ $RESERVATION_CONTROL == NO_OPTION ]]; then + echo " " + echo " Just a Note: Your reservation $res_name is INACTIVE." + echo " Use idev -r to use it when it becomes ACTIVE." + echo " To see when it begins, execute: scontrol show reservations" + echo " " + RESERVATION_USE=no + fi + + fi + fi + + if [[ $res_totl -ge 2 && $res_actv -ne 1 ]]; then + + # Don't process: $RESERVATION_CONTROL == NO_OPTION + + if [[ $RESERVATION_CONTROL == USER_SET ]]; then + if [[ $RESERVATION_NAME_FOUND != yes ]]; then + echo " idev didn't find a reservation ($RESERVATION_NAME)." + echo " Execute: \"scontrol show reservations\" to see reservations." + exit 1 + else #good to go + if [[ $RESERVATION_STATUS == INACTIVE ]]; then + + echo " We found your reservation to be INACTIVE." + echo " idev will not run until it becomes ACTIVE." + echo " Do you want to wait on it? We suggest no." + echo -n " Enter y/n [default y]: " + read -s -n 1 yorn + echo "" + if [[ $yorn != y && $yorn != n ]]; then yorn=y; fi + + if [[ $yorn == y ]]; then + echo "Reservation : --reservation=$RESERVATION_NAME (INACTIVE)" + MY_RESERVATION="#SBATCH --reservation=$RESERVATION_NAME" + MY_QUEUE=$RESERVATION_QUEUE +#CHECK +# echo "Using res. queue : -p $MY_QUEUE" + RESERVATION_USE=yes + else + echo " Adios. Use \"scontrol show reservations\" to view reservations." + exit 1 + fi + else + + #But USER_SET assures us one particular reservation is requested. + echo "Reservation : --reservation=$RESERVATION_NAME (ACTIVE)" + MY_RESERVATION="#SBATCH --reservation=$RESERVATION_NAME" + MY_QUEUE=$RESERVATION_QUEUE +#CHECK +# echo "Using res. queue : -p $MY_QUEUE" + RESERVATION_USE=yes + fi + fi + fi + + + if [[ $RESERVATION_CONTROL == FIND_IT ]]; then + if [[ $res_actv -gt 1 ]]; then + echo -e "\n You have $res_actv ACTIVE reservations." + echo -e " I'm at a loss as to which one idev should use." + echo -e " Please execute idev with the -r -p options." + echo -e " To see your ACTIVE reservations, execute: scontrol show reservations." + exit 1 + fi + if [[ $res_actv -eq 0 ]]; then + echo -e "\n You have NO ACTIVE reservations and $res_totl INACTIVE reservations." + echo -e " I'm at a loss as to which one should be used." + echo -e " Please execution idev with the -r -p options." + echo -e " To see your reservations, execute: scontrol show reservations." + exit 1 + fi + fi + + if [[ $RESERVATION_CONTROL == NO_OPTION ]]; then + if [[ $res_actv -gt 1 ]]; then + echo -e "\n NOTE: **** You have $res_actv ACTIVE reservations. ****" + echo -e " **** Idev expects that you want to use one of them. ****" + echo -e " If you want to use one of these, execute idev -r " + echo -e " To see your reservations, execute: scontrol show reservations." + echo -e " Use \"idev -r none\" to avoid this message and NOT exit idev." + exit 1 + else + echo -e "\n FYI: You have $res_totl INACTIVE reservations." + echo -e " Idev is not asking to use them because they are INACTIVE." + echo -e " If you want to wait on one of them, re-execute idev with the -r option." + echo -e "\n" + fi + fi + + fi + + fi +fi + # Array info, etc. + #http://stackoverflow.com/questions/6723426/looping-over-arrays-printing-both-index-and-value + #http://tldp.org/HOWTO/Bash-Prog-Intro-HOWTO-10.html + #http://tldp.org/LDP/abs/html/comparison-ops.html + #http://tldp.org/HOWTO/Bash-Prog-Intro-HOWTO.html#toc10 + +############################################end RESERVATION + + + + + + +#################beg POST RESERVATION +################ +# Make adjustment if reservation doesn't have a queue. +## -- when queue is '(null)' + if [[ $res_queue == '(null)' && $RESERVATION_USE == yes ]]; then + if [[ $queue_from == cmdline ]]; then + + MY_QUEUE=$CMDLINE_QUEUE + echo "Cmdline queue : -p $MY_QUEUE" + + else + + echo "Reser. queue : $res_queue" + echo " " + echo " Error: ** You must specify a queue for this reservation. **" + echo " NO queue was assigned when the reservation was made " + echo " and NO queue is specified on the idev command line." + echo " Specify a queue (-p option) that contains the nodes" + echo " of your reservation, or ask the person who made the" + echo " reservation to include a queue for the nodes." + + exit 1 + fi + #else +## # Else, use the queue of the reservation + # MY_QUEUE=$CMDLINE_QUEUE + # MY_QUEUE=$res_queue + # echo "Using res. queue : -p $MY_QUEUE" + + fi + +# If$MY_QUEUE='(null)' and does not have -p abort. +# + +#################beg POST RESERVATION + +if [[ $login_sourcing == "yes" ]] ; then #login sourcing + #SETUP Make sure user has the idev login commands. + + if [[ -n $IDEV_SETUP_BYPASS ]]; then + if [ "$IDEV_SETUP_BYPASS" != "$IDEV_SETUP_VER" ]; then + echo "The idev shell setup has changed to version $IDEV_SETUP_VER." + echo "Please read about the changes in the idev script, make appropriate" + echo "adjustments, and set your IDEV_SETUP_BYPASS var to $IDEV_SETUP_VER." + exit 1 + fi + else + $DIR/idev_utils ${idev_login_shell}_login_insertion $MY_SYS $idev_dir + fi + + if [[ $insert_option == uninstall ]]; then + echo " Uninstalling..." + $DIR/idev_utils ${idev_login_shell}_login_insertion $MY_SYS $idev_dir $insert_option + echo " Uninstall Complete." + #do_exit=yes + if [[ ! -z $idev_login_ver ]] ; then + echo "Remember to LOGOUT!" + echo "Remember to LOGOUT!" + echo "Remember to LOGOUT!" + echo "Remember to LOGOUT!" + echo "Remember to LOGOUT!" + fi + exit 0 + fi +fi # end of login sourcing + +################################### Project INFO Begin + +if [[ $check_project == yes ]]; then + + # If no project in HOME/.idevrc + # or -A option not used on cmd line + # Submit a bogus job to get projects + # This is the best I can do as a user!! + # Hmmm. I cannot redirect error on this. + + if [[ $MY_ACCOUNT == none || -z $MY_ACCOUNT ]]; then + echo "We need a project to charge for interactive use."; + echo "We will be using a dummy job submission to determine your project(s)."; + echo "We will store your (selected) project \$HOME/.idevrc file."; + echo "" + # Submit bogus job +if [[ $BATCH_SYS == SGE ]]; then +qsub <&1 >/tmp/idev_acct_job_output_$my_pid +#$ -V +#$ -pe 1way $MY_CORES +#$ -q $MY_QUEUE +#$ -l h_rt=00:01:00 +#$ $MY_RESOURCE +#$ $MY_PROJECT +EOF + + echo "Please disregard above 2 lines from the qsub utility." + echo "" + +fi + +if [[ $BATCH_SYS == SLURM ]]; then + test_queue=normal + if [[ $MY_SYS == maverick ]]; then + test_queue=gpu + fi +cat </tmp/idev_acct_job_$my_pid +#!/bin/bash +#SBATCH -J acct_tes +#SBATCH -o /dev/null +#SBATCH -p $test_queue +#SBATCH -n $CpN +#SBATCH -t 00:01:00 +EOF + +sbatch /tmp/idev_acct_job_$my_pid >& /tmp/idev_acct_job_output_$my_pid +#rm -rf /tmp/idev_acct_job_$my_pid + +fi + + # Extract Projects from file in /tmp + # (Get_projects.pl script removes it.) + # echo "HERE idev_utils get_projects $my_pid "; + project_string=`$DIR/idev_utils get_projects $my_pid`; + project_array=( $project_string ) + + # Now ask user which one he wants to use if count>1. + if [[ ${#project_array[@]} > 1 ]]; then + + echo "Please select the NUMBER of the project you want to charge.\n"; + count=1; + count=1; + for i in "${project_array[@]}" + do + echo $count $i + count=$(( $count + 1 )) + done + + echo "Please type the NUMBER(default=1) and hit return: "; + read project_no + if [[ "$project_no" == "" ]]; then project_no=1; fi + + if [ $project_no -eq $project_no 2> /dev/null ]; then + echo "" #keep shell happy + else + echo "You didn't type a number. You typed $project_no. Try again." + exit 1 + fi + + echo "$project_no is the project number you selected." + + if [[ $project_no -gt 0 && $project_no -le ${#project_array[@]} ]]; then + # if [[ $project_no > 0 && $project_no <= ${project_array[@]} ]]; then + array_no=$(( $project_no - 1)); + echo " OK I'll use ${project_array[$array_no]}. Thanks. " + echo "idev_project ${project_array[$array_no]}" >> $HOME/.idevrc + MY_ACCOUNT=${project_array[$array_no]} + else + echo "$project_no is not a valid number, please try again." + exit 1 + fi + + IDEV_ACCOUNT="-A $MY_ACCOUNT" + + else + # echo " You have a single project named ${project_array[0]}." + # MY_ACCOUNT=${project_array[0]} + # echo "idev_project ${project_array[$array_no]}" >> $HOME/.idevrc + echo " You have a single project which we will use as the default ." + MY_ACCOUNT=${project_array[0]} + echo "idev_project ${project_array[$array_no]}" >> $HOME/.idevrc + + IDEV_ACCOUNT=" " + fi + + rm -rf /tmp/idev_acct_job_output_$my_pid + fi +fi + + +################################### Project INFO End +#################################### Latest Setup Check Begin + +# Make sure user is using the latest setup. + if [[ ! -z $idev_login_ver ]] ; then #if defined-> using earlier version + echo " We have improved idev! But it will be necessary" #using IDEV_SETUP_VER now + echo " to uninstall your initial idev setup. Please execute:" + echo "" + echo " idev -uninstall" + echo "" + echo " AND THEN LOGOUT. Login and then execute \"idev\" to reinstall." + echo " " + exit + fi +#################################### Setup Check End + +if [[ $BATCH_SYS == SGE ]]; then + +mult_proj_err="ERROR: You have multiple projects we can charge." + +cat >/tmp/myjob_$USER.$$ <&2 >/tmp/$qsub_log + grep "$mult_proj_err" /tmp/$qsub_log >& /dev/null + if [[ $? == 0 && $MY_ACCOUNT =~ use_default ]]; then + exit_msg=" " + echo "" + echo "*** IDEV found that you have a new account ***" + echo "*** You must reselect a default account. ***" + echo "*** Execute idev to select a new default, ***" + echo "*** and begin your next idev session. ***" + echo "" + rm -rf $HOME/.idevrc + fi + rm -rf /tmp/$qsub_log + exit + fi + + rm -rf /tmp/myjob_$USER.$$ + + job_id=none + job_submitted=yes + + ### I should ask to use the .sge + ### maybe later. I'll just assume + ### that I have "right" just like + ### the the SGE batch system:) + ## + ### Make dir. and copy file if necesary + ##if [[ ! -d $my_intel_dir ]]; then + ## mkdir $my_intel_dir + ## cp $intel_dir/$intel_license $my_intel_dir + ##fi + ## + ### If no reasonable license file, get one + ##if [ ! -f $my_intel_dir/intel.current.*.lic ]; then + ## cp $intel_dir/$intel_license $my_intel_dir + ##fi + ## + ### If it doesn't match the System version, + ### get the new one; but remove old one + ##present_intel_license=`basename $intel_dir/*current*` + ## my_intel_license=`basename $my_intel_dir/*current*` + ## + ##if [[ $present_intel_license != $my_intel_license ]]; then + ## rm $my_intel_dir/intel.current.*.lic + ## cp $intel_dir/$intel_license $my_intel_dir + ##fi + + + + echo -e "\n After your idev job begins to run, a command prompt will appear," + echo -e " and you can begin your interactive development session. " + + echo -e " We will report the job status every $monitor_sleep seconds: (qw=queue wait, r=running).\n" + torf=1 + while [ $torf == 1 ] + do + + qstat | grep "$JOB_NAME" >& /dev/null + if [ $? ]; then + mystatus=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $5}'` + echo "Job status:" $mystatus in $MY_QUEUE queue.; echo "" + job_id=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'` + if [ x"$mystatus" == "xr" ]; then + master_node=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $8}'` + JOBID=`qstat | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'` + nodestring=`echo $master_node | sed 's/.*@//'` + + echo "--> Job is now running (masternode = $master_node) ." + echo "--> But, sleeping for $sleep_after_run seconds." + sleep $sleep_after_run + # Get master and rank 0 nodes + master_node=`echo $nodestring | sed 's/.'"$MY_SYS"'.*//'` + + # Set ssh to the node of choice + # Might be more general later. + [[ $use_node_type == master ]] && node=$master_node + if [[ $use_node_type == rank0 ]] ; then + rank0_node=`head -n 1 $HOME/.sge/job.$JOBID.hostlist.*` + rank0_node=${rank0_node%%.*} + node=$rank0_node + fi + + echo "--> Launching interactive session on $node, the $use_node_type node...OK" + # Finally! + ssh -Y -A -o "StrictHostKeyChecking no" $node + exit + fi + fi + + sleep $monitor_sleep + done +fi + +############## -------------------------------SLURM +if [[ $BATCH_SYS == SLURM ]]; then + +mult_proj_err="ERROR: You have multiple projects to charge to" + +#cat >/tmp/myjob_$USER.$$ <$HOME/.slurm/myjob_$USER.$$ <& /tmp/$qsub_log + grep "$mult_proj_err" /tmp/$qsub_log >& /dev/null + if [[ $? == 0 && $MY_ACCOUNT =~ use_default ]]; then + exit_msg=" " + echo "" + echo "*** IDEV found that you have a new project ***" + echo "*** OR your default project (account) is not valid. ***" + echo "*** The default project is being removed from ~/.idevrc. ***" + echo "*** Execute idev again to select a new default-- ***" + echo "*** and begin your next idev session. ***" + echo "" + sed '/idev_project/d' $HOME/.idevrc > $HOME/.idevrc_$$ + mv $HOME/.idevrc $HOME/.idevrc_$$_safety + mv $HOME/.idevrc_$$ $HOME/.idevrc + + fi + rm -rf /tmp/$qsub_log + exit + fi + +#rm -rf /tmp/myjob_$USER.$$ +if [ ! -d "$IDEV_KEEP_MY_JOB_DIR" ] ;then + rm -rf $HOME/.slurm/myjob_$USER.$$ +else + cp $HOME/.slurm/myjob_$USER.$$ $IDEV_KEEP_MY_JOB_DIR/myjob_$USER.$$ + rm -rf $HOME/.slurm/myjob_$USER.$$ +fi + + job_id=none + job_submitted=yes + + echo -e "\n After your idev job begins to run, a command prompt will appear," + echo -e " and you can begin your interactive development session. " + +#echo -e " We will report the job status every $monitor_sleep seconds: (qw=queue wait, r=running).\n" + echo -e " We will report the job status every $monitor_sleep seconds: (PD=pending, R=running).\n" + torf=1 +# Environment file check: parameters. + found_env_file=1 + time_waited_env_file=0 + sleep_time_env_file=5 + + while [ $torf == 1 ] + do + + squeue -p $MY_QUEUE "$squeue_opts" -u $USER | grep "$JOB_NAME" >& /dev/null + if [ $? ]; then + mystatus=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $5}'` + echo "job status: $mystatus" + job_id=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'` + if [ x"$mystatus" == "xR" ]; then + master_node=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $8}'` + JOBID=`squeue -p $MY_QUEUE "$squeue_opts" -u $USER | awk '$3 ~ /'"$JOB_NAME"'/ { print $1}'` + + master_node=`echo $master_node | sed 's/,.*//'`; + if [[ $master_node =~ \[ ]]; then #Has form c557-[100-104] + master_node=`echo $master_node | sed 's/\[//'` + master_node=`echo $master_node | sed -e 's/\(^........\).*/\1/'` + fi + nodestring=$master_node + + + echo "--> Job is now running on masternode= $master_node...OK" + echo -ne "--> Sleeping for $sleep_after_run seconds." + sleep $sleep_after_run + echo "..OK" + # Get master and rank 0 nodes + master_node=`echo $nodestring | sed 's/.'"$MY_SYS"'.*//'` + + # Set ssh to the node of choice + # Might be more general later. + # May need this SGE trick later + [[ $use_node_type == master ]] && node=$master_node + if [[ $use_node_type == rank0 ]] ; then + rank0_node=`head -n 1 $HOME/.slurm/job.$JOBID.hostlist.*` + rank0_node=${rank0_node%%.*} + node=$rank0_node + fi +# Check to make sure environment file +# is in /tmp on the compute node. +# Multi-node jobs may take up to 25 sec. +# before the script is started. + echo -ne "--> Checking to make sure your job has initialized an env for you." +##### + while [ $found_env_file == 1 ]; do + + ssh -o "StrictHostKeyChecking no" $node "ls /tmp/idev_${idev_login_shell}_env.${JOBID}" >& /dev/null + if [[ $? == 0 ]]; then + found_env_file=0 + if [[ $time_waited_env_file > 0 ]]; then + echo "" + echo "--> Oh joy! The environment is now in place...OK" + else + echo "...OK" + fi + else + + if [[ $time_waited_env_file == 0 ]]; then + echo "" + echo "Hmm, for multi-node jobs, (or when $BATCH_SYS is sleepy) we need to" + echo " give $BATCH_SYS more time to kick things off (maybe 20 sec.). " + echo " I'll check every $sleep_time_env_file secs." + echo "" + fi + echo " Time waited (sec.) : $time_waited_env_file" + sleep $sleep_time_env_file + time_waited_env_file=$(( $time_waited_env_file + 5 )) + fi + done + +##### + + echo "--> Creating interactive terminal session (login) on $use_node_type node $node." + # Finally! + echo -en "\007" + ssh -Y -A -o "StrictHostKeyChecking no" $node + exit + fi + fi + + sleep $monitor_sleep + done +fi + +fi +#trapping +#http://rimuhosting.com/knowledgebase/linux/misc/trapping-ctrl-c-in-bash +#http://www.linuxjournal.com/content/use-bash-trap-statement-cleanup-temporary-files diff --git a/idev_utils b/idev_utils new file mode 100755 index 0000000..f723dd3 --- /dev/null +++ b/idev_utils @@ -0,0 +1,634 @@ +#!/usr/bin/perl +use File::Copy; +use File::Basename; + +# This perl utility work in conjunction with the idev command +# env_replication: gathers master's enviroment in file to be sourced +# /tmp/my__env.### is the file name +# will put in $HOME/.sge in future version +# = [tcsh | bash] on these shells are supported +# tcsh_login_insertion: +# bash_login_insertion: +# Inserts commands in .login for idev +# An ssh to a compute node triggers sourcing +# the /tmp/my__enf.### file. +# get_projects +# Determine list of projects and returns to +# list to idev for user selection as the default. +# Unfortunately, if the user had only one project +# the project is not returned. Uh! So, use_default +# is returned to idev, which then uses a blank +# for "-A " in the job scripts. +# After all is done, the project name is placed +# in $HOME/.idevrc for subsequent use, obviating +# the need to call get_projects again +# +# 1/1/2011 +# All rights reserved Kent Milfeld +# +# Included routine to extract active reservations 9/4/2013 +# get_reservation +# +# Consolidated multiple perl scripts 2/2/2011 +# For single-project users, return "use_default" 2/24/2011 +# Updated in get_projects +# login_insertion includes idev's full path in PATH 3/28/2011 +# Included uninstall option tcsh/bash_login insertions 4/28/2011 +# Will remove from .login/ setup +# Will remove $HOME/.idevrc + +# DISPLAY is not set. Also, program no longer executes +# an unset/unsetenv on variables that are not set. 11/09/2011 +# Now lets "ssh -Y ..." set display for tunneling. + +# Create hash table for exceptions-- a bit cleaner. +# Set RUNNING_IDEV and NO_HOSTSORT for ibrun. +# (Means that task0 is no longer needed for TACC.) +# Remove "_" from environment (request by rtm). +# Set SHELL to login shell. Thanks to Robert Mclay. 05/01/2012 + +# Included routine to return SLURM reservations 09/20/2013 +# Remove SSH environment variables 03/27/2015 +# Allow user to list env vars to exclude in .idevrc 03/27/2015 + + + +sub env_replication(@); +sub tcsh_login_insertion(@); +sub bash_login_insertion(@); +sub get_projects($); +sub get_reservation($); #new + +$cmd =shift(@ARGV); +$arg1=shift(@ARGV); +$arg2=shift(@ARGV); +$arg3=shift(@ARGV); + +if( $cmd eq "env_replication" ){ env_replication($arg1,$arg2) ;} +if( $cmd eq "tcsh_login_insertion" ){ tcsh_login_insertion($arg1,$arg2,$arg3) ;} +if( $cmd eq "bash_login_insertion" ){ bash_login_insertion($arg1,$arg2,$arg3) ;} +if( $cmd eq "get_projects" ){ get_projects($arg1) ;} +if( $cmd eq "get_reservation" ){ get_reservation($arg1) ;} #new + +sub get_reservation($){ + +# Gets ACTIVE reservation and queue of the active reservation for a USER. +# Assumes that there will only be a single reservation. +# Users should use "idev -r " if they need a specific +# reservation (when more than one is active for them). +# To avoid using an active reservation users should use "idev -r none". +# + $status="none"; + $total = 0; + $tot_actv = 0; + $reservations=""; + $USER=shift(@_); + open(RESERVATIONS,"scontrol show reservations | egrep -e '(Users=|ReservationName=|Nodes=)' |") or die; + #open(RESERVATIONS,"cat ./see_0a0i0n | egrep -e '(Users=|ReservationName=|Nodes=)' |") or die; + while(){ + #if( $status eq "none" ){ #just find the first reservation + chomp; + if( /ReservationName=/ ){ + $res_name =$_; #do this the quick way + $res_name =~ s/ReservationName=//; + $res_name =~ s/ .*//; + } + if( /Nodes=/ ){ + if( /PartitionName=/ ){ + $partition=$_; #do this the quick way + $partition =~ s/.*PartitionName=//; + $partition =~ s/ .*//; + } + else { + $partition = "(null)"; + } + } + + if( /Users=/ ){ + if( /\b$USER\b/ ){ + $total = $total + 1; + if( /\bState=ACTIVE\b/ ){ + $status="ACTIVE"; + $tot_actv = $tot_actv + 1; + }else{ + $status="INACTIVE"; + } + $reservations = "$reservations $status $res_name $partition "; + } + } + + #} + } +# Print out information: "total tot_active status name queue" for each. + if( $status eq "none" ){ + print "0 0 none none none"; + }else { + print "$total $tot_actv $reservations"; + } +} + + + +sub env_replication(@){ + +# Only tcsh(csh) and bash shells are supported-- 99.9 of our users. +# Extract environment variables from "ENV". +# Insert commands to be sourced in /tmp/my__env. +# use export format for bash, and setenv format for tcsh + + $my_shell = shift(@_); + $my_idev = shift(@_); + + my @env_exclude_list=qw(SHELL_STARTUP_DEBUG_T0 LESSOPEN DISPLAY MANPATH LS_COLORS _ module BASH_FUNC_module() BASH_FUNC_module%% BASH_FUNC_ml() BASH_FUNC_ml%% SSH_CLIENT SSH_TTY SSH_AUTH_SOCK SSH_CONNECTION); +# my @env_exclude_list=qw(LESSOPEN DISPLAY MANPATH LS_COLORS _ module BASH_FUNC_module%% LOADEDMODULES BASH_FUNC_ml%% SSH_CLIENT SSH_TTY SSH_AUTH_SOCK SSH_CONNECTION); + my %exclude; + foreach (@env_exclude_list) { $exclude{$_} = 1; } + +# Allow users to submit a list of env vars to exclude. + if( defined $ENV{"IDEV_RM_ENV"}){ + @idev_rm_list = split(/ +/,$ENV{"IDEV_RM_ENV"}); + foreach (@idev_rm_list) { $exclude{$_} = 1; } + } + +# TACC Site variable. +# Lets ibrun know idev is running, and not to +# sort host list. + %myenv=(RUNNING_IDEV=>1,NO_HOSTSORT=>1); + +## Changed for 9.2 my_... --> idev_... +## $myenv_filename = "/tmp/idev_${my_shell}_env.${my_idev}"; + $myenv_filename = "/tmp/idev_${USER}_${my_shell}_env.${my_idev}"; + if ( $my_shell eq "bash" ){ + $CMD = "export "; + $UNCMD = "unset "; + $EQIV = "="; + } + else{ + $CMD = "setenv "; + $UNCMD = "unsetenv "; + $EQIV = " "; + } + + $len = 1; + $mykey="-"; + foreach $key (keys(%ENV)){ + $len = length($key); + if($lenmax < $len) {$lenmax=$len; $mykey=$key;} + } + #print "max=$lenmax for $mykey\n"; + + + foreach $key (keys(%ENV)){ + $value = $ENV{$key}; + $value =~ s/\n|\r//g; + + if( $exclude{$key} ){ + $unmyenv{$key} = $value; + #printf("%-$lenmax.${lenmax}s: $unmyenv{$key}\n",$key); + } + else { + #printf("%-$lenmax.${lenmax}s: $ENV{$key}\n",$key); + $myenv{$key} = $value; + #printf("%-$lenmax.${lenmax}s: $myenv{$key}\n",$key); + } + } + +# We avoid any problems users create with PERL5LIB, by +# running without it, but we put it back in their environment + + if( $ENV{"idev_has_user_PERL5LIB"} eq "yes" ){ + $myenv{"PERL5LIB"} = $ENV{"idev_user_PERL5LIB"}; + } + +# TACC FIX, set SHELL to login shell. +# OSGE sets shell to csh + $my_shell_fullpath=`which $my_shell`; + chomp($my_shell_fullpath); + $myenv{SHELL}=$my_shell_fullpath; + + open(FILE,">$myenv_filename") or die; + print "File: $myenv_filename\n"; + foreach $key (keys(%myenv)){ + print FILE "$CMD $key${EQIV}\"$myenv{$key}\"\n"; + } + #foreach $key (keys(%unmyenv)){ + # print FILE "$UNCMD $key\n"; + #} + close(FILE); + + #--------------------------------- alias + + exit 0; + $myenv_filename = "/tmp/myalias.$$"; + @obviate=qw( ); + + + #open(FILE,"tcsh alias |") or die; + # + #while(){ + # chomp; + # print "$_\n"; + #} + + $len = 1; + $mykey="-"; + foreach $key (keys(%ENV)){ + $len = length($key); + if($lenmax < $len) {$lenmax=$len; $mykey=$key;} + } + print "max=$lenmax for $mykey\n"; + + + + # Check + open(FILE,">$myalias_filename") or die; + print "File: $myalias_filename\n"; + foreach $key (keys(%myalias)){ + print FILE "setalias $key \"$myalias{$key}\"\n"; + } + foreach $key (keys(%unmyalias)){ + print FILE "unsetalias $key\n"; + } + close(FILE); + +} #end of login_insert subroutine + +#------------------------------------------------------------- + + +# Prototype the perl function call -- only allow one argument ($SYSTEM). +sub tcsh_login_insertion(@){ + $SYSTEM=shift(@_); + $IDEV_DIR=shift(@_); + $IDEV_RM=shift(@_); + + # Include command in .login if needed. cmds_add (1=added,0=found) + ($status, $HPCUT_VER)=csh_shell_setup($SYSTEM,$IDEV_DIR,$IDEV_RM); + if($status eq "Command Insert rejected."){ + print "Status:$status HPCUT_VER:$HPCUT_VER\n"; + exit(1); + } +} + +sub csh_shell_setup (@) { +# csh_shell_setup +# Inserts HPCUT commands into .login file for C shell, if not present +# Prompts user for permission if commands not found. +# #IDEV BEGIN is the sentinal that indicates commands are present +# Version Number detected for automatic updates. + +# HPCUT LOGIN COMMANDS Commands required in .login for C shell. +# $, " and ' must be protected with backslashes. + + my($system,$idev_dir) = @_; + $HOME=$ENV{HOME}; + $LOGIN_FILE = "$HOME/.login"; + $IDEVRC_FILE = "$HOME/.idevrc"; +# $LOGIN_FILE = "$HOME/.login_test"; + +# If idev in /usr/bin, comment out path insert query +# /usr/bin should be in path +# query used when idev is not installed in /usr/bin +# Doing with comments-- add/noadd would be complicated +# by double substitutions-- may do this better later. + $line_do_nodo=" "; + if( $idev_dir eq "/user/bin" ){$line_do_nodo="#";} + + $LOGIN_BEGIN = '#IDEV BEGIN'; + $LOGIN_END = '#IDEV END'; + $LOGIN_CMDS = + "#IDEV BEGIN (This line used to detect idev setup-- do not remove.) \n" . + " \n" . + "$line_do_nodo echo \$PATH | egrep ${idev_dir}\'(\$|:)\' >& /dev/null \n" . + "$line_do_nodo if ( \$status ) then \n" . + "$line_do_nodo set path=($idev_dir \${path}) \n" . + "$line_do_nodo endif \n" . + " \n" . + " setenv idev_hostname `hostname` \n" . + " setenv IDEV_SETUP_VER 0.91 \n" . + " #if on compute node \n" . + " setenv idev_ip `echo \$idev_hostname | sed \"s/.$system.*//\"` \n" . + " if ( \$idev_ip =~ i[0-9][0-9][0-9]-[0-9][0-9][0-9] || \\\n" . + " \$idev_ip =~ c[0-9][0-9][0-9]-[0-9][0-9][0-9] || \\\n" . + " \$idev_ip =~ vis[0-9] || \$idev_ip =~ visbig ) then \n\n" . + " # ls returns \"0\" if true, and !0 if false for if statement \n" . + " ls /tmp/idev_tcsh_env.[0-9]* >& /dev/null \n" . + " setenv idev_env_status \$status \n\n" . + " if( ! \$idev_env_status ) then \n" . + " source /tmp/idev_tcsh_env* #source env file \n" . + " cd \$IDEV_PWD #go to submit dir. \n" . + " endif \n\n" . + " endif \n" . + " unsetenv idev_hostname idev_ip idev_env_status \n" . + "#IDEV END (This line used to detect idev setup-- do not remove.) \n"; + + + $LOGIN_INSERT = + "#IDEV BEGIN (This line used to detect idev setup-- do not remove.) \n" . + " if (-r \$HOME/.idev_tcsh) then \n" . + " source \$HOME/.idev_tcsh \n" . + " endif \n" . + "#IDEV END (This line used to detect idev setup-- do not remove.) \n"; + +# " echo \$PATH | egrep $USER\'/bin(\$|:)\' >& /dev/null \n" . +# " if ( ! \$status ) then \n" . +# " set path=(\${path} \$HOME/bin) \n" . + + $LOGIN_VERSION = 'setenv\s+idev_ver\s+(\d+\.\d+)'; + + $found_hpcut_cmds = 0; + if(-e $LOGIN_FILE) { + open ( FILE,"<$LOGIN_FILE") or die "ERROR: Cannot open $LOGIN_FILE"; + while() { if(/^$LOGIN_BEGIN/) { $found_hpcut_cmds = 1;} + if(/$LOGIN_VERSION/) { $HPCUT_VER = $1;} } + } + else{ die "Hmm, you don't have a $LOGIN_FILE file, I'm bailing\n"; } + close(FILE); + + $status = "Found idev cmds in .login file."; + if( $found_hpcut_cmds == 0 && $IDEV_RM ne "uninstall"){ + print "Idev needs to insert a few startup commands in your $LOGIN_FILE file.\n"; + print "(To see the commands, set the IDEV_LOGIN_CMS to yes, and rerun idev.)\n"; + print "May I insert the commands? (yes,no, default is yes) >"; + $answer=<>; + + $found_yes=0; + if( $answer =~ /^y[es]*/i ){ $found_yes=1; } + if( $answer eq "\n" ){ $found_yes=1; } + if( !$found_yes ){ + print "Your $LOGIN_FILE file will not be touched. Bailing out.\n"; + return "Command Insert rejected.", "0";} + print "\nAppending these commands to your $LOGIN_FILE file:\n\n"; + print "$LOGIN_INSERT"; + print "-------------------------------------------\n\n"; + + open ( FILE,">>$LOGIN_FILE") or die "ERROR: Cannot open $LOGIN_FILE"; + print FILE "$LOGIN_INSERT"; + close( FILE); + + open ( FILE,">$HOME/.idev_tcsh") or die "ERROR: Cannot open $HOME/.idev_tcsh"; + print FILE "$LOGIN_CMDS"; + close( FILE); + + $status="Cmds added."; + $HPCUT_VER="latest"; + } +# Remove lines between #IDEV BEGIN +# and #IDEV END in login +# Copy old version to .login.$$ +# Make backup in /tmp--super safe + if($IDEV_RM eq "uninstall" ){ + copy($LOGIN_FILE, "/tmp/.login.$$") or die ".login backup to /tmp failed"; + move($LOGIN_FILE, "${LOGIN_FILE}.$$"); + open ( FILE1, ">$LOGIN_FILE") or die "ERROR: Cannot open $LOGIN_FILE." . + "MOVE ${LOGIN_FILE}.$$ back to ${LOGIN_FILE}"; + open ( FILE2, "${LOGIN_FILE}.$$") or die "ERROR: Cannot open ${LOGIN_FILE}.$$."; + while(){ + + if( /^#IDEV BEGIN/ ){ + while(){ + if( /^#IDEV END/ ){ + $_=; last; } + } + } + + print FILE1 + + } + close(FILE1); + close(FILE2); + unlink($IDEVRC_FILE); + unlink("$HOME/.idev_tcsh"); + unlink("$HOME/.idevrc"); + } + + return $status, $HPCUT_VER; +} + +#------------------------------------------------------ + +# +#set -x +# Interactive DEVelement (idev) for batch jobs +# Inserts login commands needed by Idev. +# Kent Milfeld 1/22/2011 +# all rights reserved +# Now inserts startup cmds in startup file user uses. 2/8/2011 +# +# Prototype the perl function call -- only allow one argument ($SYSTEM). + +# At login bash executes /etc/profile, and then: +# bash inspects ~/.bash_profile, ~/.bash_login, and ~/.profile in order, +# It executes only one: the first one that exists and is readable. +# Idev uses the same order and state to determine where to put login info. +# If none exit, we will use ~/.bash_profile + + +sub bash_login_insertion(@){ + $SYSTEM=shift(@_); + $IDEV_DIR=shift(@_); + $IDEV_RM=shift(@_); + + # Include command in .login if needed. cmds_add (1=added,0=found) + ($status, $HPCUT_VER)=sh_shell_setup($SYSTEM,$IDEV_DIR,$IDEV_RM); + if($status eq "Command Insert rejected."){ + print "Status:$status HPCUT_VER:$HPCUT_VER\n"; + exit(1); + } +} + +sub sh_shell_setup (@) { +# bash_shell_setup +# Inserts HPCUT commands into .bash_login file for Bash shell, if not present +# HPCUT commands into .login file for C shell, if not present +# Prompts user for permission if commands not found. +# #IDEV BEGIN is the sentinal that indicates commands are present +# Version Number detected for automatic updates. + +# HPCUT LOGIN COMMANDS Commands required in .bash_login for Bash shell. +# $, " and ' must be protected with backslashes. + + my($system,$idev_dir,$idev_rm) = @_; + $HOME=$ENV{HOME}; +# Escape / for bash [[ ]] test. + +# If idev in /usr/bin, comment out path insert query +# /usr/bin should be in path +# query used when idev is not installed in /usr/bin +# Doing with comments-- add/noadd would be complicated +# by double substitutions-- may do this better later. + $line_do_nodo=" "; + if( $idev_dir eq "/user/bin" ){$line_do_nodo="#";} + + $LOGIN_BEGIN = '#IDEV BEGIN'; + $LOGIN_CMDS = + "#IDEV BEGIN (This line used to detect idev setup-- do not remove.) \n" . + " \n" . + "$line_do_nodo if [[ ! \$PATH =~ ${idev_dir}(\$|:) ]]; then \n" . + "$line_do_nodo export PATH=$idev_dir:\${PATH} \n" . + "$line_do_nodo fi \n" . + " \n" . + " export idev_hostname=`hostname` \n" . + " export IDEV_SETUP_VER=0.91 \n" . + " #if on compute node \n" . + " export idev_ip=`echo \$idev_hostname | sed \"s/.$SYSTEM.*//\"` \n" . + " if [[ \$idev_ip =~ i[0-9][0-9][0-9]-[0-9][0-9][0-9] || \\\n" . + " \$idev_ip =~ c[0-9][0-9][0-9]-[0-9][0-9][0-9] || \\\n" . + " \$idev_ip =~ vis[0-9] || \$idev_ip =~ visbig ]]; then \n" . + " \n" . + " # ls returns \"0\" if true, and !0 if false for if statement \n" . + " ls /tmp/idev_bash_env.[0-9]* &> /dev/null \n" . + " export idev_env_status=\$? \n" . + " \n" . + " if [ \$idev_env_status -eq 0 ]; then \n" . + " . /tmp/idev_bash_env* #source env file \n" . + " cd \$IDEV_PWD #go to submit dir. \n" . + " fi \n" . + " \n" . + " fi \n" . + " unset idev_hostname idev_ip idev_env_status \n" . + "#IDEV END (This line used to detect idev setup-- do not remove.) \n"; + + $LOGIN_INSERT = + "#IDEV BEGIN (This line used to detect idev setup-- do not remove.) \n" . + " if [ -r \$HOME/.idev_bash ] ; then \n" . + " source \$HOME/.idev_bash \n" . + " fi \n" . + "#IDEV END (This line used to detect idev setup-- do not remove.) \n"; + +# " if [[ ! \$PATH =~ \$USER\/bin(\$|:) ]]; then \n" . +# " export PATH=\${PATH}:\$HOME/bin \n" . +# " fi \n" . + + $LOGIN_VERSION = 'setenv\s+idev_ver\s+(\d+\.\d+)'; + +# GNU bash searches for these files in this order +# First one found is sourced, not others are sourced. +# 1.) .bash_profile 2.) .bash_login 3.) .profile +# idev will use the first one found + $LOGIN_FILE="none"; +$cnt=0; + if( -r "$HOME/.bash_profile" && $cnt == 0) { $LOGIN_FILE="$HOME/.bash_profile"; $cnt++ } + if( -r "$HOME/.bash_login" && $cnt == 0) { $LOGIN_FILE="$HOME/.bash_login"; $cnt++;} + if( -r "$HOME/.profile" && $cnt == 0) { $LOGIN_FILE="$HOME/.profile"; $cnt++;} + + $found_hpcut_cmds = 0; + if( $LOGIN_FILE ne "none") { + open ( FILE,"<$LOGIN_FILE") or die "ERROR: Cannot open $HOME/$LOGIN_FILE"; + while() { if(/^$LOGIN_BEGIN/) { $found_hpcut_cmds = 1;} + if(/$LOGIN_VERSION/) { $HPCUT_VER = $1;} } + } + else{ + $LOGIN_FILE="$HOME/.bash_profile"; + print "Hmm, you don't have a $LOGIN_FILE file, I'm creating one.\n"; + print "Idev will use: $LOGIN_FILE.\n"; + open ( FILE,">$LOGIN_FILE") or die "ERROR: Cannot create $LOGIN_FILE"; + } + close(FILE); + + $status = "Found idev cmds in the bash login file ($LOGIN_FILE)."; + if( $found_hpcut_cmds == 0 && $idev_rm ne "uninstall"){ + print "Idev needs to insert a few startup commands in your $LOGIN_FILE file.\n"; + print "(To see the commands, set the IDEV_LOGIN_CMS to yes, and rerun idev.)\n"; + print "May I insert the commands? (yes,no, default is yes) >"; + $answer=<>; + + $found_yes=0; + if( $answer =~ /^y[es]*/i ){ $found_yes=1; } + if( $answer eq "\n" ){ $found_yes=1; } + if( !$found_yes ){ + print "Your $LOGIN_FILE file will not be touched. Bailing out.\n"; + return "Command Insert rejected.", "0";} + print "\nAppending these commands to your $LOGIN_FILE file:\n\n"; + print "$LOGIN_INSERT"; + print "-------------------------------------------\n\n"; + open ( FILE,">>$LOGIN_FILE") or die "ERROR: Cannot open $LOGIN_FILE"; + print FILE "$LOGIN_INSERT"; + close(FILE); + + open (FILE,">$HOME/.idev_bash") or die "ERROR: Cannot open $HOME/.idev_bash"; + print FILE "$LOGIN_CMDS"; + close(FILE); + + $status="Cmds added."; + $HPCUT_VER="latest"; + } + +# Remove lines between #IDEV BEGIN +# and #IDEV END in +# Copy old version to .$$ +# Make backup in /tmp--super safe + if($idev_rm eq "uninstall" ){ + $bash_login=basename($LOGIN_FILE); + copy($LOGIN_FILE, "/tmp/$bash_login.$$") or die "$bash_login backup to /tmp failed"; + move($LOGIN_FILE, "${LOGIN_FILE}.$$"); + open ( FILE1, ">$LOGIN_FILE") or die "ERROR: Cannot open $LOGIN_FILE." . + "MOVE ${LOGIN_FILE}.$$ back to ${LOGIN_FILE}"; + open ( FILE2, "${LOGIN_FILE}.$$") or die "ERROR: Cannot open ${LOGIN_FILE}.$$."; + while(){ + + if( /^#IDEV BEGIN/ ){ + while(){ + if( /^#IDEV END/ ){ + $_=; last; } + } + } + + print FILE1 + + } + close(FILE1); + close(FILE2); + unlink($IDEVRC_FILE); + unlink("$HOME/.idev_bash"); + unlink("$HOME/.idevrc"); + } + + + return $status, $HPCUT_VER; +} +#http://www.zytrax.com/tech/web/regex.htm + +#------------------------------------------------------------------- + +# Extracts lines from failed submit file in /tmp (idev_submit_$$) +# Projects are listed after "Note that your available projects are"... +# Removes files when finished and returns array of project names +# submission doesn't fail for single project users. +# return "use_default" to tell idev to use blank string for project +# line in job resource specification. +# "has been submitted" will be returned in this case + +sub get_projects($){ + + $ppid=shift(@_); + open(SUBMIT_TEST,"/tmp/idev_acct_job_output_$ppid") or die; + $capture="off"; + $count=0; + while(){ + chomp; + if( /Note that your available projects are:/ ) { $capture="on"; } + if( /--------------------------------------/ ) { $capture="off";} + if( /FAILED/ ) { $capture="off";} + if( $capture eq "on" ) { + if( $count gt 0 ) { push(@projects,$_); } + $count++; + } +# Lonestar + if( /^Your job \d+ \("[^"]+"\) has been submitted/) {push(@projects,"use_default"); } +# Stampede + if( /^Submitted batch job \d+/) {push(@projects,"use_default"); } + } + unlink("/tmp/idev_submit_$ppid"); + print "@projects\n"; + +} #end of get_projects +# +# +# +# +# +# +# +# +#