#!/bin/csh
#
#  last update = 26 March 2012
#
#  This is a C-shell script to execute GAMESS, by typing
#       rungms JOB VERNO NCPUS >& JOB.log &
#  JOB    is the name of the 'JOB.inp' file to be executed,
#  VERNO  is the number of the executable you chose at 'lked' time,
#  NCPUS  is the number of processors to be used.
#
#  Unfortunately execution is harder to standardize than compiling,
#  so you have to do a bit more than name your machine type here:
#
#    a) choose the target for execution from the following list:
#           sockets, mpi, altix, cray-xt, ibm64-sp, sgi64
#       IBM Blue Gene uses separate execution files: ~/gamess/machines/ibm-bg
#
#       choose "sockets" if your compile time target was any of these:
#             axp64, hpux32, hpux64, ibm32, ibm64, linux32,
#             mac32, mac64, sgi32, sun32, sun64
#       as all of these systems use TCP/IP sockets.  Do not name your
#       specific compile time target, instead choose "sockets".
#
#       If your target was 'linux64', you may chose "sockets" or "mpi",
#       according to how you chose to compile.  The MPI example below
#       should be carefully matched against info found in 'readme.ddi'!
#
#           Search on the words typed in capital letters just below
#           in order to find the right place to choose each one:
#    b) choose a directory SCR where large temporary files can reside.
#       This should be the fastest possible disk access, very spacious,
#       and almost certainly a local disk.
#       Translation: do not put these files on a slow network file system!
#    c) choose a directory USERSCR on the file server where small ASCII
#       supplementary output files should be directed.
#       Translation: it is OK to put this on a network file system!
#    d) name the location GMSPATH of your GAMESS binary.
#    e) change the the VERNO default to the version number you chose when
#       running "lked" as the VERNO default, and maybe NCPUS' default.
#    f) make sure that the ERICFMT file name and MCPPATH pathname point to
#       your file server's GAMESS tree, so that all runs can find them.
#       Again, a network file system is quite OK for these two.
#    g) customize the execution section for your target below,
#       each has its own list of further requirements.
#    h) it is unwise to have every user take a copy of this script, as you
#       can *NEVER* update all the copies later on.  Instead, it is better
#       to ask other users to create an alias to point to a common script,
#       such as this in their C-shell .login file,
#             alias gms '/u1/mike/gamess/rungms'
#    i) it is entirely possible to make 'rungms' run in a batch queue,
#       be it PBS, DQS, et cetera.  This is so installation dependent
#       that we leave it to up to you, although we give examples.
#       See ~/gamess/tools, where there are two examples of "front-end"
#       scripts which can use this file as the "back-end" actual job.
#       We use the front-end "gms" on local Infiniband clusters using
#       both Sun Grid Engine (SGE), and Portable Batch System (PBS).
#       See also a very old LoadLeveler "ll-gms" for some IBM systems.
#
set TARGET=sockets
set SCR=/scr/$USER
set USERSCR=~$USER/scr
set GMSPATH=/u1/mike/gamess
#
set JOB=$1      # name of the input file xxx.inp, give only the xxx part
set VERNO=$2    # revision number of the executable created by 'lked' step
set NCPUS=$3    # number of compute processes to be run
#
# provide defaults if last two arguments are not given to this script
if (null$VERNO == null) set VERNO=00
if (null$NCPUS == null) set NCPUS=1
#
#  ---- the top third of the script is input and other file assignments ----
#
echo "----- GAMESS execution script 'rungms' -----"
set master=`hostname`
echo This job is running on host $master
echo under operating system `uname` at `date`
#
#      Batch schedular, if any, should provide its own working directory,
#      on every assigned node (if not, modify schedular's prolog script).
#      The SCHED variable, and schedular assigned work space, is used
#      below only in the MPI section.  See that part for more info.
                      set SCHED=none
if ($?PBS_O_LOGNAME)  set SCHED=PBS
if ($?SGE_O_LOGNAME)  set SCHED=SGE
if ($SCHED == SGE) then
   set SCR=$TMPDIR
   echo "SGE has assigned the following compute nodes to this run:"
   uniq $TMPDIR/machines
endif
if ($SCHED == PBS) then
   set SCR=/scratch/$PBS_JOBID
   echo "PBS has assigned the following compute nodes to this run:"
   uniq $PBS_NODEFILE
endif
#
echo "Available scratch disk space (Kbyte units) at beginning of the job is"
df -k $SCR
echo "GAMESS temporary binary files will be written to $SCR"
echo "GAMESS supplementary output files will be written to $USERSCR"

#        this added as experiment, February 2007
#        its intent is to detect large arrays allocated off the stack
limit stacksize 8192

#  Grab a copy of the input file.
#  In the case of examNN jobs, file is in tests/standard subdirectory.
#  In the case of exam-vbNN jobs, file is in vb2000's tests subdirectory.
if ($JOB:r.inp == $JOB) set JOB=$JOB:r      # strip off possible .inp
echo "Copying input file $JOB.inp to your run's scratch directory..."
if (-e $JOB.inp) then
   set echo
   cp  $JOB.inp  $SCR/$JOB.F05
   unset echo
else
   if (-e tests/standard/$JOB.inp) then
      set echo
      cp  tests/standard/$JOB.inp  $SCR/$JOB.F05
      unset echo
   else
      if (-e tests/$JOB.inp) then
         set echo
         cp  tests/$JOB.inp  $SCR/$JOB.F05
         unset echo
      else
         echo "Input file $JOB.inp does not exist."
         echo "This job expected the input file to be in directory `pwd`"
         echo "Please fix your file name problem, and resubmit."
         exit 4
      endif
   endif
endif

#    define many environment variables setting up file names.
#    anything can be overridden by a user's own choice, read 2nd.
source $GMSPATH/gms-files.csh
if (-e $HOME/.gmsrc) then
   echo "reading your own $HOME/.gmsrc"
   source $HOME/.gmsrc
endif
#
#    If a $GDDI input group is present, the calculation will be using
#    subgroups within DDI (the input NGROUP=0 means this isn't GDDI).
#
#    The master within each group must have a copy of INPUT, which is
#    dealt with below (prior to execution), once we know something about
#    the host names where INPUT is required.  The INPUT does not have
#    the global rank appended to its name, unlike all other files.
#
#    OUTPUT and PUNCH (and perhaps many other files) are opened on all
#    processes (not just the master in each subgroup), but unique names
#    will be generated by appending the global ranks.  Note that OUTPUT
#    is not opened by the master in the first group, but is used by all
#    other groups.  Typically, the OUTPUT from the first group's master
#    is the only one worth saving, unless perhaps if runs crash out.
#
#    The other files that GDDI runs might use are already defined above.
#
set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=0 ' | wc -l`
if ($ngddi > 0) then
   set GDDIjob=true
   echo "This is a GDDI run, keeping various output files on local disks"
   set echo
   setenv  OUTPUT $SCR/$JOB.F06
   setenv   PUNCH $SCR/$JOB.F07
   unset echo
else
   set GDDIjob=false
endif

#    data left over from a previous run might be precious, stop if found.
if ((-e $PUNCH) || (-e $MAKEFP) || (-e $TRAJECT) || (-e $RESTART) ) then
   echo "Please save, rename, or erase these files from a previous run:"
   echo "     $PUNCH,"
   echo "     $TRAJECT,"
   echo "     $RESTART, and/or"
   echo "     $MAKEFP,"
   echo "and then resubmit this computation."
   exit 4
endif

#  ---- the middle third of the script is to execute GAMESS ----
#
#  we show execution sections that should work for
#        sockets, mpi, altix, cray-xt, ibm64-sp, sgi64
#  and then two others
#        cray-x1, necsx
#  which are not mentioned at the top of this file, as they are quite stale.
#
#   Most workstations run DDI over TCP/IP sockets, and therefore execute
#   according to the following clause.  The installer must
#      a) Set the path to point to the DDIKICK and GAMESS executables.
#      b) Build the HOSTLIST variable as a word separated string, i.e. ()'s.
#         There should be one host name for every compute process that is
#         to be run.  DDIKICK will automatically generate a set of data
#         server processes (if required) on the same hosts.
#   An extended explanation of the arguments to ddikick.x can be found
#   in the file gamess/ddi/readme.ddi, if you have any trouble executing.
#
if ($TARGET == sockets) then
#
#        adjust the path pointing to GAMESS and DDIKICK binaries
#           The default path to GAMESS was already set above!
#     At Iowa State, we have many operating systems, and store files
#     in different partitions according to which system is being used.
#     The other nodes have a separate directory for each machine,
#     based on their host names.
#
#       special compilation for IBM AIX pSeries p4+   (uname AIX)
   if (`hostname` == ti.msg.chem.iastate.edu) set GMSPATH=/ti/mike/gamess
#       special compilation for Digital AXP500        (uname OSF1)
   if (`hostname` == in.msg.chem.iastate.edu) set GMSPATH=/in/mike/gamess
   if (`hostname` == sn.msg.chem.iastate.edu) set GMSPATH=/in/mike/gamess
#       special compilation for Apple Xserve (G4)     (uname Darwin)
   if (`hostname` == te.msg.chem.iastate.edu) set GMSPATH=/users/mike/desktop/gamess
#       special compilation for Sun E450 uSPARC       (uname also= SunOS)
   if (`hostname` == sc.msg.chem.iastate.edu) set GMSPATH=/sc/mike/gamess
#       special compilation for Sun SunFire 280R      (uname also= SunOS)
   if (`hostname` == hf.msg.chem.iastate.edu) set GMSPATH=/hf/mike/gamess
   if (`hostname` == ta.msg.chem.iastate.edu) set GMSPATH=/hf/mike/gamess
#       special compilation for Sun V40Z Opteron uS3  (uname also= SunOS)
   if (`hostname` == as.msg.chem.iastate.edu) set GMSPATH=/as/mike/gamess
#       special compilation for HP rx2600 Itan2       (uname HP-UX)
   if (`hostname` == zr.msg.chem.iastate.edu) set GMSPATH=/zr/mike/gamess
   if (`hostname` == nb.msg.chem.iastate.edu) set GMSPATH=/zr/mike/gamess
#       special compilation for SGI Altix 450         (uname also= Linux)
   if (`hostname` == br.msg.chem.iastate.edu) set GMSPATH=/br/mike/gamess
#       special compilation for SGI XE210             (uname also= Linux)
   if (`hostname` == se.msg.chem.iastate.edu) set GMSPATH=/se/mike/gamess
   if (`hostname` == sb.msg.chem.iastate.edu) set GMSPATH=/se/mike/gamess

#      -- some special settings for certain operating systems --

   set os=`uname`
#         IBM's AIX needs special setting if node is more than a 4-way SMP
   if ($os == AIX) setenv EXTSHM ON
#         Fedora Core 1 can't run DDI processes w/o placing a finite
#         but large limit on the stack size (2**27 bytes seems OK)
   if ($os == Linux) limit stacksize 131072
#         In case this Linux system is using Intel's Math Kernel Library
#         to obtain its BLAS, we insist each process runs single-threaded.
#         one variable is for MKL up to 9, the other from 10 on up.
   if ($os == Linux) setenv MKL_SERIAL YES
   if ($os == Linux) setenv MKL_NUM_THREADS 1

#         it is unlikely that you would need to change DDI_VER from 'new'!
#         some antique system lacking pthreads, for example, might have
#         to use the old DDI code, so we keep an execution example below.
   set DDI_VER='new'
   if (`hostname` == antique.msg.chem.iastate.edu) set DDI_VER='old'

#
#       Six examples of how to build the HOSTLIST are shown....
#           terminology: CPU= processor core,
#                       NODE= physical enclosure (box/blade)
#
#       1. Sequential execution is sure to be on this very same host
   if ($NCPUS == 1) then
      set NNODES=1
      set HOSTLIST=(`hostname`)
   endif
#
#       2. This is an example of how to run on a multi-core SMP enclosure,
#          where all CPUs (aka COREs) are inside a -single- NODE.
#     At other locations, you may wish to consider some of the examples
#     that follow below, after commenting out this ISU specific part.
   if ($NCPUS > 1) then
      switch (`hostname`)
         case se.msg.chem.iastate.edu:
         case sb.msg.chem.iastate.edu:
            if ($NCPUS > 2) set NCPUS=2
            set NNODES=1
            set HOSTLIST=(`hostname`:cpus=$NCPUS)
            breaksw
         case br.msg.chem.iastate.edu:
            if ($NCPUS > 4) set NCPUS=4
            set NNODES=1
            set HOSTLIST=(`hostname`:cpus=$NCPUS)
            breaksw
         case cd.msg.chem.iastate.edu:
         case zn.msg.chem.iastate.edu:
         case ni.msg.chem.iastate.edu:
         case co.msg.chem.iastate.edu:
         case pb.msg.chem.iastate.edu:
         case bi.msg.chem.iastate.edu:
         case po.msg.chem.iastate.edu:
         case at.msg.chem.iastate.edu:
         case sc.msg.chem.iastate.edu:
            if ($NCPUS > 4) set NCPUS=4
            set NNODES=1
            set HOSTLIST=(`hostname`:cpus=$NCPUS)
            breaksw
         case ga.msg.chem.iastate.edu:
         case ge.msg.chem.iastate.edu:
         case gd.msg.chem.iastate.edu:
            if ($NCPUS > 6) set NCPUS=6
            set NNODES=1
            set HOSTLIST=(`hostname`:cpus=$NCPUS)
            breaksw
         default:
            echo I do not know how to run this node in parallel.
            exit 20
      endsw
   endif
#
#       3. How to run in a single computer, namely the "localhost", so
#          this computer needn't have a proper Internet name.
#          This example also presumes SysV was deliberately *not* chosen
#          when DDI was compiled, so that host names have to be repeated,
#          instead of using the simpler localhost:cpus=$NCPU form.
#
#          This example is appropriate for use with the pre-compiled
#          Apple binary from our web site, provided it is uncommented,
#          and the passage #2 just above is deleted or commented out.
#
#--   set HOSTLIST=()
#--   @ n=1
#--   while ($n <= $NCPUS)
#--      set HOSTLIST=($HOSTLIST localhost)
#--      @ n++
#--   end
#--   set NNODES=$NCPUS
#
#       4. A phony example, of four dual processors (arbitrary names)
#          Since their names never change, we just can just specify them.
#          Note that we can use a short name like 'bb' if and only if
#          system name resolution can map them onto the true host names.
   if (`hostname` == aa.msg.chem.iastate.edu) then
      set NCPUS=8
      set NNODES=4
      set HOSTLIST=(aa:cpus=2 bb:cpus=2 cc:cpus=2 dd:cpus=2)
   endif
#
#       5. An example of 16 uniprocessor boxes in a Beowulf-type cluster.
#          Because they are uniprocessors, we just set NNODES = NCPUS.
#          Their host names fall into the pattern fly1 to fly16,
#          which we can turn into a HOSTLIST with a small loop.
   if (`hostname` == fly1.fi.ameslab.gov) then
      set NNODES=$NCPUS
      set HOSTLIST=()
      set nmax=$NCPUS
      if ($nmax > 16) set nmax=16
      @ CPU=1
      while ($CPU <= $nmax)
         set HOSTLIST=($HOSTLIST fly$CPU)
         @ CPU++
      end
      unset $CPU
   endif
#
#       6. Scalable Computing Lab's clusters running PBS batch queues.
#          Here the task is to manipulate the dynamically assigned host
#          names into the HOSTLIST string for the kickoff program,
#          and to request the host name of the fast network adapters.
#
    if ($?PBS_JOBID) then
#
#         The IBM cluster has two Gigabit adapters in each 4-way SMP,
#         while the AXP cluster is based on a Myrinet network.
      if (`uname` == AIX)   set NETEXT=".gig,.gig2"
      if (`uname` == Linux) set NETEXT=".myri"
#
#         repeated host names in the PBS host file indicate being assigned
#         CPUs in the same SMP enclosure, which we must count up correctly.
#         Fortunately PBS gives duplicate host names in a row, not scrambled.
#         The number of hosts in the PBS node file (nmax) should equal the
#         requested processor count, NCPUS.  We need to count duplicates
#         in order to learn the number of SMP enclosures, NNODES, and how
#         many CPUs inside each SMP were assigned (NSMPCPU).  For example,
#         if we are assigned the host names "a a a b b c c c" we must build
#         the string "a:cpus=3 b:cpus=2 c:cpus=3" so that ddikick.x will
#         know the SMP structure of the assigned node names.  (C-shell handles
#         variable substitution followed by colon gracefully by ${HOST}:cpus.)
#
      set HOSTLIST=()
      set nmax=`wc -l $PBS_NODEFILE`
      set nmax=$nmax[1]
      if ($nmax != $NCPUS) then
         echo There is processor count confusion
         exit
      endif
#            1st host in the list is sure to be a new SMP enclosure
      set MYNODE=`sed -n -e "1 p" $PBS_NODEFILE`
      set MYNODE=`echo $MYNODE | awk '{split($0,a,"."); print a[1]}'`
#            IPROC counts assigned processors (up to NCPUS),
#            NNODES counts number of SMP enclosures.
#            NSMPCPU counts processors in the current SMP enclosure
      @ IPROC = 2
      @ NNODES = 1
      @ NSMPCPU = 1
      set spacer1=":cpus="
      set spacer2=":netext="
      while($IPROC <= $nmax)
         set MYPROC=`sed -n -e "$IPROC p" $PBS_NODEFILE`
         set MYPROC=`echo $MYPROC | awk '{split($0,a,"."); print a[1]}'`
         if($MYPROC != $MYNODE) then
            set HOSTLIST = ($HOSTLIST $MYNODE$spacer1$NSMPCPU$spacer2$NETEXT)
            set MYNODE=$MYPROC
            @ NSMPCPU = 0
            @ NNODES++
         endif
         @ IPROC++
         @ NSMPCPU++
      end
      set HOSTLIST = ($HOSTLIST $MYNODE$spacer1$NSMPCPU$spacer2$NETEXT)
   endif
#
#        we have now finished setting up a correct HOSTLIST.
#        uncomment the next two if you are doing script debugging.
#--echo "The generated host list is"
#--echo $HOSTLIST
#
#
#        choose remote shell execution program.
#    Parallel run do initial launch of GAMESS on remote nodes by the
#    following program.  Note that the authentication keys for ssh
#    must have been set up correctly.
#    If you wish, choose 'rsh' using .rhosts authentication on next line.
   setenv DDI_RSH ssh
#
   if($DDI_RSH == ssh) then
      setenv DDI_RCP scp
   else
      setenv DDI_RCP rcp
   endif

#        One way to be sure that the master node of each subgroup
#        has its necessary copy of the input file is to stuff a
#        copy of the input file onto every single node right here.
   if ($GDDIjob == true) then
      @ n=2   # master in master group already did 'cp' above
      while ($n <= $NNODES)
         set host=$HOSTLIST[$n]
         set host=`echo $host | cut -f 1 -d :` # drop anything behind a colon
         echo $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
              $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
         @ n++
      end
   endif

#
#        Just make sure we have the binaries, before we try to run
#
   if ((-x $GMSPATH/gamess.$VERNO.x) && (-x $GMSPATH/ddikick.x)) then
   else
      echo The GAMESS executable gamess.$VERNO.x or else
      echo the DDIKICK executable ddikick.x
      echo could not be found in directory $GMSPATH,
      echo or else they did not properly link to executable permission.
      exit 8
   endif
#
#        OK, now we are ready to execute!
#    The kickoff program initiates GAMESS process(es) on all CPUs/nodes.
#
   if ($DDI_VER == new) then
      set echo
      $GMSPATH/ddikick.x $GMSPATH/gamess.$VERNO.x $JOB \
          -ddi $NNODES $NCPUS $HOSTLIST \
          -scr $SCR < /dev/null
      unset echo
   else
      set path=($GMSPATH $path)
      set echo
      ddikick.x $JOB $GMSPATH gamess.$VERNO.x $SCR $NCPUS $HOSTLIST < /dev/null
      unset echo
   endif
endif

#      ------ end of the TCP/IP socket execution section -------



#                     - a typical MPI example -
#
#         This section is customized to two possible MPI libraries:
#             Intel MPI (default),  or MVAPICH2 (commented out).
#             We do not know tunings to use openMPI correctly!!!
#         This section is customized to two possible batch schedulars:
#             Sun Grid Engine (SGE), or Portable Batch System (PBS)
#         used on two different Infiniband-based Rocks clusters.
#
#         See ~/gamess/tools/gms, which is a front-end script to submit
#         this file 'rungms' as a back-end script, to either schedular.
#
#                   if you are using some other MPI:
#         See ~/gamess/ddi/readme.ddi for information about launching
#         processes using other MPI libraries (each may be different).
#         Again: we do not know how to run openMPI effectively.
#
#                   if you are using some other batch schedular:
#         Illustrating other batch schedular's way's of providing the
#         hostname list is considered beyond the scope of this script.
#         Suffice it to say that
#             a) you will be given hostnames at run time
#             b) a typical way is a disk file, named by an environment
#                variable, containing the names in some format.
#             c) another typical way is an blank separated list in some
#                environment variable.
#         Either way, whatever the batch schedular gives you must be
#         sliced-and-diced into the format required by your MPI kickoff.
#
if ($TARGET == mpi) then
   #
   #      Besides the usual three arguments to 'rungms' (see top),
   #      we'll pass in a "processers per node" value, that is,
   #      all nodes are presumed to have equal numbers of cores.
   #
   #      When using on the Infiniband clusters at Iowa State,
   #      just pick target=mpi, version no. of the iMPI, and kickoff style.
   #
   set PPN=$4
   #
   #  Allow for compute process and data servers (one pair per core)
   #  note that NCPUS = #cores, and NPROCS = #MPI processes
   #
   @ NPROCS = $NCPUS + $NCPUS
   #
   #  Argonne's MPICH2, offers two possible kick-off procedures,
   #  guided by two disk files (A and B).
   #
   #  Kickoff procedure #1 uses mpd demons, which potentially collide
   #  if the same user runs multiple jobs that end up on the same nodes.
   #  This is called "3steps" here because three commands (mpdboot,
   #  mpiexec, mpdallexit) are needed to run.
   #  Kickoff procedure #2 is said to be faster, and involves only
   #  one command (mpiexec.hydra), so it is called "hydra" here.
   #
   #  Note that "mpirun" is not shown, since it cannot understand
   #  how to place a c.p. and a d.s. pair on a single core.
   #
   #  Other MPI implementations are often derived from Argonne's, so
   #  often offer these two styles.
   #  For example, iMPI and MVAPICH2 can choose "3steps",
   #  while iMPI (as of version 4.0) can choose "hydra".
   #
   set MPI_KICKOFF_STYLE=hydra
   #
   #  A. build HOSTFILE,
   #     This is always used below during file cleaning, and while
   #     creating the PROCFILE at step B, so we always make it.
   #     This file is explicitly used by "3steps" initiation.
   #
   setenv HOSTFILE $SCR/$JOB.nodes.mpd
   if (-e $HOSTFILE) rm $HOSTFILE
   touch $HOSTFILE
   #
   if ($NCPUS == 1) then
             # Serial run must be on this node itself!
      echo `hostname` >> $HOSTFILE
      set NNODES=1
   else
             # Parallel run gets node names from schedular's assigned list:
      if ($SCHED == SGE) then
         uniq $TMPDIR/machines $HOSTFILE
         set NNODES=`wc -l $HOSTFILE`
         set NNODES=$NNODES[1]
      endif
      if ($SCHED == PBS) then
         uniq $PBS_NODEFILE $HOSTFILE
         set NNODES=`wc -l $HOSTFILE`
         set NNODES=$NNODES[1]
      endif
   endif
   #           uncomment these if you are still setting up...
   #--echo '-----debug----'
   #--echo HOSTFILE $HOSTFILE contains
   #--cat $HOSTFILE
   #--echo '--------------'
   #
   #  B. the next file forces explicit "which process on what node" rules.
   #     The contents depend on the kickoff style.  This file is how
   #     we tell MPI to double-book the cores with two processes,
   #     thus accounting for both compute processes and data servers.
   #
   setenv PROCFILE $SCR/$JOB.processes.mpd
   if (-e $PROCFILE) rm $PROCFILE
   touch $PROCFILE

   switch ($MPI_KICKOFF_STYLE)

   case 3steps:

   #
   if ($NCPUS == 1) then
      echo "-n $NPROCS -host `hostname` /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
   else
      if ($NNODES == 1) then
             # when all processes are inside a single node, it is simple!
             # all MPI processes, whether compute processes or data servers,
             # are just in this node.   (note: NPROCS = 2*NCPUS!)
         echo "-n $NPROCS -host `hostname` /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
      else
             # For more than one node, we want PPN compute processes on
             # each node, and of course, PPN data servers on each.
             # Hence, PPN2 is doubled up.
             # Front end script 'gms' is responsible to ensure that NCPUS
             # is a multiple of PPN, and that PPN is less than or equals
             # the actual number of cores in the node.
         @ PPN2 = $PPN + $PPN
         @ n=1
         while ($n <= $NNODES)
            set host=`sed -n -e "$n p" $HOSTFILE`
            set host=$host[1]
            echo "-n $PPN2 -host $host /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
            @ n++
         end
      endif
   endif
   breaksw

   case hydra:

   if ($NNODES == 1) then
             # when all processes are inside a single node, it is simple!
             # all MPI processes, whether compute processes or data servers,
             # are just in this node.   (note: NPROCS = 2*NCPUS!)
      @ PPN2 = $PPN + $PPN
      echo "`hostname`:$NPROCS" > $PROCFILE
   else
             # For more than one node, we want PPN compute processes on
             # each node, and of course, PPN data servers on each.
             # Hence, PPN2 is doubled up.
             # Front end script 'gms' is responsible to ensure that NCPUS
             # is a multiple of PPN, and that PPN is less than or equals
             # the actual number of cores in the node.
      @ PPN2 = $PPN + $PPN
      @ n=1
      while ($n <= $NNODES)
         set host=`sed -n -e "$n p" $HOSTFILE`
         set host=$host[1]
         echo "${host}:$PPN2" >> $PROCFILE
         @ n++
      end
   endif
   breaksw
   endsw
   #           uncomment these if you are still setting up...
   #--echo '-----debug----'
   #--echo PROCFILE $PROCFILE contains
   #--cat $PROCFILE
   #--echo '--------------'
   #
   #     add Intel MPI to the library path and to the execution path.
   #        version number might be 3.2.1, 4.0.1.007, 4.0.2.003, or ...
   #
   setenv LD_LIBRARY_PATH /opt/intel/impi/4.0.2.003/lib64:$LD_LIBRARY_PATH
   set path=(/opt/intel/impi/4.0.2.003/bin64 $path)
   #
   #  add MVAPICH2 to the execution path
   #--set path=(/usr/mpi/gcc/mvapich2-1.2p1/bin $path)
   #
   #     tunings below are specific to Intel MPI 3.2 and/or 4.0:
   #        a very important option avoids polling for incoming messages
   #           which allows us to compile DDI in pure "mpi" mode,
   #           and get sleeping data servers if the run is SCF level.
   #        trial and error showed process pinning slows down GAMESS runs,
   #        set debug option to 2 or to 5 to see messages while kicking off,
   #        set statistics option to 1 or 2 to collect messaging info,
   #        iMPI 3.2 allows specification of a DAPL 2.0 library,
   #        iMPI 4.0 defaults fabric to shm,dapl: forcing dapl only is faster.
   #
   set echo
   setenv I_MPI_WAIT_MODE enable
   setenv I_MPI_PIN disable
   setenv I_MPI_DEBUG 0
   setenv I_MPI_STATS 0
   setenv I_MPI_FABRICS dapl
   setenv I_MPI_DAT_LIBRARY libdat2.so
   #      in case someone wants to try the "tag matching interface",
   #      an option which unfortunately ignores the WAIT_MODE in 4.0.2!
   #--setenv I_MPI_FABRICS tmi
   #--setenv I_MPI_TMI_LIBRARY libtmi.so
   #--setenv I_MPI_TMI_PROVIDER psm
   #--setenv TMI_CONFIG /share/apps/intel/impi/4.0.2.003/intel64/etc/tmi.conf
   unset echo
   #
   #      similar tunings for MVAPICH2 are
   #--setenv MV2_USE_BLOCKING 1
   #--setenv MV2_ENABLE_AFFINITY 0
   #
   #         ... thus ends setting up the process initiation,
   #             tunings, pathnames, library paths, for the MPI.
   #
   #  set up Intel MKL (math kernel library).  As of version 10, GAMESS
   #  links MKL statically, and also for single threaded execution.
   #  So, you probably don't need to sweat this...
   #--setenv LD_LIBRARY_PATH /opt/intel/mkl/10.0.3.020/lib/em64t
   #--setenv LD_LIBRARY_PATH /opt/intel/composerxe-2011.1.107/mkl/lib/intel64
   #--setenv MKL_SERIAL YES
   #--setenv MKL_NUM_THREADS 1
   #
   #     Set up Fragment MO runs (or the other runs exploiting subgroups).
   #     One way to be sure that the master node of each subgroup
   #     has its necessary copy of the input file is to stuff a
   #     copy of the input file onto every single node right here.
   if ($GDDIjob == true) then
      setenv DDI_RCP scp
      set nmax=`wc -l $HOSTFILE`
      set nmax=$nmax[1]
      set lasthost=$master
      @ n=2   # input has already been copied into the master node.
      while ($n <= $nmax)
         set host=`sed -n -e "$n p" $HOSTFILE`
         set host=$host[1]
         if ($host != $lasthost) then
            echo $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
                 $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
            set lasthost=$host
         endif
         @ n++
      end
   endif
   #
   #      Next pertains to thread count in case GAMESS has been
   #      linked to "libcchem" (see that program's readme file),
   #      otherwise, these settings will just be ignored.
   #
   #      the profiling variable inserts extra timing output into the
   #      log files.  Omit it, if you like.
   #
   #      "libcchem" controls the CPU thread count by NUM_THREADS,
   #      which defaults to the number of CPU cores, if not given.
   #      Note that the number of CPU threads doing quantum chemistry
   #      will always be one less than NUM_THREADS, as one core is
   #      required to do GPU/CPU management things, except if the
   #      number of cores is just 1 (of course it runs 1 thread then).
   #
   #      "libcchem" controls the GPU devices used by GPU_DEVICES,
   #      which is a list of the devices to be used.  If not given,
   #      which safest, all avaliable GPUs will be detected and used.
   #
   #      You will need to put a path to your CUDA run time libraries here.
   #
   #      Finally, note that the front end 'gms' script must edit
   #      the string 'NUMGPU=0' to the desired number, after which,
   #      the if clause will actually be executed!
   #      Or, hardwire this to non-zero if all your nodes own GPUs.
   #      Setting GPU_DEVICES to -1 runs CPU threads, only.
   #
   @ NUMGPU=0
   if ($NUMGPU > 0) then
      @ NUMCPU = $PPN - 1
      echo libcchem kernels will use $NUMCPU cores and $NUMGPU GPUs per node...
      set echo
      setenv CCHEM_PROFILE 1
      setenv NUM_THREADS $PPN
      #--if ($NUMGPU == 0) setenv GPU_DEVICES -1
      #--if ($NUMGPU == 2) setenv GPU_DEVICES 0,1
      #--if ($NUMGPU == 4) setenv GPU_DEVICES 0,1,2,3
      setenv LD_LIBRARY_PATH /share/apps/cuda/lib64:$LD_LIBRARY_PATH
      ###### LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH
      unset echo
   else
      setenv GPU_DEVICES -1
   endif
   #
   #  Now, at last, we can actually kick-off the MPI processes...
   #
   echo "MPI kickoff will run GAMESS on $NCPUS cores in $NNODES nodes."
   echo "The binary to be executed is /home/mike/gamess/gamess.$VERNO.x"
   echo "MPI will run $NCPUS compute processes and $NCPUS data servers,"
   echo "    placing $PPN of each process type onto each node."
   echo "The scratch disk space on each node is $SCR, with free space"
   df -k $SCR
   #
   chdir $SCR
   #
   switch ($MPI_KICKOFF_STYLE)

   case 3steps:
      #
      #  a) bring up a 'ring' of MPI demons
      #
      set echo
      mpdboot --rsh=ssh -n $NNODES -f $HOSTFILE
      #
      #  b) kick off the compute processes and the data servers
      #
      mpiexec -configfile $PROCFILE < /dev/null
      #
      #  c) shut down the 'ring' of MPI demons
      #
      mpdallexit
      unset echo
      breaksw
   #
   case hydra:
      set echo
      setenv I_MPI_HYDRA_ENV all
      setenv I_MPI_PERHOST $PPN2
      mpiexec.hydra -f $PROCFILE -n $NPROCS \
            /home/mike/gamess/gamess.$VERNO.x < /dev/null
      unset echo
      breaksw
   case default:
      echo No valid MPI startup procedure chosen.
      exit
   endsw
   #
   #    keep HOSTFILE, as it is passed to the file erasing step below
   rm -f $PROCFILE
   #
endif
#      ------ end of the MPI execution section -------


#     SGI Altix or ICE, using ProPack's mpt MPI library, and PBS batch queues
#
if ($TARGET == altix) then
#
#  James Ianni bumped up two values in the script from Dave Anderson,
#  but not the third one shown.
#
   setenv MPI_BUFS_THRESHOLD 32
#    default:  96 pages (1 page = 16KB), Max:  1 million pages
   setenv MPI_BUFS_PER_PROC 512
#    default:  32 pages (1 page = 16KB), Max:  1 million pages
   setenv MPI_BUFS_PER_HOST 32

   setenv GMSPATH /usr/local/u/boatzj/gamess
   cat ${PBS_NODEFILE} | sort  > $SCR/$JOB.nodes.$$
   cat $SCR/$JOB.nodes.$$ $SCR/$JOB.nodes.$$ | sort > $SCR/$JOB.2xnodes.$$
   setenv PBS_NODEFILE $SCR/$JOB.2xnodes.$$

#-debug
#--   echo "Contents of PBS_NODEFILE are ..."
#--   cat $PBS_NODEFILE
#--   echo "PBS_NODEFILE has the following number of hostnames:"
#--   cat $PBS_NODEFILE | wc -l
#-debug

   @ NPROCS = $NCPUS + $NCPUS
   chdir $SCR
   set echo
   mpiexec_mpt -v -n $NPROCS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
   rm $SCR/$JOB.nodes.$$
   rm $SCR/$JOB.2xnodes.$$
endif


#   CRAY-XTn (N=3,4,5) running GAMESS/DDI over MPI wants you to
#      a) set the path to point to the GAMESS executable
#      b) set SMP_SIZE to the number of cores in each XT node
#
#   This machine runs only one MPI process/core, with most of these
#   able to be compute processes.  DDI_DS_PER_NODE lets you pick
#   how many of processes are to function as data servers.
#
#   The TPN variable below lets you use more memory, by wasting
#   some of the processors, if that is needed to do your run.
#   Not quite sure how 'aprun' picks this parameter up...
#
#   This machine doesn't allow FORTRAN to access the file server
#   directly.  As a work-around, input data like the error function
#   table has to be copied to the working disk, and possible output
#   files rescued from there after the run ends.
#   Since the working disk is one massive file system seen by all
#   nodes, GDDI is easy: all files read the very same INPUT file,
#   so nothing very complicated need be done to set up GDDI runs.
#
#   The two variables controlling run-time behavior are still
#   under experimentation.
#
if ($TARGET == cray-xt) then
       #   Jerry's XT
   set GMSPATH=/u/home/boatzj/gamess
   set SMP_SIZE=8  # number of cpus/cores per node (XT3 = dual CPU nodes)
       #   Meng-Shiou's XT
   #--set GMSPATH=/lustre/scratch/wums/QChem/gamess_Mar10_ORG/gamess
   #--set SMP_SIZE=12  # number of cpus/cores per node (XT5 = hex CPU nodes)

   set TPN=$4      # number of processes per node (TPN=tasks/node)
   if (null$TPN == null) set TPN=$SMP_SIZE
   if ($TPN > $SMP_SIZE) set TPN=$SMP_SIZE

   if (!(-e $SCR/$JOB)) mkdir $SCR/$JOB
   cp    $ERICFMT $SCR/$JOB/ericfmt.dat
   cp -r $MCPPATH $SCR/$JOB/

   set echo
   chdir $SCR/$JOB
   setenv ERICFMT $SCR/$JOB/ericfmt.dat
   setenv MCPPATH $SCR/$JOB/mcpdata

   setenv DDI_DS_PER_NODE 1
#---   setenv MPICH_UNEX_BUFFER_SIZE 90000000
   setenv MPICH_MAX_SHORT_MSG_SIZE 4000
   aprun -n $NCPUS -N $TPN $GMSPATH/gamess.$VERNO.x $JOB
   unset echo

   if (-e $SCR/scr/$JOB.efp)   cp $SCR/scr/$JOB.efp $USERSCR
   if (-e $SCR/scr/$JOB.gamma) cp $SCR/scr/$JOB.gamma $USERSCR
   if (-e $SCR/scr/$JOB.trj)   cp $SCR/scr/$JOB.trj $USERSCR
   if (-e $SCR/scr/$JOB.rst)   cp $SCR/scr/$JOB.rst $USERSCR
   if (-e $SCR/scr/$JOB.dat)   cp $SCR/scr/$JOB.dat $USERSCR
   rm -f  $SCR/$JOB/ericfmt.dat
   rm -rf $SCR/$JOB/mcpdata
   rmdir  $SCR/$JOB
endif


#   The IBM SP running DDI using mixed LAPI/MPI messaging wants you to
#      a) set the path to point to the GAMESS executable
#      b) define hosts in a host file, which are probably defined by
#         a batch queue system.  An example for LoadLeveler is given.
#   Please note that most IBM systems schedule their batch jobs with
#   the LoadLeveler software product.  Please see gamess/tools/llgms for
#   a "front-end" script that submits this script as a "back-end" job,
#   with all necessary LL accouterments inserted at the top of the job.
#
if ($TARGET == ibm64-sp) then
#
#     point this to where your GAMESS executable is located
   set path=($path /u1/mike/gamess)
#
#     error messages defaulted to American English, try C if lacking en_US
   setenv LOCPATH /usr/lib/nls/loc:/usr/vacpp/bin
   setenv LANG en_US
#
#     this value is picked up inside DDI, then used in a "chdir $SCR"
   setenv DDI_SCRATCH $SCR
#
#     define the name of a host name file.
#
   setenv HOSTFILE $SCR/$JOB.poehosts
   if (-e $HOSTFILE) rm -f $HOSTFILE
#
#        If the job was scheduled by LoadLeveler, let LL control everything.
#
   if ($?LOADLBATCH) then
#        just get POE to tell us what nodes we were dynamically assigned to.
      /usr/bin/poe hostname -stdoutmode ordered > $HOSTFILE
      set SMP_SIZE = $TPN
#
#        Otherwise, if this was not an LoadLeveler job, here's a hack!
#        It is unlikely this will match your SP's characteristics, as
#        we just guess its a 4-way node, 4 processors, run interactively.
#        It is here mainly to illustrate the sort of MP_XXX's you need.
   else
      set SMP_SIZE=4
      set NCPUS=4
      set NNODES=1
      echo `hostname`  > $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE

      echo "Variables controlling Parallel Environment process kickoff are"
      set echo
      setenv MP_NODES          $NNODES
      setenv MP_PROCS          $NCPUS
      setenv MP_HOSTFILE       $HOSTFILE
      setenv MP_CPU_USE        unique
      setenv MP_ADAPTER_USE    dedicated
#       GAMESS is implemented using both MPI and LAPI active messages.
      setenv MP_MSG_API        MPI,LAPI
      setenv MP_EUILIB         us
#       SP systems with one switch adapter might use css0, not striping csss
      setenv MP_EUIDEVICE      csss
      setenv MP_RESD           no
      unset echo
   endif

#       and now we are ready to execute, using poe to kick off the tasks.
   @ NNODES = ($NCPUS - 1) / $SMP_SIZE + 1
   echo "Running $NCPUS processes on $NNODES nodes ($SMP_SIZE-way SMP)."
   set echo
   /usr/bin/poe gamess.$VERNO.x $JOB -stdinmode none
   unset echo
endif


#   SGI Origin (a very old machine) running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
#   NOTE!  This does not mean SGI Altix or ICE!!!
#   We've heard that setting the environment variable
#       SMA_SYMMETRIC_SIZE to 2147483648 (2 GB)
#   may be helpful if you see DDI_SHPALLOC error messages.
#
if ($TARGET == sgi64) then
   set GMSPATH=/home/hbar4/people/schmidt/gamess
   chdir $SCR
   set echo
   mpirun -np $NCPUS $GMSPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif


#   Cray X1 running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
#   this is not mentioned above, as it hasn't been verified for ages and ages.
if ($TARGET == cray-x1) then
   set GMSPATH=/u1/mike/gamess
   set OPTS="-m exclusive"
   if ($NCPUS > 16) then
      set PERNODE=16
   else
      set PERNODE=$NCPUS
   endif
   chdir $SCR
   set echo
   aprun -c core=0 -n $NCPUS -N $PERNODE $OPTS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif


#   NEC SX Series wants you to
#      a) set the path variable to point to the GAMESS executable
#   this is not mentioned above, as it hasn't been verified for ages and ages.
if ($TARGET == necsx) then
   set GMSPATH=/u1/mike/gamess

   chdir $SCR
   setenv F_RECLUNIT BYTE
   setenv F_ABORT YES
   setenv F_ERROPT1 252,252,2,2,1,1,2,1
   setenv F_PROGINF detail
   setenv F_SETBUF 4096
   echo Running $NCPUS compute processes and $NCPUS data server processes...
   @ NPROCS = $NCPUS + $NCPUS
   set echo
   mpirun -np $NPROCS $GMSPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif
#
#  ---- the bottom third of the script is to clean up all disk files ----
#  It is quite useful to display to users how big the disk files got to be.
#
echo ----- accounting info -----
#
#   in the case of GDDI runs, we save the first PUNCH file only.
#   If something goes wrong, the .F06.00x, .F07.00x, ... from the
#   other groups are potentially interesting to look at.
if ($GDDIjob == true) cp $SCR/$JOB.F07 ~/scr/$JOB.dat
#
#   Clean up the master's scratch directory.
#
echo Files used on the master node $master were:
ls -lF $SCR/$JOB.*
rm -f  $SCR/$JOB.F*
#
#   Clean/Rescue any files created by the VB2000 plug-in
if (-e $SCR/$JOB.V84)        mv $SCR/$JOB.V84     $USERSCR
if (-e $SCR/$JOB.V80)        rm -f $SCR/$JOB.V*
if (-e $SCR/$JOB.TEMP02)     rm -f $SCR/$JOB.TEMP*
if (-e $SCR/$JOB.orb)        mv $SCR/$JOB.orb     $USERSCR
if (-e $SCR/$JOB.vec)        mv $SCR/$JOB.vec     $USERSCR
if (-e $SCR/$JOB.mol)        mv $SCR/$JOB.mol     $USERSCR
if (-e $SCR/$JOB.molf)       mv $SCR/$JOB.molf    $USERSCR
if (-e $SCR/$JOB.mkl)        mv $SCR/$JOB.mkl     $USERSCR
if (-e $SCR/$JOB.xyz)        mv $SCR/$JOB.xyz     $USERSCR
ls $SCR/${JOB}-*.cube > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.cube $USERSCR
rm -f $SCR/${JOB}.lis
ls $SCR/${JOB}-*.grd > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.grd $USERSCR
rm -f $SCR/${JOB}.lis
ls $SCR/${JOB}-*.csv > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.csv $USERSCR
rm -f $SCR/${JOB}.lis
#
#   Clean up scratch directory of remote nodes.
#
#   This may not be necessary, e.g. on a T3E where all files are in the
#   same directory, and just got cleaned out by the previous 'rm'.  Many
#   batch queue managers provide cleaning out of scratch directories.
#   It still may be interesting to the user to see the sizes of files.
#
#   The 'lasthost' business prevents multiple cleanup tries on SMP nodes.
#
if ($TARGET == sockets) then
   set nmax=${#HOSTLIST}
   set lasthost=$HOSTLIST[1]
   @ n=2   # master already cleaned above
   while ($n <= $nmax)
      set host=$HOSTLIST[$n]
      set host=`echo $host | cut -f 1 -d :`   # drop anything behind a colon
      if ($host != $lasthost) then
         echo Files from $host are:
         $DDI_RSH $host -l $USER -n "ls -l $SCR/$JOB.*"
         $DDI_RSH $host -l $USER -n "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
endif
#
#    This particular example is for the combination iMPI, w/SGE or PBS.
#    We have inherited a file of unique node names from above.
#    There is an option to rescue the output files from group DDI runs,
#    such as FMO, in case you need to see the other group's outputs.
if ($TARGET == mpi) then
   set nnodes=`wc -l $HOSTFILE`
   set nnodes=$nnodes[1]
   @ n=1
   set master=`hostname`
           # burn off the .local suffix in our cluster's hostname
   set master=$master:r
   while ($n <= $nnodes)
      set host=`sed -n -e "$n p" $HOSTFILE`
           # in case of openMPI, unwanted stuff may follow the hostname
      set host=$host[1]
      if ($host != $master) then
         echo Files used on node $host were:
         #---------FMO rescue------
         #--if ($GDDIjob == true) then
         #--   echo "========= OUTPUT from node $host is =============="
         #--   ssh $host -l $USER "cat $SCR/$JOB.F06*"
         #--endif
         #---------FMO rescue------
         ssh $host -l $USER "ls -l $SCR/$JOB.*"
         ssh $host -l $USER "rm -f $SCR/$JOB.*"
      endif
      @ n++
   end
#          clean off the last file on the master's scratch disk.
   rm -f $HOSTFILE
   #
   if ($?I_MPI_STATS) then
      if ($I_MPI_STATS > 0) mv $SCR/stats.txt ~/$JOB.$NCPUS.stats
   endif
endif
#
#   IBM SP cleanup code...might need to be something other than 'rsh'.
#
if ($TARGET == ibm64-sp) then
   set lasthost=$master
   @ n=2   # we already cleaned up the master node just above.
   @ nmax=$NCPUS
   while ($n <= $nmax)
      set host=`sed -n -e "$n p" $HOSTFILE`
      if ($host != $lasthost) then
         echo Files used on node $host were:
         rsh $host "ls -l $SCR/$JOB.*"
         rsh $host "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
   rm -f $HOSTFILE
endif
#
#  and this is the end
#
date
time
exit
