#!/bin/csh
#
#  last update = 3 August 2011
#
#  This is a C-shell script to execute GAMESS, by typing
#       rungms JOB VERNO NCPUS >& JOB.log &
#  JOB    is the name of the 'JOB.inp' file to be executed,
#  VERNO  is the number of the executable you chose at 'lked' time,
#  NCPUS  is the number of processors to be used.
#
#  Unfortunately execution is harder to standardize than compiling,
#  so you have to do a bit more than name your machine type here:
#
#    a) choose the target for execution from the following list:
#           sockets, mpi, altix, cray-xt, ibm64-sp, sgi64
#       IBM Blue Gene uses separate execution files, see ~/gamess/misc/ibm-bg
#
#       choose "sockets" if your compile time target was any of these:
#             axp64, hpux32, hpux64, ibm32, ibm64, linux32,
#             mac32, mac64, sgi32, sun32, sun64
#       as all of these systems use TCP/IP sockets.  Do not name your
#       specific compile time target, instead choose "sockets".
#
#       If your target was 'linux64', you may chose "sockets" or "mpi",
#       according to how you chose to compile.  The MPI example below
#       should be carefully matched against info found in 'readme.ddi'!
#
#           Search on the words typed in capital letters just below
#           in order to find the right place to choose each one:
#    b) choose a directory SCR where large temporary files can reside.
#       This should be the fastest possible disk access, very spacious,
#       and almost certainly a local disk.
#       Translation: do not put these files on a slow network file system!
#    c) change the the VERNO default to the version number you chose when
#       running "lked" as the VERNO default, and maybe NCPUS' default.
#    d) perhaps change the location of the PUNCH, TRAJECT, MAKEFP, and
#       RESTART files below.  These, along with the standard output, should
#       probably be routed to a central file server (using a network file
#       system to place these in user's normal storage is quite OK).
#    e) make sure that the ERICFMT file name and MCPPATH pathname point to
#       your file server's GAMESS tree, so that all runs can find them.
#       Again, a network file system is quite OK for these two.
#    f) customize the execution section for your target below,
#       each has its own list of further requirements.
#    g) it is unwise to have every user take a copy of this script, as you
#       can *NEVER* update all the copies later on.  Instead, it is better
#       to ask other users to create an alias to point to a common script,
#       such as this in their C-shell .login file,
#             alias gms '/u1/mike/gamess/rungms'
#    h) it is entirely possible to make 'rungms' run in a batch queue,
#       be it PBS, DQS, et cetera.  This is so installation dependent
#       that we leave it to up to you, although we give examples.
#       See ~/gamess/misc, where there are two examples using this
#       file as the backend: one for Sun Grid Engine (sge-gms), and
#       one for LoadLeveler "ll-gms" on the IBM SP line.
#
set GMS_PATH=$GMSPATH
set TARGET=sockets
set SCR=./
set USERSCR=./
set WORKDIR=`pwd`
#
set JOB=$1      # name of the input file xxx.inp, give only the xxx part
set VERNO=$2    # revision number of the executable created by 'lked' step
set NCPUS=$3    # number of compute processes to be run
#
# provide defaults if last two arguments are not given to this script
if (null$VERNO == null) set VERNO=00
if (null$NCPUS == null) set NCPUS=1
#
#  ---- the top third of the script is input and other file assignments ----
#
echo "----- GAMESS execution script 'rungms' -----"
set master=`hostname`
echo This job is running on host $master
echo under operating system `uname` at `date`
echo "Available scratch disk space (Kbyte units) at beginning of the job is"
df -k $SCR

#        this added as experiment, February 2007
#        its intent is to detect large arrays allocated off the stack
limit stacksize 8192

#  Grab a copy of the input file.
#  In the case of EXAMnn jobs, this file might be in the "tests" subdirectory.
if ($JOB:r.inp == $JOB) set JOB=$JOB:r      # strip off possible .inp
echo "Copying input file $JOB.inp to your run's scratch directory..."
if (-e $JOB.inp) then
   set echo
   cp  $JOB.inp  $SCR/$JOB.F05
   unset echo
else if (-e $SCR/$JOB.F05) then
	echo "input file already present"
else
   if (-e $GMSPATH/tests/$JOB.inp) then
      set echo
      cp  $GMSPATH/tests/$JOB.inp  $SCR/$JOB.F05
      unset echo
   else
      echo "Input file $JOB.inp does not exist."
      echo "This job expected the input file to be in directory `pwd`"
      echo "Please fix your file name problem, and resubmit."
      exit 4
   endif
endif

source $GMSPATH/gms-files.csh
if (-e $HOME/.gmsrc) then
	source $HOME/.gmsrc
endif

#
#    If a $GDDI input group is present, the calculation will be using
#    subgroups within DDI (the input NGROUP=0 means this isn't GDDI).
#
#    The master within each group must have a copy of INPUT, which is
#    dealt with below (prior to execution), once we know something about
#    the host names where INPUT is required.  The INPUT does not have
#    the global rank appended to its name, unlike all other files.
#
#    OUTPUT and PUNCH (and perhaps many other files) are opened on all
#    processes (not just the master in each subgroup), but unique names
#    will be generated by appending the global ranks.  Note that OUTPUT
#    is not opened by the master in the first group, but is used by all
#    other groups.  Typically, the OUTPUT from the first group's master
#    is the only one worth saving, unless perhaps if runs crash out.
#
#    The other files that GDDI runs might use are already defined above.
#
set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=0 ' | wc -l`
if ($ngddi > 0) then
   set GDDIjob=true
   echo "This is a GDDI run, keeping various output files on local disks"
   set echo
   setenv  OUTPUT $SCR/$JOB.F06
   setenv   PUNCH $SCR/$JOB.F07
   unset echo
else
   set GDDIjob=false
endif

#    data left over from a previous run might be precious, stop if found.
if ((-e $PUNCH) || (-e $MAKEFP) || (-e $TRAJECT) || (-e $RESTART) ) then
   echo "Please save, rename, or erase these files from a previous run:"
   echo "     $PUNCH,"
   echo "     $TRAJECT,"
   echo "     $RESTART, and/or"
   echo "     $MAKEFP,"
   echo "and then resubmit this computation."
   exit 4
endif

#  ---- the middle third of the script is to execute GAMESS ----
#
#  we show execution sections that should work for
#        sockets, mpi, altix, cray-xt, ibm64-sp, sgi64
#  and then two others
#        cray-x1, necsx
#  which are not mentioned at the top of this file, as they are quite stale.
#
#   Most workstations run DDI over TCP/IP sockets, and therefore execute
#   according to the following clause.  The installer must
#      a) Set the path to point to the DDIKICK and GAMESS executables.
#      b) Build the HOSTLIST variable as a word separated string, i.e. ()'s.
#         There should be one host name for every compute process that is
#         to be run.  DDIKICK will automatically generate a set of data
#         server processes (if required) on the same hosts.
#   An extended explanation of the arguments to ddikick.x can be found
#   in the file gamess/ddi/readme.ddi, if you have any trouble executing.
#
if ($TARGET == sockets) then
#
#        set the path pointing to GAMESS and DDIKICK binaries
#     At Iowa State, we have many operating systems, and store files
#     in different partitions according to which system is being used.
#
#     One such partition is fed to all nodes of the same type and
#     operating system, e.g. the /cu directory below feeds all of
#     our 32 bit PCs running RedHat Fedora Core 1, by means of NFS.
#
#     Note that 'uname' is not a terribly specific way to choose
#     a list of all machines running a specific O/S release, so we
#     use the host names for finer control, for example, we have
#     three different Solaris releases on two different chip types.
#
   set os=`uname`
   set DDI_VER='new'
#      -- some special settings for certain operating systems --

#         IBM's AIX needs special setting if node is more than a 4-way SMP
   if ($os == AIX) setenv EXTSHM ON
#         Fedora Core 1 can't run DDI processes w/o placing a finite
#         but large limit on the stack size (2**27 bytes seems OK)
   if ($os == Linux) limit stacksize 131072
#         In case this Linux system is using Intel's Math Kernel Library
#         to obtain its BLAS, we insist each process runs single-threaded.
#         one variable is for MKL up to 9, the other from 10 on up.
   if ($os == Linux) setenv MKL_SERIAL YES
   if ($os == Linux) setenv MKL_NUM_THREADS 1

#
#       Six examples of how to build the HOSTLIST are shown....
#           terminology: CPU= processor core,
#                       NODE= physical enclosure (box/blade)
#
#       1. Sequential execution is sure to be on this very same host
#   if ($NCPUS == 1) then
#      set NNODES=1
#      set HOSTLIST=(`hostname`)
#   endif
#
#   The following will always run as is, giving you NCPUS number of 
#   processes on the launching computer. Activate the 4 lines above
#   and setup appropriate lines below this section with this section
#   commented out to run in paralle on multiple nodes.
# 
   set HOSTLIST=(localhost)
   @ n=2
   while ($n <= $NCPUS)
      set HOSTLIST=($HOSTLIST localhost)
      @ n++
   end
   @ n--
   set NNODES=$n
#
#
#        choose remote shell execution program.
#    Parallel run do initial launch of GAMESS on remote nodes by the
#    following program.  Note that the authentication keys for ssh
#    must have been set up correctly.
#    If you wish, choose 'rsh' using .rhosts authentication on next line.
   setenv DDI_RSH ssh
#
   if($DDI_RSH == ssh) then
      setenv DDI_RCP scp
   else
      setenv DDI_RCP rcp
   endif

#        One way to be sure that the master node of each subgroup
#        has its necessary copy of the input file is to stuff a
#        copy of the input file onto every single node right here.
   if ($GDDIjob == true) then
      @ n=2   # master in master group already did 'cp' above
      while ($n <= $NNODES)
         set host=$HOSTLIST[$n]
         set host=`echo $host | cut -f 1 -d :` # drop anything behind a colon
         echo $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
              $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
         @ n++
      end
   endif

#
#        Just make sure we have the binaries, before we try to run
#
   if ((-x $EXEPATH/gamess.$VERNO.x) && (-x $EXEPATH/ddikick.x)) then
   else
      echo The GAMESS executable gamess.$VERNO.x or else
      echo the DDIKICK executable ddikick.x
      echo could not be found in directory $EXEPATH,
      echo or else they did not properly link to executable permission.
      exit 8
   endif
#
#        OK, now we are ready to execute!
#    The kickoff program initiates GAMESS process(es) on all CPUs/nodes.
#
   if ($DDI_VER == new) then
      set echo
      $EXEPATH/ddikick.x $EXEPATH/gamess.$VERNO.x $JOB \
          -ddi $NNODES $NCPUS $HOSTLIST \
          -scr $SCR < /dev/null
      unset echo
   else
      set path=($EXEPATH $path)
      set echo
      ddikick.x $JOB $EXEPATH gamess.$VERNO.x $SCR $NCPUS $HOSTLIST < /dev/null
      unset echo
   endif
endif

#      ------ end of the TCP/IP socket execution sections -------

#                     - a typical MPI example -
#
#         This section is customized to two possible MPI libraries:
#             Intel MPI (default),  or MVAPICH2 (commented out).
#             We do not know tunings to use openMPI correctly!!!
#         This section is customized to two possible batch schedulars:
#             Sun Grid Engine (SGE), or Portable Batch System (PBS)
#         used on two different Infiniband-based Rocks clusters.
#
#         See ~/gamess/misc/gms, which is a front-end script to submit
#         this file 'rungms' as a back-end script, to either schedular.
#
#                   if you are using some other MPI:
#         See ~/gamess/ddi/readme.ddi for information about launching
#         processes using other MPI libraries (each may be different).
#         Again: we do not know how to run openMPI effectively.
#
#                   if you are using some other batch schedular:
#         Illustrating other batch schedular's way's of providing the
#         hostname list is considered beyond the scope of this script.
#         Suffice it to say that
#             a) you will be given hostnames at run time
#             b) a typical way is a disk file, named by an environment
#                variable, containing the names in some format.
#             c) another typical way is an blank separated list in some
#                environment variable.
#         Either way, whatever the batch schedular gives you must be
#         sliced-and-diced into the format required by your MPI kickoff.
#
if ($TARGET == mpi) then
   #
   #      Besides the usual three arguments to 'rungms' (see top),
   #      we'll pass in a "processers per node" value, that is,
   #      all nodes are presumed to have equal numbers of cores.
   #
   #      When using on the Infiniband clusters at Iowa State,
   #      just pick target=mpi, version no. of the iMPI, and kickoff style.
   #
   set PPN=$4
   #
   #  Allow for compute process and data servers (one pair per core)
   #  note that NCPUS = #cores, and NPROCS = #MPI processes
   #
   @ NPROCS = $NCPUS + $NCPUS
   #
   #  Argonne's MPICH2, offers two possible kick-off procedures,
   #  guided by two disk files (A and B).
   #
   #  Kickoff procedure #1 uses mpd demons, which potentially collide
   #  if the same user runs multiple jobs that end up on the same nodes.
   #  This is called "3steps" here because three commands (mpdboot,
   #  mpiexec, mpdallexit) are needed to run.
   #  Kickoff procedure #2 is said to be faster, and involves only
   #  one command (mpiexec.hydra), so it is called "hydra" here.
   #
   #  Note that "mpirun" is not shown, since it cannot understand
   #  how to place a c.p. and a d.s. pair on a single core.
   #
   #  Other MPI implementations are often derived from Argonne's, so
   #  often offer these two styles.
   #  For example, iMPI and MVAPICH2 can choose "3steps",
   #  while iMPI (as of version 4.0) can choose "hydra".
   #
   set MPI_KICKOFF_STYLE=hydra
   #
   #  A. build HOSTFILE,
   #     This is always used below during file cleaning, and while
   #     creating the PROCFILE at step B, so we always make it.
   #     This file is explicitly used by "3steps" initiation.
   #
   setenv HOSTFILE $SCR/$JOB.nodes.mpd
   if (-e $HOSTFILE) rm $HOSTFILE
   touch $HOSTFILE
   #
   if ($NCPUS == 1) then
             # Serial run must be on this node itself!
      echo `hostname` >> $HOSTFILE
      set NNODES=1
   else
             # Parallel run gets node names from schedular's assigned list:
      if ($SCHED == SGE) then
         uniq $TMPDIR/machines $HOSTFILE
         set NNODES=`wc -l $HOSTFILE`
         set NNODES=$NNODES[1]
      endif
      if ($SCHED == PBS) then
         uniq $PBS_NODEFILE $HOSTFILE
         set NNODES=`wc -l $HOSTFILE`
         set NNODES=$NNODES[1]
      endif
   endif
   #           uncomment these if you are still setting up...
   #--echo '-----debug----'
   #--echo HOSTFILE $HOSTFILE contains
   #--cat $HOSTFILE
   #--echo '--------------'
   #
   #  B. the next file forces explicit "which process on what node" rules.
   #     The contents depend on the kickoff style.  This file is how
   #     we tell MPI to double-book the cores with two processes,
   #     thus accounting for both compute processes and data servers.
   #
   setenv PROCFILE $SCR/$JOB.processes.mpd
   if (-e $PROCFILE) rm $PROCFILE
   touch $PROCFILE

   switch ($MPI_KICKOFF_STYLE)

   case 3steps:

   #
   if ($NCPUS == 1) then
      echo "-n $NPROCS -host `hostname` /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
   else
      if ($NNODES == 1) then
             # when all processes are inside a single node, it is simple!
             # all MPI processes, whether compute processes or data servers,
             # are just in this node.   (note: NPROCS = 2*NCPUS!)
         echo "-n $NPROCS -host `hostname` /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
      else
             # For more than one node, we want PPN compute processes on
             # each node, and of course, PPN data servers on each.
             # Hence, PPN2 is doubled up.
             # Front end script 'gms' is responsible to ensure that NCPUS
             # is a multiple of PPN, and that PPN is less than or equals
             # the actual number of cores in the node.
         @ PPN2 = $PPN + $PPN
         @ n=1
         while ($n <= $NNODES)
            set host=`sed -n -e "$n p" $HOSTFILE`
            set host=$host[1]
            echo "-n $PPN2 -host $host /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
            @ n++
         end
      endif
   endif
   breaksw

   case hydra:

   if ($NNODES == 1) then
             # when all processes are inside a single node, it is simple!
             # all MPI processes, whether compute processes or data servers,
             # are just in this node.   (note: NPROCS = 2*NCPUS!)
      @ PPN2 = $PPN + $PPN
      echo "`hostname`:$NPROCS" > $PROCFILE
   else
             # For more than one node, we want PPN compute processes on
             # each node, and of course, PPN data servers on each.
             # Hence, PPN2 is doubled up.
             # Front end script 'gms' is responsible to ensure that NCPUS
             # is a multiple of PPN, and that PPN is less than or equals
             # the actual number of cores in the node.
      @ PPN2 = $PPN + $PPN
      @ n=1
      while ($n <= $NNODES)
         set host=`sed -n -e "$n p" $HOSTFILE`
         set host=$host[1]
         echo "${host}:$PPN2" >> $PROCFILE
         @ n++
      end
   endif
   breaksw
   endsw
   #           uncomment these if you are still setting up...
   #--echo '-----debug----'
   #--echo PROCFILE $PROCFILE contains
   #--cat $PROCFILE
   #--echo '--------------'
   #
   #     add Intel MPI to the library path and to the execution path.
   #        version number might be 3.2.1, 4.0.1.007, 4.0.2.003, or ...
   #
   setenv LD_LIBRARY_PATH /opt/intel/impi/4.0.2.003/lib64:$LD_LIBRARY_PATH
   set path=(/opt/intel/impi/4.0.2.003/bin64 $path)
   #
   #  add MVAPICH2 to the execution path
   #--set path=(/usr/mpi/gcc/mvapich2-1.2p1/bin $path)
   #
   #     tunings below are specific to Intel MPI 3.2 and/or 4.0:
   #        a very important option avoids polling for incoming messages
   #           which allows us to compile DDI in pure "mpi" mode,
   #           and get sleeping data servers if the run is SCF level.
   #        trial and error showed process pinning slows down GAMESS runs,
   #        set debug option to 2 or to 5 to see messages while kicking off,
   #        set statistics option to 1 or 2 to collect messaging info,
   #        iMPI 3.2 allows specification of a DAPL 2.0 library,
   #        iMPI 4.0 defaults fabric to shm,dapl: forcing dapl only is faster.
   #
   set echo
   setenv I_MPI_WAIT_MODE enable
   setenv I_MPI_PIN disable
   setenv I_MPI_DEBUG 0
   setenv I_MPI_STATS 0
   setenv I_MPI_FABRICS dapl
   setenv I_MPI_DAT_LIBRARY libdat2.so
   #      in case someone wants to try the "tag matching interface",
   #      an option which unfortunately ignores the WAIT_MODE in 4.0.2!
   #--setenv I_MPI_FABRICS tmi
   #--setenv I_MPI_TMI_LIBRARY libtmi.so
   #--setenv I_MPI_TMI_PROVIDER psm
   #--setenv TMI_CONFIG /share/apps/intel/impi/4.0.2.003/intel64/etc/tmi.conf
   unset echo
   #
   #      similar tunings for MVAPICH2 are
   #--setenv MV2_USE_BLOCKING 1
   #--setenv MV2_ENABLE_AFFINITY 0
   #
   #         ... thus ends setting up the process initiation,
   #             tunings, pathnames, library paths, for the MPI.
   #
   #  set up Intel MKL (math kernel library).  As of version 10, GAMESS
   #  links MKL statically, and also for single threaded execution.
   #  So, you probably don't need to sweat this...
   #--setenv LD_LIBRARY_PATH /opt/intel/mkl/10.0.3.020/lib/em64t
   #--setenv LD_LIBRARY_PATH /opt/intel/composerxe-2011.1.107/mkl/lib/intel64
   #--setenv MKL_SERIAL YES
   #--setenv MKL_NUM_THREADS 1
   #
   #     Set up Fragment MO runs (or the other runs exploiting subgroups).
   #     One way to be sure that the master node of each subgroup
   #     has its necessary copy of the input file is to stuff a
   #     copy of the input file onto every single node right here.
   if ($GDDIjob == true) then
      setenv DDI_RCP scp
      set nmax=`wc -l $HOSTFILE`
      set nmax=$nmax[1]
      set lasthost=$master
      @ n=2   # input has already been copied into the master node.
      while ($n <= $nmax)
         set host=`sed -n -e "$n p" $HOSTFILE`
         set host=$host[1]
         if ($host != $lasthost) then
            echo $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
                 $DDI_RCP $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
            set lasthost=$host
         endif
         @ n++
      end
   else
      set GDDIjob=false
   endif
   #
   #      Next pertains to thread count in case GAMESS has been
   #      linked to "libcchem" (see that program's readme file),
   #      otherwise, these settings will just be ignored.
   #
   #      the profiling variable inserts extra timing output into the
   #      log files.  Omit it, if you like.
   #
   #      "libcchem" controls the CPU thread count by NUM_THREADS,
   #      which defaults to the number of CPU cores, if not given.
   #      Note that the number of CPU threads doing quantum chemistry
   #      will always be one less than NUM_THREADS, as one core is
   #      required to do GPU/CPU management things, except if the
   #      number of cores is just 1 (of course it runs 1 thread then).
   #
   #      "libcchem" controls the GPU devices used by GPU_DEVICES,
   #      which is a list of the devices to be used.  If not given,
   #      which safest, all avaliable GPUs will be detected and used.
   #
   #      You will need to put a path to your CUDA run time libraries here.
   #
   #      Finally, note that the front end 'gms' script must edit
   #      the string 'NUMGPU=0' to the desired number, after which,
   #      the if clause will actually be executed!
   #      Or, hardwire this to non-zero if all your nodes own GPUs.
   #      Setting GPU_DEVICES to -1 runs CPU threads, only.
   #
   @ NUMGPU=0
   if ($NUMGPU > 0) then
      @ NUMCPU = $PPN - 1
      echo libcchem kernels will use $NUMCPU cores and $NUMGPU GPUs per node...
      set echo
      setenv CCHEM_PROFILE 1
      setenv NUM_THREADS $PPN
      #--if ($NUMGPU == 0) setenv GPU_DEVICES -1
      #--if ($NUMGPU == 2) setenv GPU_DEVICES 0,1
      #--if ($NUMGPU == 4) setenv GPU_DEVICES 0,1,2,3
      setenv LD_LIBRARY_PATH /share/apps/cuda/lib64:$LD_LIBRARY_PATH
      ###### LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH
      unset echo
   else
      setenv GPU_DEVICES -1
   endif
   #
   #  Now, at last, we can actually kick-off the MPI processes...
   #
   echo "MPI kickoff will run GAMESS on $NCPUS cores in $NNODES nodes."
   echo "The binary to be executed is /home/mike/gamess/gamess.$VERNO.x"
   echo "MPI will run $NCPUS compute processes and $NCPUS data servers,"
   echo "    placing $PPN of each process type onto each node."
   echo "The scratch disk space on each node is $SCR, with free space"
   df -k $SCR
   #
   chdir $SCR
   #
   switch ($MPI_KICKOFF_STYLE)

   case 3steps:
      #
      #  a) bring up a 'ring' of MPI demons
      #
      set echo
      mpdboot --rsh=ssh -n $NNODES -f $HOSTFILE
      #
      #  b) kick off the compute processes and the data servers
      #
      mpiexec -configfile $PROCFILE < /dev/null
      #
      #  c) shut down the 'ring' of MPI demons
      #
      mpdallexit
      unset echo
      breaksw
   #
   case hydra:
      set echo
      setenv I_MPI_HYDRA_ENV all
      setenv I_MPI_PERHOST $PPN2
      mpiexec.hydra -f $PROCFILE -n $NPROCS \
            /home/mike/gamess/gamess.$VERNO.x < /dev/null
      unset echo
      breaksw
   case default:
      echo No valid MPI startup procedure chosen.
      exit
   endsw
   #
   #    keep HOSTFILE, as it is passed to the file erasing step below
   rm -f $PROCFILE
   #
endif
#      ------ end of the MPI execution section -------


#     SGI Altix or ICE, using ProPack's mpt MPI library, and PBS batch queues
#
if ($TARGET == altix) then
#
#  James Ianni bumped up two values in the script from Dave Anderson,
#  but not the third one shown.
#
   setenv MPI_BUFS_THRESHOLD 32
#    default:  96 pages (1 page = 16KB), Max:  1 million pages
   setenv MPI_BUFS_PER_PROC 512
#    default:  32 pages (1 page = 16KB), Max:  1 million pages
   setenv MPI_BUFS_PER_HOST 32

   setenv EXEPATH /usr/local/u/boatzj/gamess
   cat ${PBS_NODEFILE} | sort  > $SCR/$JOB.nodes.$$
   cat $SCR/$JOB.nodes.$$ $SCR/$JOB.nodes.$$ | sort > $SCR/$JOB.2xnodes.$$
   setenv PBS_NODEFILE $SCR/$JOB.2xnodes.$$

#-debug
#--   echo "Contents of PBS_NODEFILE are ..."
#--   cat $PBS_NODEFILE
#--   echo "PBS_NODEFILE has the following number of hostnames:"
#--   cat $PBS_NODEFILE | wc -l
#-debug

   @ NPROCS = $NCPUS + $NCPUS
   chdir $SCR
   set echo
   mpiexec_mpt -v -n $NPROCS $EXEPATH/gamess.$VERNO.x $JOB
   unset echo
   rm $SCR/$JOB.nodes.$$
   rm $SCR/$JOB.2xnodes.$$
endif


#   CRAY-XTn (N=3,4,5) running GAMESS/DDI over MPI wants you to
#      a) set the path to point to the GAMESS executable
#      b) set SMP_SIZE to the number of cores in each XT node
#
#   This machine runs only one MPI process/core, with most of these
#   able to be compute processes.  DDI_DS_PER_NODE lets you pick
#   how many of processes are to function as data servers.
#
#   The TPN variable below lets you use more memory, by wasting
#   some of the processors, if that is needed to do your run.
#   Not quite sure how 'aprun' picks this parameter up...
#
#   This machine doesn't allow FORTRAN to access the file server
#   directly.  As a work-around, input data like the error function
#   table has to be copied to the working disk, and possible output
#   files rescued from there after the run ends.
#   Since the working disk is one massive file system seen by all
#   nodes, GDDI is easy: all files read the very same INPUT file,
#   so nothing very complicated need be done to set up GDDI runs.
#
#   The two variables controlling run-time behavior are still
#   under experimentation.
#
if ($TARGET == cray-xt) then
       #   Jerry's XT
   set EXEPATH=/u/home/boatzj/gamess
   set SMP_SIZE=8  # number of cpus/cores per node (XT3 = dual CPU nodes)
       #   Meng-Shiou's XT
   #--set EXEPATH=/lustre/scratch/wums/QChem/gamess_Mar10_ORG/gamess
   #--set SMP_SIZE=12  # number of cpus/cores per node (XT5 = hex CPU nodes)

   set TPN=$4      # number of processes per node (TPN=tasks/node)
   if (null$TPN == null) set TPN=$SMP_SIZE
   if ($TPN > $SMP_SIZE) set TPN=$SMP_SIZE

   if (!(-e $SCR/$JOB)) mkdir $SCR/$JOB
   cp    $ERICFMT $SCR/$JOB/ericfmt.dat
   cp -r $MCPPATH $SCR/$JOB/

   set echo
   chdir $SCR/$JOB
   setenv ERICFMT $SCR/$JOB/ericfmt.dat
   setenv MCPPATH $SCR/$JOB/mcpdata

   setenv DDI_DS_PER_NODE 1
#---   setenv MPICH_UNEX_BUFFER_SIZE 90000000
   setenv MPICH_MAX_SHORT_MSG_SIZE 4000
   aprun -n $NCPUS -N $TPN $EXEPATH/gamess.$VERNO.x $JOB
   unset echo

   if (-e $SCR/scr/$JOB.efp)   cp $SCR/scr/$JOB.efp ~$USER/scr
   if (-e $SCR/scr/$JOB.gamma) cp $SCR/scr/$JOB.gamma ~$USER/scr
   if (-e $SCR/scr/$JOB.trj)   cp $SCR/scr/$JOB.trj ~$USER/scr
   if (-e $SCR/scr/$JOB.rst)   cp $SCR/scr/$JOB.rst ~$USER/scr
   if (-e $SCR/scr/$JOB.dat)   cp $SCR/scr/$JOB.dat ~$USER/scr
   rm -f  $SCR/$JOB/ericfmt.dat
   rm -rf $SCR/$JOB/mcpdata
   rmdir  $SCR/$JOB
endif


#   The IBM SP running DDI using mixed LAPI/MPI messaging wants you to
#      a) set the path to point to the GAMESS executable
#      b) define hosts in a host file, which are probably defined by
#         a batch queue system.  An example for LoadLeveler is given.
#   Please note that most IBM systems schedule their batch jobs with
#   the LoadLeveler software product.  Please see gamess/misc/llgms for
#   a "front-end" script that submits this script as a "back-end" job,
#   with all necessary LL accouterments inserted at the top of the job.
#
if ($TARGET == ibm64-sp) then
#
#     point this to where your GAMESS executable is located
   set path=($path /u1/mike/gamess)
#
#     error messages defaulted to American English, try C if lacking en_US
   setenv LOCPATH /usr/lib/nls/loc:/usr/vacpp/bin
   setenv LANG en_US
#
#     this value is picked up inside DDI, then used in a "chdir $SCR"
   setenv DDI_SCRATCH $SCR
#
#     define the name of a host name file.
#
   setenv HOSTFILE $SCR/$JOB.poehosts
   if (-e $HOSTFILE) rm -f $HOSTFILE
#
#        If the job was scheduled by LoadLeveler, let LL control everything.
#
   if ($?LOADLBATCH) then
#        just get POE to tell us what nodes we were dynamically assigned to.
      /usr/bin/poe hostname -stdoutmode ordered > $HOSTFILE
      set SMP_SIZE = $TPN
#
#        Otherwise, if this was not an LoadLeveler job, here's a hack!
#        It is unlikely this will match your SP's characteristics, as
#        we just guess its a 4-way node, 4 processors, run interactively.
#        It is here mainly to illustrate the sort of MP_XXX's you need.
   else
      set SMP_SIZE=4
      set NCPUS=4
      set NNODES=1
      echo `hostname`  > $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE

      echo "Variables controlling Parallel Environment process kickoff are"
      set echo
      setenv MP_NODES          $NNODES
      setenv MP_PROCS          $NCPUS
      setenv MP_HOSTFILE       $HOSTFILE
      setenv MP_CPU_USE        unique
      setenv MP_ADAPTER_USE    dedicated
#       GAMESS is implemented using both MPI and LAPI active messages.
      setenv MP_MSG_API        MPI,LAPI
      setenv MP_EUILIB         us
#       SP systems with one switch adapter might use css0, not striping csss
      setenv MP_EUIDEVICE      csss
      setenv MP_RESD           no
      unset echo
   endif

#       and now we are ready to execute, using poe to kick off the tasks.
   @ NNODES = ($NCPUS - 1) / $SMP_SIZE + 1
   echo "Running $NCPUS processes on $NNODES nodes ($SMP_SIZE-way SMP)."
   set echo
   /usr/bin/poe gamess.$VERNO.x $JOB -stdinmode none
   unset echo
endif

#   SGI Origin (a very old machine) running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
#   NOTE!  This does not mean SGI Altix or ICE!!!
#   We've heard that setting the environment variable
#       SMA_SYMMETRIC_SIZE to 2147483648 (2 GB)
#   may be helpful if you see DDI_SHPALLOC error messages.
#
if ($TARGET == sgi64) then
   set EXEPATH=/home/hbar4/people/schmidt/gamess
   chdir $SCR
   set echo
   mpirun -np $NCPUS $EXEPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif


#   Cray X1 running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
#   this is not mentioned above, as it hasn't been verified for ages and ages.
if ($TARGET == cray-x1) then
   set EXEPATH=/u1/mike/gamess
   set OPTS="-m exclusive"
   if ($NCPUS > 16) then
      set PERNODE=16
   else
      set PERNODE=$NCPUS
   endif
   chdir $SCR
   set echo
   aprun -c core=0 -n $NCPUS -N $PERNODE $OPTS $EXEPATH/gamess.$VERNO.x $JOB
   unset echo
endif


#   NEC SX Series wants you to
#      a) set the path variable to point to the GAMESS executable
#   this is not mentioned above, as it hasn't been verified for ages and ages.
if ($TARGET == necsx) then
   set EXEPATH=/u1/mike/gamess

   chdir $SCR
   setenv F_RECLUNIT BYTE
   setenv F_ABORT YES
   setenv F_ERROPT1 252,252,2,2,1,1,2,1
   setenv F_PROGINF detail
   setenv F_SETBUF 4096
   echo Running $NCPUS compute processes and $NCPUS data server processes...
   @ NPROCS = $NCPUS + $NCPUS
   set echo
   mpirun -np $NPROCS $EXEPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif
#
#  ---- the bottom third of the script is to clean up all disk files ----
#  It is quite useful to display to users how big the disk files got to be.
#
echo ----- accounting info -----
#
#   in the case of GDDI runs, we save the first group master's files only.
if ($GDDIjob == true) then
   cat $OUTPUT
   cp $SCR/$JOB.F06.* ~/scr
                        cp $SCR/$JOB.F07 ~/scr/$JOB.dat
   if(-e $SCR/$JOB.F04) cp $SCR/$JOB.F04 ~/scr/$JOB.trj
   if(-e $SCR/$JOB.F35) cp $SCR/$JOB.F35 ~/scr/$JOB.rst
endif

#
#   Clean up the master's scratch directory.
#
echo Files used on the master node $master were:
ls -lF $SCR/$JOB.*
rm -f  $SCR/$JOB.F*
#
#   Clean up scratch directory of remote nodes.
#
#   This may not be necessary, e.g. on a T3E where all files are in the
#   same directory, and just got cleaned out by the previous 'rm'.  Many
#   batch queue managers provide cleaning out of scratch directories.
#   It still may be interesting to the user to see the sizes of files.
#
#   The 'lasthost' business prevents multiple cleanup tries on SMP nodes.
#
if ($TARGET == sockets) then
   set nmax=${#HOSTLIST}
   set lasthost=$HOSTLIST[1]
   @ n=2   # master already cleaned above
   while ($n <= $nmax)
      set host=$HOSTLIST[$n]
      set host=`echo $host | cut -f 1 -d :`   # drop anything behind a colon
      if ($host != $lasthost) then
         echo Files from $host are:
         $DDI_RSH $host -l $USER -n "ls -l $SCR/$JOB.*"
         $DDI_RSH $host -l $USER -n "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
endif
#
#    This particular example is for the combination iMPI, w/SGE or PBS.
#    We have inherited a file of unique node names from above.
#    There is an option to rescue the output files from group DDI runs,
#    such as FMO, in case you need to see the other group's outputs.
if ($TARGET == mpi) then
   set nnodes=`wc -l $HOSTFILE`
   set nnodes=$nnodes[1]
   @ n=1
   set master=`hostname`
           # burn off the .local suffix in our cluster's hostname
   set master=$master:r
   while ($n <= $nnodes)
      set host=`sed -n -e "$n p" $HOSTFILE`
           # in case of openMPI, unwanted stuff may follow the hostname
      set host=$host[1]
      if ($host != $master) then
         echo Files used on node $host were:
         #---------FMO rescue------
         #--if ($GDDIjob == true) then
         #--   echo "========= OUTPUT from node $host is =============="
         #--   ssh $host -l $USER "cat $SCR/$JOB.F06*"
         #--endif
         #---------FMO rescue------
         ssh $host -l $USER "ls -l $SCR/$JOB.*"
         ssh $host -l $USER "rm -f $SCR/$JOB.*"
      endif
      @ n++
   end
#          clean off the last file on the master's scratch disk.
   rm -f $HOSTFILE
   #
   if ($?I_MPI_STATS) then
      if ($I_MPI_STATS > 0) mv $SCR/stats.txt ~/$JOB.$NCPUS.stats
   endif
endif
#
#   IBM SP cleanup code...might need to be something other than 'rsh'.
#
if ($TARGET == ibm64-sp) then
   set lasthost=$master
   @ n=2   # we already cleaned up the master node just above.
   @ nmax=$NCPUS
   while ($n <= $nmax)
      set host=`sed -n -e "$n p" $HOSTFILE`
      if ($host != $lasthost) then
         echo Files used on node $host were:
         rsh $host "ls -l $SCR/$JOB.*"
         rsh $host "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
   rm -f $HOSTFILE
endif
#
#  and this is the end
#
date
time
exit
