#!/bin/csh -f
# @ job_type = parallel
# @ class = large
# @ input = /dev/null
# @ output = $(Executable).$(Cluster).$(Process).out
# @ error = $(Executable).$(Cluster).$(Process).err
# @ notify_user = pblaha@susi.theochem.tuwien.ac.at
# @ notification = always
# @ node_usage = not_shared
# @ initialdir = ~/lapw/nanomesh/pt_nanomesh_9x10
# @ tasks_per_node = 2
# @ node = 32
# @ queue

set mpijob=16
# -----------------------------------------------
# The example above requests 32 nodes, each one having two processors (i.e. 64 cores)
# setting mpijob=16 means that one selects 16 cores for mpi-parallelelization of a single k-point
# and obviously 4 k-points are run in parallel
#------------------------------------------------
#
#------------------------------------------------
limit coredumpsize 0
setenv SCRATCH /scratch
setenv OMP_NUM_THREADS 1 
# optional settings to reduce NFS traffic (*.vector files on local /scratch)
# note: make sure k-points and number of processors match! 
# vector files are lost after calculations (unless explicitly copied)
# -----------------------------------------------
#
# the "loadleveler" returns the names of the selected nodes in $LOADL_PROCESSOR_LIST
# for other queuing systems you have to find out how to get this info      
# for PBS: $PBS_NODEFILE 
# for SGE: set proclist=`cat $TMPDIR/machines`
#

echo -n 'running in '
pwd
echo nodes for this job: $LOADL_PROCESSOR_LIST 

set proclist=`echo $LOADL_PROCESSOR_LIST`
set nproc=$#proclist
echo number of processors: $nproc

#---------- writing .machines file ------------------
echo '#' > .machines
# example for an MPI parallel lapw0 
echo -n 'lapw0:' >> .machines
set i=1
while ($i <= $nproc )
  echo -n "$proclist[$i] " >>.machines
  @ i = $i + 1
  # remove comment when you want to use only every second core for lapw0mpi (memory-limitations)
  ####@ i = $i + 1
end
echo ' ' >>.machines

#example for k-point and mpi parallel lapw1/2
set i=1
while ($i <= $nproc )
  echo -n '1:' >>.machines
  @ i1 = $i + $mpijob
  @ i2 = $i1 - 1
  echo $proclist[$i-$i2] >>.machines
  set i=$i1
end
echo 'granularity:1' >>.machines
echo 'extrafine:1' >>.machines
echo 'lapw2_vector_split:1 ' >>.machines
# eventually set lapw2_vector_split:2 for memory-limitations
# --------- end of .machines file

# --------- writing  .processes and .machine1 file (only needed in case you start with lapw2)
echo  -n 'init:' > .processes
echo $proclist >> .processes
echo '1 : ' $proclist[1] " :  1 : $mpijob : 1" >> .processes

rm .machine1
set i=1
while ($i <= $nproc )
  echo $proclist[$i] >>.machine1
  @ i = $i + 1
end
# -------------  end of setup --------------
# now select some WIEN2k commands


min -NI -ex -j "run_lapw -p -I -i 99 -fc 3 -r 60 -it "

#run_lapw -p -NI -i 99 -fc 3 -r 60 -it0