#PBS -S /bin/bash 
#PBS -N crm_highlift_float_cart
#PBS -q gpu_normal
#PBS -W group_list=<your project group ID>
#PBS -l select=32:ncpus=128:ngpus=4:model=mil_a100:mem=500GB,place=scatter:excl
#PBS -l walltime=10:00:00
#PBS -j oe
#PBS -r n

#BENCH reference crm_highlift_float_cart.data.iter
#BENCH tolerance_abs 1.0e-2
#BENCH tolerance_rel 1.0e-2
#BENCH compare ctu_avg 4
#BENCH exclude_vars maxcfl, maxmach, minpressure, maxpressure, minuvel, maxuvel, minvvel, maxvvel, minwvel, maxwvel, mintemperature, maxtemperature, threshold_*, loadmin, loadmax, loadbalance, nt, time, ctu, mx_*, my_*, mz_*, cmx_*, cmy_*, cmz_*, fy_visc_wing_fuse, fs_visc_wing_fuse
#BENCH save_vars cl_airplane, cd_airplane, cmy_airplane


log_file="run.log"

# Load LAVA module
module purge
module use /swbuild/LAVA/LAVA_modules
module load LAVA

export LAVA_OUTPUT_INTERVAL=1000000

# Set environment
export MPI_USE_CUDA=1
export CUDA_VISIBLE_DEVICES=0,1,2,3
export OMP_PLACES=cores
export OMP_PROC_BIND=true

# Change to working directory 
cd $PBS_O_WORKDIR


mpiexec -N 4 --bind-to none --report-bindings mbind.x -t 32 lava.3D.NS1.NVIDIA.OPENMPI.FLOAT run.inputs 2>&1 |& tee -a ${log_file}