#PBS -S /bin/bash 
#PBS -N genesis_capsule_cart
#PBS -q gpu_normal
#PBS -W group_list=<your project group ID>
#PBS -l select=4:ncpus=72:mpiprocs=1:model=gh200:ngpus=1:mem=460gb
#PBS -l walltime=8:00:00
#PBS -j oe
#PBS -r n

#BENCH reference genesis_capsule_cart.data.iter
#BENCH tolerance_abs 1.0e-2
#BENCH tolerance_rel 1.0e-2
#BENCH compare ctu_avg 20
#BENCH exclude_vars maxcfl, maxmach, minpressure, maxpressure, minuvel, maxuvel, minvvel, maxvvel, minwvel, maxwvel, mintemperature, maxtemperature, threshold_*, nt, time, ctu, ncells_*, blocks_*, mx_*, my_*, mz_*, cmx_*, cmy_*, cmz_*, fy_*, fs_*, cy_*, cs_*, fl_*, fz_*, cl_*, cz_*, *_visc_*
#BENCH save_vars cl_aeroshell, cd_aeroshell, cmy_aeroshell

log_file="run.log"

# Load LAVA module
module purge
module use /swbuild/LAVA/LAVA_modules
module load LAVA


# Set environment
export MPI_USE_CUDA=1
export OMP_NUM_THREADS=72
export CUDA_VISIBLE_DEVICES=0
export OMP_PLACES=cores
export OMP_PROC_BIND=true
export LAVA_OUTPUT_INTERVAL=1000000
 
# Change to working directory
cd $PBS_O_WORKDIR


mpiexec -N 4 --bind-to none --report-bindings mbind.x -t 32 $exec run.inputs 2>&1 |& tee -a ${log_file}


# Run LAVA with NUMA binding (1 MPI rank per node, 1 GPU per node)
mpiexec -N 1 --report-bindings --hostfile $PBS_NODEFILE \
    -x PATH -x LD_LIBRARY_PATH -x CUDA_VISIBLE_DEVICES \
    numactl --cpunodebind 0 --membind 0 \
    mbind.x -t $OMP_NUM_THREADS \
    lava.3D.NS1.NVIDIA.ARM.HPCX run.inputs 2>&1 |& tee -a ${log_file}