#!/bin/bash
#PBS -S /bin/bash
#PBS -N crm_highlift_unst
#PBS -q gpu_normal
#PBS -W group_list=<your project group ID>
#PBS -l select=60:ncpus=72:mpiprocs=1:model=gh200:ngpus=1:mem=460gb
#PBS -l walltime=24:00:00
#PBS -j oe
#PBS -k oed
#PBS -r n

#BENCH reference crm_highlift_unst.data.iter
#BENCH tolerance_abs 1.0e-6
#BENCH tolerance_rel 1.0e-6

# Load LAVA module
module purge
module use /swbuild/LAVA/LAVA_modules
module load LAVA

# Set environment
export OMP_NUM_THREADS=48
export CUDA_VISIBLE_DEVICES=0

# Change to working directory
cd $PBS_O_WORKDIR

# Run LAVA with NUMA binding (1 MPI rank per node, 1 GPU per node)
log_file="run.log"
mpiexec -N 1 --report-bindings --hostfile $PBS_NODEFILE \
    -x PATH -x LD_LIBRARY_PATH -x CUDA_VISIBLE_DEVICES \
    numactl --cpunodebind 0 --membind 0 \
    mbind.x -t $OMP_NUM_THREADS \
    lava.3D.NS1.NVIDIA.ARM.HPCX unst_hlpw5_case2p4.inputs 2>&1 |& tee -a ${log_file}