#!/bin/bash -l
#SBATCH --job-name=example_apptainer_overlay
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=8
#SBATCH --mem-per-cpu=3G
#SBATCH --gres=gpu:1
#SBATCH --output my_apptainer-job_%j.out
#SBATCH --partition=gpus
#SBATCH --time=02:00:00

# Load modules
module load NVHPC/26.5-CUDA-13.2.0

# Change to work dir
cd $SLURM_SUBMIT_DIR

# Execute the application targeting your overlay-backed path
apptainer exec --overlay my-overlay.img rocky9.sif python3 train_llm.py --output_dir /opt/my-software/outputs --cache_dir /opt/my-software/cache