MiniWeather (cpeCray + cudatoolkit)

Download source code

git clone https://github.com/mrnorman/miniWeather.git
cp -r miniWeather/c collision
cd collision

Load modules

module purge
module load cpeCray/23.03
module load cray-parallel-netcdf/1.12.3.3
module load cudatoolkit/23.3_11.8
module load craype-accel-nvidia80

Configure

export CXX="CC -craype-verbose"
export LDFLAGS="-lcudart"
cmake . -DOPENMP_FLAGS="-fopenmp" -DOPENMP45_FLAGS="-fopenmp" \
        -DNX=2000 -DNZ=1000 -DSIM_TIME=1000 -DOUT_FREQ=50 -DDATA_SPEC=DATA_SPEC_COLLISION

Make

make

Job script

CPU partition

#!/bin/bash
#SBATCH -p compute             # Partition
#SBATCH -N 1                   # Number of nodes
#SBATCH --ntasks-per-node=16   # Number of MPI processes per node
#SBATCH --cpus-per-task=8      # Number of OpenMP threads per MPI process
#SBATCH -t 5-00:00:00          # Job runtime limit
#SBATCH -A ltXXXXXX            # Billing account  *** USER EDIT ***
#SBATCH -J MiniWeather         # Job name

module purge
module load cpeCray/23.03
module load cray-parallel-netcdf/1.12.3.3
module load cudatoolkit/23.3_11.8
module load craype-accel-nvidia80

export PATH=/--path-to-collision-dir--:${PATH}     # *** USER EDIT ***
export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}  

srun --cpus-per-task=${SLURM_CPUS_PER_TASK} openmp

GPU partition

#!/bin/bash
#SBATCH -p gpu                 # Partition
#SBATCH -N 1                   # Number of nodes
#SBATCH --gpus-per-node=1      # Number of GPU card per node
#SBATCH --ntasks-per-node=1    # Number of MPI processes per node
#SBATCH --cpus-per-task=16     # Number of OpenMP threads per MPI process
#SBATCH -t 5-00:00:00          # Job runtime limit
#SBATCH -A ltXXXXXX            # Billing account  *** USER EDIT ***
#SBATCH -J MiniWeather         # Job name

module purge
module load cpeCray/23.03
module load cray-parallel-netcdf/1.12.3.3
module load cudatoolkit/23.3_11.8
module load craype-accel-nvidia80

export PATH=/--path-to-collision-dir--:${PATH}     # *** USER EDIT ***
export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}  

srun --cpus-per-task=${SLURM_CPUS_PER_TASK} --gpus-per-task=1 openmp45

(Optional) Post-processing

module purge
module load Miniconda3/22.11.1-1

conda activate netcdf-py39
ncview ./output.nc
conda deactivate

Note: Must login with X11 enabled, that is, echo ${DISPLAY} should not be empty.