diff --git a/config/queue.csv b/config/queue.csv index 30d8c2d..1e7573e 100644 --- a/config/queue.csv +++ b/config/queue.csv @@ -1,5 +1,5 @@ queue,submit_cmd,template -SLURM_RIKYU,sbatch,"-p ${queue_group} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads} --gpus-per-node=${numproc_node}" +SLURM_RIKYU,sbatch,"-p ${queue_group} --account=${account} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads} --gpus=${proc}" FJ,pjsub,"-L rscunit=rscunit_ft01,rscgrp=${queue_group},elapse=${elapse},node=${nodes} --mpi max-proc-per-node=${numproc_node} -x PJM_LLIO_GFSCACHE=/vol0002:/vol0003:/vol0004:/vol0005" PJM_GENKAI,pjsub,"-L rscgrp=${queue_group},elapse=${elapse},node=${nodes} --mpi proc=${proc}" SLURM_RC,sbatch,"-p ${queue_group} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads}" diff --git a/config/system.csv b/config/system.csv index 9d2f860..e602904 100644 --- a/config/system.csv +++ b/config/system.csv @@ -1,9 +1,9 @@ -system,mode,tag_build,tag_run,queue,queue_group -RIKYU,cross,ai4ss_login,ai4ss_jacamar,SLURM_RIKYU,1n1gpu +system,mode,tag_build,tag_run,queue,queue_group,account +RIKYU,cross,ai4ss_login,ai4ss_jacamar,SLURM_RIKYU,gpu,rkp00012 Fugaku,cross,fugaku_login1,fugaku_jacamar,FJ,small FugakuCN,native,,fugaku_jacamar,FJ,small RC_GH200,native,,cloud_jacamar,SLURM_RC,qc-gh200 -RC_DGXSP,native,,cloud_jacamar,SLURM_RC,ng-dgx-3h +RC_DGXSP,native,,cloud_jacamar,SLURM_RC,ng-dgx-m1 RC_GENOA,native,,cloud_jacamar,SLURM_RC,genoa RC_FX700,native,,cloud_jacamar,SLURM_RC,fx700 MiyabiG,cross,miyabi_g_login,miyabi_g_jacamar,PBS_Miyabi,debug-g diff --git a/docs/cx/BENCHKIT_SPEC.md b/docs/cx/BENCHKIT_SPEC.md index e9b8dde..c62e56c 100644 --- a/docs/cx/BENCHKIT_SPEC.md +++ b/docs/cx/BENCHKIT_SPEC.md @@ -251,7 +251,7 @@ BenchKit の system / queue 定義層である。 - `queue.csv` に scheduler template を持つ - app 側の `list.csv` から system 固有事情を切り離す -`system.csv` は system 名、mode、runner tag、queue、queue_group などの正本である。 +`system.csv` は system 名、mode、runner tag、queue、queue_group、scheduler account などの正本である。 This is the system and queue definition layer. @@ -261,7 +261,8 @@ Main roles: - `queue.csv` holds scheduler templates - system-specific concerns are separated from app-side `list.csv` -`system.csv` is the source of truth for items such as system name, mode, runner tag, queue, and queue_group. +`system.csv` is the source of truth for items such as system name, mode, +runner tag, queue, queue_group, and scheduler account. ### 6.4 ポータル層 / Portal Layer diff --git a/docs/guides/add-site.md b/docs/guides/add-site.md index 943f409..96baec5 100644 --- a/docs/guides/add-site.md +++ b/docs/guides/add-site.md @@ -547,12 +547,12 @@ command_delay = "30s" ### `config/system.csv` にシステムを追加 ```csv -system,mode,tag_build,tag_run,queue,queue_group +system,mode,tag_build,tag_run,queue,queue_group,account # 既存エントリ... # cross モード(ビルドと実行が別ノード) -NewSystem,cross,newsystem_login,newsystem_jacamar,PBS_NewSystem,default +NewSystem,cross,newsystem_login,newsystem_jacamar,PBS_NewSystem,default, # native モード(同一ノードでビルドと実行) -NewSystemLN,native,,newsystem_login,none,default +NewSystemLN,native,,newsystem_login,none,default, ``` - `system`: システム名(アプリの `list.csv` から参照される) @@ -561,6 +561,7 @@ NewSystemLN,native,,newsystem_login,none,default - `tag_run`: 実行用GitLab Runnerタグ(`native`の場合はbuild_runジョブ用) - `queue`: `config/queue.csv` のキュー名(ログインノードは `none`) - `queue_group`: キューグループ名 +- `account`: scheduler account/project。不要な拠点では空欄 ### `config/queue.csv` にキューシステムを追加(必要な場合) @@ -571,7 +572,7 @@ queue,submit_cmd,template PBS_NewSystem,qsub,"-q ${queue_group} -l select=${nodes} -l walltime=${elapse} -W group_list=your_group" ``` -テンプレート内で使える変数:`${queue_group}`, `${nodes}`, `${numproc_node}`, `${nthreads}`, `${elapse}`, `${proc}`(`nodes * numproc_node`), `${cpu_per_node}`, `${gpu_per_node}`, `${cpu_sockets}`(`nodes * cpu_per_node`), `${gpu_cards}`(`nodes * gpu_per_node`) +テンプレート内で使える変数:`${queue_group}`, `${account}`, `${nodes}`, `${numproc_node}`, `${nthreads}`, `${elapse}`, `${proc}`(`nodes * numproc_node`), `${cpu_per_node}`, `${gpu_per_node}`, `${cpu_sockets}`(`nodes * cpu_per_node`), `${gpu_cards}`(`nodes * gpu_per_node`) `${cpu_per_node}` と `${gpu_per_node}` は `config/system_info.csv` から取得します。CPU socket 数や GPU card 数を scheduler に明示するサイトでは、`system_info.csv` の値も投入条件に使われます。 diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 793cf0a..8f529d8 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -167,6 +167,25 @@ case "${system}" in # -DUSE_FJMPI=OFF # ) # ;; + RIKYU) + module purge + module load nvhpc/26.3 + cmake_args=( + "${common_cmake_args[@]}" + -DCMAKE_Fortran_COMPILER=mpif90 + -DCMAKE_C_COMPILER=mpicc + -DOPENMP_FLAGS=-Mnoopenmp + -DUSE_OPENACC=ON + -DUSE_CUDA=ON + -DUSE_MPI_DEFAULT=ON + -DCMAKE_SYSTEM_PROCESSOR=openacc + -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" + -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" + -DCMAKE_CUDA_ARCHITECTURES=100 + -DCMAKE_CUDA_FLAGS=-arch=sm_100 + -DFORTRAN_COMPILER_HAS_MPI_VERSION3=OFF + ) + ;; *) echo "Unknown system: ${system}" >&2 exit 1 diff --git a/programs/salmon/list.csv b/programs/salmon/list.csv index 386804d..ed4e159 100644 --- a/programs/salmon/list.csv +++ b/programs/salmon/list.csv @@ -4,3 +4,7 @@ RC_GH200,yes,1,1,1,0:10:00 RC_DGXSP,yes,1,1,1,1:00:00 RC_GENOA,yes,1,1,48,2:00:00 RC_FX700,no,1,1,48,0:30:00 +RIKYU,yes,1,4,1,0:30:00 +RIKYU,yes,2,4,1,0:30:00 +RIKYU,yes,3,4,1,0:30:00 +RIKYU,yes,4,4,1,0:30:00 diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 1689c38..b482be0 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -13,6 +13,7 @@ RESULTS_DIR="${PWD}/results" WORK_DIR="${PWD}/salmon_run" INPUT_ARCHIVE_DEFAULT="/vol0003/rccs-sdt/data/a01010/benchmark_data/SALMON.tar.gz" INPUT_ARCHIVE_CLOUD="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON/SALMON.tar.gz" +INPUT_ARCHIVE_RIKYU="/data1/rkp00012/CX_input/SALMON/SALMON.tar.gz" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" mkdir -p "${RESULTS_DIR}" @@ -29,6 +30,11 @@ case "${system}" in exec_gs=(-stdin Si-1-1-1.nml ./salmon) exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) ;; + RIKYU) + input_archive="${BK_SALMON_INPUT_ARCHIVE:-${INPUT_ARCHIVE_RIKYU}}" + exec_gs=(./salmon) + exec_rt=(./salmon) + ;; RC_GH200|RC_DGXSP|RC_GENOA) input_archive="${INPUT_ARCHIVE_CLOUD}" exec_gs=(./salmon) @@ -47,7 +53,7 @@ fi uses_stdin_input() { case "$1" in - RC_GH200|RC_DGXSP|RC_GENOA) + RIKYU|RC_GH200|RC_DGXSP|RC_GENOA) return 0 ;; *) @@ -108,6 +114,35 @@ case "${system}" in fi export OMP_NUM_THREADS="${nthreads}" ;; + RIKYU) + module purge + module load nvhpc/26.3 + export OMP_NUM_THREADS="${nthreads}" + for nml in Si-1-1-1.nml Si-1-1-1-tddft.nml; do + awk -v nproc_ob="${n_ranks}" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' "$nml" > "$nml.tmp" && mv "$nml.tmp" "$nml" + done + if [[ "${n_ranks}" -gt 1 ]]; then + cat > wrapper.sh <<'WRAPPER' +#!/bin/bash +NCUDA_GPUS=${NCUDA_GPUS:-$(nvidia-smi -L | wc -l)} +if [ "$OMPI_COMM_WORLD_LOCAL_SIZE" -gt "$NCUDA_GPUS" ]; then + if [ "$OMPI_COMM_WORLD_LOCAL_RANK" -eq 0 ]; then + nvidia-cuda-mps-control -d + fi + sleep 10 +fi +export CUDA_VISIBLE_DEVICES=$((${OMPI_COMM_WORLD_LOCAL_RANK} % ${NCUDA_GPUS})) +exec "$@" +WRAPPER + chmod +x wrapper.sh + export UCX_IB_GPU_DIRECT_RDMA=no + fi + ;; # RC_FX700) # FX700 currently fails during GS initialization even with the Fujitsu # topology guard patch applied. Keep this route disabled until verified. @@ -122,6 +157,13 @@ run_salmon() { local logfile="$1" shift case "${system}" in + RIKYU) + if [[ "${n_ranks}" -gt 1 ]]; then + mpirun -n "${n_ranks}" ./wrapper.sh "$@" > "${logfile}" 2>&1 + else + mpirun -n "${n_ranks}" "$@" > "${logfile}" 2>&1 + fi + ;; RC_GENOA) mpirun -n "${n_ranks}" --bind-to core --map-by "ppr:${numproc_node}:node:PE=${nthreads}" "$@" > "${logfile}" 2>&1 ;; diff --git a/scripts/job_functions.sh b/scripts/job_functions.sh index 2aeffc6..fe91fec 100644 --- a/scripts/job_functions.sh +++ b/scripts/job_functions.sh @@ -64,6 +64,14 @@ get_system_queue_group() { return 0 } +# System_CSVからscheduler accountを取得する +# Account-less systems return an empty string. +get_system_account() { + local system="$1" + awk -F, -v s="$system" '$1==s {print $7}' "$SYSTEM_FILE" + return 0 +} + # Queue templates can request aggregate CPU sockets or GPU cards. Pull the # per-node values from system_info.csv instead of duplicating them in queue.csv. get_system_cpu_per_node() { diff --git a/scripts/matrix_generate.sh b/scripts/matrix_generate.sh index 25d69e3..10383e5 100644 --- a/scripts/matrix_generate.sh +++ b/scripts/matrix_generate.sh @@ -69,6 +69,7 @@ for listfile in programs/*/list.csv; do mode=$(get_system_mode "$system") queue_group=$(get_system_queue_group "$system") + account=$(get_system_account "$system") # Skip if mode or queue_group is empty (system not found in System_CSV) if [[ -z "$mode" || -z "$queue_group" ]]; then @@ -88,7 +89,7 @@ for listfile in programs/*/list.csv; do [[ "$gpu_per_node" =~ ^[0-9]+$ ]] || gpu_per_node=0 cpu_sockets=$((nodes * cpu_per_node)) gpu_cards=$((nodes * gpu_per_node)) - export elapse nodes queue_group numproc_node nthreads proc cpu_per_node gpu_per_node cpu_sockets gpu_cards + export elapse nodes queue_group account numproc_node nthreads proc cpu_per_node gpu_per_node cpu_sockets gpu_cards read -r submit_cmd template <<< "$(get_queue_template "$system")" if [[ -z "$submit_cmd" || -z "$template" ]]; then diff --git a/scripts/test_submit.sh b/scripts/test_submit.sh index c8835e5..577e65c 100644 --- a/scripts/test_submit.sh +++ b/scripts/test_submit.sh @@ -85,13 +85,14 @@ fi # --- System_CSV から mode/queue_group を取得 --- mode=$(get_system_mode "$system") queue_group=$(get_system_queue_group "$system") +account=$(get_system_account "$system") # --- 選択された設定を表示 --- echo "Selected configuration from $list_file (line $list_csv_line_num):" echo " $line" echo "" echo "Parsed values:" -echo " system=$system, enable=$enable, mode=$mode (from system.csv), queue_group=$queue_group (from system.csv)" +echo " system=$system, enable=$enable, mode=$mode (from system.csv), queue_group=$queue_group (from system.csv), account=${account:-}" echo " nodes=$nodes, numproc_node=$numproc_node, nthreads=$nthreads, elapse=$elapse" # --- 投入用スクリプト作成 --- @@ -221,9 +222,14 @@ case "$system" in -l elapstim_req=${elapse} -v OMP_NUM_THREADS=${nthreads} script.sh ;; RIKYU) - echo sbatch -p $queue_group -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus-per-node=${numproc_node} \ + if [[ -z "$account" ]]; then + echo "Error: RIKYU requires an account in config/system.csv" >&2 + exit 1 + fi + proc=$((nodes * numproc_node)) + echo sbatch -p $queue_group --account=$account -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus=$proc \ --wrap="bash programs/$code/run.sh $system $nodes $numproc_node $nthreads" - sbatch -p $queue_group -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus-per-node=${numproc_node} \ + sbatch -p $queue_group --account=$account -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus=$proc \ --wrap="bash programs/${code}/run.sh $system $nodes $numproc_node $nthreads" ;; RC_GH200|RC_DGXSP|RC_GENOA|RC_FX700)