From f814206045da7f5676a9d5f4fc9f03e978e9c932 Mon Sep 17 00:00:00 2001 From: AtlantaPepsi Date: Wed, 2 Sep 2026 22:49:19 -0500 Subject: [PATCH 1/3] removing dependency on ppa -> gpg-agent, which is absent on this img --- .github/workflows/codeql.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 313f0dee..fdcd5276 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -30,11 +30,8 @@ jobs: - name: Install OS requirements timeout-minutes: 10 run: | - apt update - apt install -y software-properties-common - apt-add-repository -y ppa:git-core/ppa apt-get update - apt install -y git + DEBIAN_FRONTEND=noninteractive apt-get install -y git - name: Checkout repository uses: actions/checkout@v4 From 7eb0bd91a328649e3994715a1c57c6c5be929af5 Mon Sep 17 00:00:00 2001 From: AtlantaPepsi Date: Tue, 15 Sep 2026 12:34:14 -0500 Subject: [PATCH 2/3] disable some multinode smoketests; modify subExec policy for rings preset --- docs/reference/presets.rst | 6 ++--- src/client/Presets/Rings.hpp | 40 ++++++++++++++++++++++++-------- src/client/Presets/SmokeTest.hpp | 20 ++++++++++++++-- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/docs/reference/presets.rst b/docs/reference/presets.rst index 2f05764a..285371d8 100644 --- a/docs/reference/presets.rst +++ b/docs/reference/presets.rst @@ -518,8 +518,8 @@ To modify the behavior of the rings preset, use the following environment variab - ``0`` * - ``NUM_SUB_EXEC`` - - SubExecutors (CUs for GFX, batch items for DMA) per transfer. - - ``8`` + - SubExecutors (CUs for GFX, batch items for DMA) per transfer. ``0`` uses all available subexecutors on the Executor. + - ``0`` * - ``RING_SIZE`` - Number of GPUs per ring. Must evenly divide ``numRanks x NUM_GPU_DEVICES``. Default is one ring covering the whole pod. @@ -550,7 +550,7 @@ Example output MEM_TYPE = 0 : Using default GPU memory (0=default, 1=fine-grained, 2=uncached, 3=managed) NUM_GPU_DEVICES = 4 : Using 4 GPUs NUM_QUEUE_PAIRS = 0 : Using 0 queue pairs for NIC transfers - NUM_SUB_EXEC = 8 : Using 8 subExecutors/CUs per Transfer + NUM_SUB_EXEC = 0 : Using all available subexecutors/CUs per Transfer USE_DMA_EXEC = 0 : Using GFX executor USE_REMOTE_READ = 0 : Using SRC as executor STRIDE = 1 : Reordering devices by taking 1 steps diff --git a/src/client/Presets/Rings.hpp b/src/client/Presets/Rings.hpp index 9226370f..63032137 100644 --- a/src/client/Presets/Rings.hpp +++ b/src/client/Presets/Rings.hpp @@ -47,7 +47,7 @@ int RingsPreset(EnvVars& ev, int memTypeIdx = EnvVars::GetEnvVar("MEM_TYPE" , 0); int numGpus = EnvVars::GetEnvVar("NUM_GPU_DEVICES", numDetectedGpus); int numQueuePairs = EnvVars::GetEnvVar("NUM_QUEUE_PAIRS", 0); - int numSubExecs = EnvVars::GetEnvVar("NUM_SUB_EXEC" , 8); + int numSubExecs = EnvVars::GetEnvVar("NUM_SUB_EXEC" , 0); int showDetails = EnvVars::GetEnvVar("SHOW_DETAILS" , 0); int useDmaExec = EnvVars::GetEnvVar("USE_DMA_EXEC" , 0); int useTdmExec = EnvVars::GetEnvVar("USE_TDM_EXEC" , 0); @@ -78,6 +78,10 @@ int RingsPreset(EnvVars& ev, Utils::Print("[ERROR] Num queue pairs must be non-negative\n"); return ERR_FATAL; } + if (numSubExecs < 0) { + Utils::Print("[ERROR] NUM_SUB_EXEC must be non-negative (0 uses all available subexecutors)\n"); + return ERR_FATAL; + } int totalGpus = numRanks * numGpus; if (totalGpus % ringSize) { @@ -95,7 +99,10 @@ int RingsPreset(EnvVars& ev, ev.Print("MEM_TYPE" , memTypeIdx , "Using %s GPU memory (%s)", devMemTypeStr.c_str(), Utils::GetAllGpuMemTypeStr().c_str()); ev.Print("NUM_GPU_DEVICES", numGpus , "Using %d GPUs", numGpus); ev.Print("NUM_QUEUE_PAIRS", numQueuePairs, "Using %d queue pairs for NIC transfers", numQueuePairs); - ev.Print("NUM_SUB_EXEC" , numSubExecs , "Using %d subexecutors/CUs per Transfer", numSubExecs); + if (numSubExecs == 0) + ev.Print("NUM_SUB_EXEC" , numSubExecs, "Using all available subexecutors/CUs per Transfer"); + else + ev.Print("NUM_SUB_EXEC" , numSubExecs, "Using %d subexecutors/CUs per Transfer", numSubExecs); ev.Print("USE_DMA_EXEC" , useDmaExec , "Using %s executor", useDmaExec ? "DMA" : "GFX"); ev.Print("USE_TDM_EXEC" , useTdmExec , "Using %s executor", useTdmExec ? "TDM" : "GFX"); ev.Print("USE_REMOTE_READ", useRemoteRead, "Using %s as executor", useRemoteRead ? "DST" : "SRC"); @@ -105,16 +112,9 @@ int RingsPreset(EnvVars& ev, } } - Utils::Print("GPU-%s Rings benchmark:\n", execName); - Utils::Print("==============================\n"); - Utils::Print("[%lu bytes per Transfer] [%s:%d] [MemType:%s] [NIC QueuePairs:%d] [#Ranks:%d]\n", - numBytesPerTransfer, execName, numSubExecs, - devMemTypeStr.c_str(), numQueuePairs, numRanks); - TransferBench::ConfigOptions cfg = ev.ToConfigOptions(); int numRings = totalGpus / ringSize; - Utils::Print("Running %d parallel ring(s) each of %d devices. All numbers in GB/s:\n", numRings, ringSize); // Determine ordering of GPUs for the rings based on stride std::vector indices(totalGpus); @@ -143,7 +143,7 @@ int RingsPreset(EnvVars& ev, t.srcs = {memDevices[srcIdx]}; t.dsts = {memDevices[dstIdx]}; t.exeDevice = {exeType, memDevices[exeIdx].memIndex, memDevices[exeIdx].memRank}; - t.numSubExecs = numSubExecs; + t.numSubExecs = (numSubExecs > 0) ? numSubExecs : TransferBench::GetNumSubExecutors(t.exeDevice); transfers.push_back(t); // Build NIC transfers between these GPUs as well if requested @@ -157,6 +157,26 @@ int RingsPreset(EnvVars& ev, } } + // NUM_SUB_EXEC=0: each hop already took its own executor's count. + // Report it in the header only if every selected GPU executor agrees, otherwise "var" + std::string seStr = std::to_string(numSubExecs); + if (numSubExecs == 0) { + int seCount = 0; + for (auto const& t : transfers) { + if (t.exeDevice.exeType != exeType) continue; // skip NIC overlays + if (seCount == 0) seCount = t.numSubExecs; + else if (t.numSubExecs != seCount) { seCount = -1; break; } + } + seStr = (seCount < 0) ? "variable" : std::to_string(seCount); + } + + Utils::Print("GPU-%s Rings benchmark:\n", execName); + Utils::Print("==============================\n"); + Utils::Print("[%lu bytes per Transfer] [%s:%s] [MemType:%s] [NIC QueuePairs:%d] [#Ranks:%d]\n", + numBytesPerTransfer, execName, seStr.c_str(), + devMemTypeStr.c_str(), numQueuePairs, numRanks); + Utils::Print("Running %d parallel ring(s) each of %d devices. All numbers in GB/s:\n", numRings, ringSize); + TransferBench::TestResults results; if (!TransferBench::RunTransfers(cfg, transfers, results)) { for (auto const& err : results.errResults) diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index b6880f91..52ee69a4 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -269,9 +269,17 @@ int SmokeTestPreset(EnvVars& ev, MemType cpuMemType = Utils::GetCpuMemType(cpuMemTypeIdx); MemType gpuMemType = Utils::GetGpuMemType(gpuMemTypeIdx); std::set testsToRun(testList.begin(), testList.end()); + bool skipCollectiveByDefault = false; if (testList.empty()) { for (int testIdx = 1; testIdx <= NUM_SMOKE_TESTS; testIdx++) testsToRun.insert(testIdx); + // Multi-node default: H2D/D2H/D2D only. + // Broadcast, gather, and all-to-all remain available via TEST_LIST + if (numRanks > 1) { + skipCollectiveByDefault = true; + for (int t : {5, 6, 7, 8, 13, 14, 15, 16}) + testsToRun.erase(t); + } } vector sizeList; @@ -312,8 +320,13 @@ int SmokeTestPreset(EnvVars& ev, ev.Print("RUN_PARALLEL", runParallel, "Running GPUs %s", runParallel ? "in parallel" : "serially"); ev.Print("SIZE_LIST" , sizeStrList.size(), "Transfer sizes tested: %s", ev.GetStr(sizeStrList).c_str()); ev.Print("SE_MAX_BYTES", seMaxBytesStr, "Each SubExecutor can work on at most %lu bytes", seMaxBytes); - ev.Print("TEST_LIST" , testsToRun.size(), testList.empty() ? "Running all tests (can also filter with 'dma','gfx','fast')" - : "Running Tests: %s", ev.GetStr(testList).c_str()); + if (!testList.empty()) + ev.Print("TEST_LIST", testsToRun.size(), "Running Tests: %s", ev.GetStr(testList).c_str()); + else if (skipCollectiveByDefault) + ev.Print("TEST_LIST", testsToRun.size(), + "Running H2D/D2H/D2D only (broadcast/gather/a2a off by default on multi-node; set TEST_LIST to enable)"); + else + ev.Print("TEST_LIST", testsToRun.size(), "Running all tests (can also filter with 'dma','gfx','fast')"); ev.Print("USE_BDMA" , useBdma, "Using %s dma executor", useBdma ? "batched" : "standard"); printf("\n"); } @@ -417,6 +430,9 @@ int SmokeTestPreset(EnvVars& ev, } else { Utils::Print("All tests passed\n"); } + if (skipCollectiveByDefault) { + Utils::Print("[WARN] Broadcast / Gather / AllToAll tests are disabled by default on multi-node; set TEST_LIST to enable them\n"); + } if (numRanks > 1 && Utils::GetRankPerPodMap().size() != 1) { Utils::Print("[WARN] Copy (D2D) / Broadcast / Gather / AllToAll tests are skipped if ranks are not in same pod\n"); } From e986a3e7c7cc7f978beda7246da414f93460b04c Mon Sep 17 00:00:00 2001 From: AtlantaPepsi Date: Fri, 18 Sep 2026 16:22:00 -0500 Subject: [PATCH 3/3] TransferBench v1.70.02 --- CHANGELOG.md | 8 ++++++++ CMakeLists.txt | 2 +- src/client/EnvVars.hpp | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1267b42..22c87b02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ Documentation for TransferBench is available at [https://rocm.docs.amd.com/projects/TransferBench](https://rocm.docs.amd.com/projects/TransferBench). +## v1.70.02 +### Modified +- rings preset defaults `NUM_SUB_EXEC` to 0, which uses all available subexecutors per Transfer +- smoketest disables broadcast, gather, and all-to-all by default on multi-node. Set `TEST_LIST` to enable them + +## v1.70.01 +Skipped due to misnaming of 1.70.00 + ## v1.70.00 ### Added - Adding support for Tensor Data Mover (TDM)-based executor [T] on supported hardware (gfx1250, NVIDIA sm_90+ via TMA). diff --git a/CMakeLists.txt b/CMakeLists.txt index de1361fd..8252b5f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,7 +104,7 @@ set(ENV{ROCM_PATH} "${ROCM_PATH}") #================================================================================================== set(TRANSFERBENCH_VERSION_MAJOR 1) set(TRANSFERBENCH_VERSION_MINOR 70) -set(TRANSFERBENCH_VERSION_PATCH_FALLBACK "00") +set(TRANSFERBENCH_VERSION_PATCH_FALLBACK "02") # Auto-compute patch from git: count commits since the last v..* tag. # Falls back to TRANSFERBENCH_VERSION_PATCH_FALLBACK when git is unavailable, diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index a62201c5..575f6a67 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -44,7 +44,7 @@ THE SOFTWARE. #include #include -#define CLIENT_VERSION "01" +#define CLIENT_VERSION "02" #include "TransferBench.hpp" using namespace TransferBench;