From 234252e03daf972c7dd12dde48196cbea4fe2256 Mon Sep 17 00:00:00 2001 From: Anav Prasad Date: Tue, 14 Jul 2026 13:45:17 +0000 Subject: [PATCH] label virtual devices in description; add GPUx2 server CI jobs --- .github/workflows/server-self-hosted.yml | 18 ++++++++++++++++ ggml/src/ggml-cuda/common.cuh | 1 + ggml/src/ggml-cuda/ggml-cuda.cu | 26 ++++++++++++++++-------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml index 2dcd6d7425..58af7e1551 100644 --- a/.github/workflows/server-self-hosted.yml +++ b/.github/workflows/server-self-hosted.yml @@ -141,6 +141,24 @@ jobs: export LLAMA_ARG_BACKEND_SAMPLING=1 pytest -v -x -m "not slow" + - name: Tests (GPUx2) + id: server_integration_tests_gpu2 + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export GGML_CUDA_DEVICES=2 + pytest -v -x -m "not slow" + + - name: Tests (GPUx2, backend-sampling) + id: server_integration_tests_gpu2_backend_sampling + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1 + pytest -v -x -m "not slow" + server-kleidiai: runs-on: ah-ubuntu_22_04-c8g_8x diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 3f9a4f496f..7a6b47e53c 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1131,6 +1131,7 @@ struct ggml_cuda_device_info { bool supports_cooperative_launch; // whether cooperative launch is supported int physical_device; // backing physical CUDA device for this (virtual) device int physical_share_count; // number of (virtual) devices sharing this device's physical GPU + int virtual_index; // index of this (virtual) device among those sharing its physical GPU }; cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {}; diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 853abacc36..1b5d42e056 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -242,13 +242,13 @@ static ggml_cuda_device_info ggml_cuda_init() { info.device_count = GGML_CUDA_MAX_DEVICES; } - // map each (virtual) device to a backing physical device (round-robin) and - // store virtual device count per physical device + // map each (virtual) device to a backing physical device (round-robin), assign each its index + // among the (virtual) devices sharing that physical GPU, and store the per-physical share count int physical_share_count[GGML_CUDA_MAX_DEVICES] = {}; GGML_ASSERT(info.device_count == 0 || info.physical_device_count > 0); for (int id = 0; id < info.device_count; ++id) { info.devices[id].physical_device = id % info.physical_device_count; - physical_share_count[info.devices[id].physical_device]++; + info.devices[id].virtual_index = physical_share_count[info.devices[id].physical_device]++; } int64_t total_vram = 0; @@ -4421,10 +4421,21 @@ int ggml_backend_cuda_get_device_count() { return ggml_cuda_info().device_count; } -void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { +static std::string ggml_cuda_device_description(int device) { cudaDeviceProp prop; CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(device))); - snprintf(description, description_size, "%s", prop.name); + + const ggml_cuda_device_info & info = ggml_cuda_info(); + std::string description = prop.name; + if (info.device_count > info.physical_device_count) { + description += " (physical device " + std::to_string(info.devices[device].physical_device) + + ", virtual device " + std::to_string(info.devices[device].virtual_index) + ")"; + } + return description; +} + +void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { + snprintf(description, description_size, "%s", ggml_cuda_device_description(device).c_str()); } static int ggml_cuda_physical_device_share_count(int device) { @@ -5269,10 +5280,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { ggml_backend_cuda_device_context * dev_ctx = new ggml_backend_cuda_device_context; dev_ctx->device = i; dev_ctx->name = GGML_CUDA_NAME + std::to_string(i); - - cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, physical_id)); - dev_ctx->description = prop.name; + dev_ctx->description = ggml_cuda_device_description(i); char pci_bus_id[32] = {}; CUDA_CHECK(cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), physical_id));