From 7d7954caa712d91abeb0114d7e7abdc609ee1132 Mon Sep 17 00:00:00 2001 From: Erik Schultheis Date: Sun, 28 Jul 2024 10:51:27 +0200 Subject: [PATCH 1/6] nvml for more detailed gpu status info --- Makefile | 2 +- llmc/mfu.h | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++ train_gpt2.cu | 6 +++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 40721ed..cba1a37 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ CUDA_OUTPUT_FILE = -o $@ # NVCC flags # -t=0 is short for --threads, 0 = number of CPUs on the machine NVCC_FLAGS = -O3 -t=0 --use_fast_math -std=c++17 -NVCC_LDFLAGS = -lcublas -lcublasLt +NVCC_LDFLAGS = -lcublas -lcublasLt -lnvidia-ml NVCC_INCLUDES = NVCC_LDLIBS = NCLL_INCUDES = diff --git a/llmc/mfu.h b/llmc/mfu.h index 7753305..5f8934f 100644 --- a/llmc/mfu.h +++ b/llmc/mfu.h @@ -4,12 +4,22 @@ #include #include #include +#include // tied to enum PrecisionMode, in a future refactor make them the same #define MFUH_PRECISION_FP32 0 #define MFUH_PRECISION_FP16 1 #define MFUH_PRECISION_BF16 2 +inline void nvml_check(nvmlReturn_t status, const char *file, int line) { + if (status != NVML_SUCCESS) { + printf("[NVML ERROR] at file %s:%d:\n%s\n", file, line, nvmlErrorString(status)); + exit(EXIT_FAILURE); + } +}; +#define nvmlCheck(err) (nvml_check(err, __FILE__, __LINE__)) + + typedef struct { float TF_32; // tensor-core performance 32 bit float BF_16_32; // bf16 with 32 bit accumulate @@ -134,4 +144,63 @@ float get_flops_promised(const char* device, int precision_mode) { return -1.0f; // ¯\_(ツ)_/¯ } +nvmlDevice_t nvml_get_device() { + static bool needs_init = true; + static nvmlDevice_t device; + if(needs_init) { + needs_init = false; + nvmlCheck(nvmlInit()); + nvmlCheck(nvmlDeviceGetHandleByIndex_v2(0, &device)); + } + return device; +} + +struct GPUUtilInfo { + unsigned int clock; + unsigned int max_clock; + unsigned int power; + unsigned int power_limit; + unsigned int fan; + unsigned long long throttle; + + float gpu_utilization; + float mem_utilization; +}; + +GPUUtilInfo get_gpu_utilization_info() { + GPUUtilInfo info; + nvmlDevice_t device = nvml_get_device(); + nvmlCheck(nvmlDeviceGetClockInfo(device, NVML_CLOCK_SM, &info.clock)); + nvmlCheck(nvmlDeviceGetMaxClockInfo(device, NVML_CLOCK_SM, &info.max_clock)); + nvmlCheck(nvmlDeviceGetPowerManagementLimit(device, &info.power_limit)); + nvmlCheck(nvmlDeviceGetPowerUsage(device, &info.power)); + nvmlCheck(nvmlDeviceGetCurrentClocksThrottleReasons(device, &info.throttle)); + nvmlCheck(nvmlDeviceGetFanSpeed(device, &info.fan)); + // other potentially interesting functions + // nvmlDeviceGetPcieThroughput + // nvmlDeviceGetPcieSpeed + // nvmlDeviceGetNumGpuCores + // nvmlDeviceGetMemoryBusWidth + nvmlSample_t buffer[64]; + nvmlValueType_t v_type; + unsigned int sample_count = 64; + nvmlCheck(nvmlDeviceGetSamples(device, NVML_GPU_UTILIZATION_SAMPLES, 0, &v_type, &sample_count, buffer)); + float gpu_utilization = 0.f; + for(unsigned i = 0; i < sample_count; ++i) { + gpu_utilization += (float)buffer[i].sampleValue.uiVal; + } + gpu_utilization /= (float)sample_count; + sample_count = 64; + nvmlCheck(nvmlDeviceGetSamples(device, NVML_MEMORY_UTILIZATION_SAMPLES, 0, &v_type, &sample_count, buffer)); + float mem_utilization = 0.f; + for(unsigned i = 0; i < sample_count; ++i) { + mem_utilization += (float)buffer[i].sampleValue.uiVal; + } + mem_utilization /= (float)sample_count; + + info.gpu_utilization = gpu_utilization; + info.mem_utilization = mem_utilization; + return info; +} + #endif // MFU_H diff --git a/train_gpt2.cu b/train_gpt2.cu index 8f11091..10cebcb 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -1829,6 +1829,12 @@ int main(int argc, char *argv[]) { printf0("step %4d/%d | loss %7.6f (%+.2fz)| norm %6.4f (%+.2fz)| lr %.2e | %.2f ms | %.1f%% bf16 MFU | %.0f tok/s\n", step + 1, train_num_batches, model.mean_loss, zloss, grad_norm, zgrad, step_learning_rate, time_elapsed_ms, 100*mfu, bias_corrected_ema_tokens_per_second); + if((step + 1) % 100 == 0) { + GPUUtilInfo gpu_info = get_gpu_utilization_info(); + printf0("GPU Status: Compute: %2.1f%% | Memory: %2.1f%% | Fan: %2d%%\n", gpu_info.gpu_utilization, gpu_info.mem_utilization, gpu_info.fan); + printf0(" Clock: %4d / %4d\n", gpu_info.clock, gpu_info.max_clock); + printf0(" Power: %4d / %4d\n", gpu_info.power, gpu_info.power_limit); + } logger_log_train(&logger, step, model.mean_loss, step_learning_rate, grad_norm); // disable the profiler after 3 steps of optimization From 743062142cf27cb2a736ad9069add1423d3ea7e5 Mon Sep 17 00:00:00 2001 From: Erik Schultheis Date: Sun, 28 Jul 2024 11:07:37 +0200 Subject: [PATCH 2/6] units + more frequent reporting --- llmc/mfu.h | 7 ++++--- train_gpt2.cu | 7 +++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llmc/mfu.h b/llmc/mfu.h index 5f8934f..2abca26 100644 --- a/llmc/mfu.h +++ b/llmc/mfu.h @@ -181,16 +181,17 @@ GPUUtilInfo get_gpu_utilization_info() { // nvmlDeviceGetPcieSpeed // nvmlDeviceGetNumGpuCores // nvmlDeviceGetMemoryBusWidth - nvmlSample_t buffer[64]; + constexpr const int BUFFER_LIMIT = 128; + nvmlSample_t buffer[BUFFER_LIMIT]; nvmlValueType_t v_type; - unsigned int sample_count = 64; + unsigned int sample_count = BUFFER_LIMIT; nvmlCheck(nvmlDeviceGetSamples(device, NVML_GPU_UTILIZATION_SAMPLES, 0, &v_type, &sample_count, buffer)); float gpu_utilization = 0.f; for(unsigned i = 0; i < sample_count; ++i) { gpu_utilization += (float)buffer[i].sampleValue.uiVal; } gpu_utilization /= (float)sample_count; - sample_count = 64; + sample_count = BUFFER_LIMIT; nvmlCheck(nvmlDeviceGetSamples(device, NVML_MEMORY_UTILIZATION_SAMPLES, 0, &v_type, &sample_count, buffer)); float mem_utilization = 0.f; for(unsigned i = 0; i < sample_count; ++i) { diff --git a/train_gpt2.cu b/train_gpt2.cu index 10cebcb..6e1c9cd 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -1829,11 +1829,10 @@ int main(int argc, char *argv[]) { printf0("step %4d/%d | loss %7.6f (%+.2fz)| norm %6.4f (%+.2fz)| lr %.2e | %.2f ms | %.1f%% bf16 MFU | %.0f tok/s\n", step + 1, train_num_batches, model.mean_loss, zloss, grad_norm, zgrad, step_learning_rate, time_elapsed_ms, 100*mfu, bias_corrected_ema_tokens_per_second); - if((step + 1) % 100 == 0) { + if((step + 1) % 10 == 0) { GPUUtilInfo gpu_info = get_gpu_utilization_info(); - printf0("GPU Status: Compute: %2.1f%% | Memory: %2.1f%% | Fan: %2d%%\n", gpu_info.gpu_utilization, gpu_info.mem_utilization, gpu_info.fan); - printf0(" Clock: %4d / %4d\n", gpu_info.clock, gpu_info.max_clock); - printf0(" Power: %4d / %4d\n", gpu_info.power, gpu_info.power_limit); + printf0(" compute %2.1f%% | memory: %2.1f%% | fan: %2d%% | clock: %4d MHz / %4d MHz | power %4d W / %4d W\n", + gpu_info.gpu_utilization, gpu_info.mem_utilization, gpu_info.fan, gpu_info.clock, gpu_info.max_clock, gpu_info.power / 1000, gpu_info.power_limit / 1000); } logger_log_train(&logger, step, model.mean_loss, step_learning_rate, grad_norm); From 294a6bca553f7291f30107064279ebee87802aa9 Mon Sep 17 00:00:00 2001 From: Erik Schultheis Date: Sun, 28 Jul 2024 12:07:22 +0200 Subject: [PATCH 3/6] throttle reason --- llmc/mfu.h | 18 ++++++++++++++++-- train_gpt2.cu | 5 +++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/llmc/mfu.h b/llmc/mfu.h index 2abca26..c765342 100644 --- a/llmc/mfu.h +++ b/llmc/mfu.h @@ -161,12 +161,24 @@ struct GPUUtilInfo { unsigned int power; unsigned int power_limit; unsigned int fan; - unsigned long long throttle; float gpu_utilization; float mem_utilization; + const char* throttle_reason; }; +const char* get_throttle_reason(unsigned long long bits) { + if(bits & (nvmlClocksThrottleReasonSwPowerCap | nvmlClocksThrottleReasonHwPowerBrakeSlowdown)) { + return "power cap"; + } else if (bits & (nvmlClocksThrottleReasonSwThermalSlowdown | nvmlClocksThrottleReasonHwThermalSlowdown)) { + return "thermal cap"; + } else if (bits & (nvmlClocksThrottleReasonAll)) { + return "other cap"; + } else { + return "no cap"; + } +} + GPUUtilInfo get_gpu_utilization_info() { GPUUtilInfo info; nvmlDevice_t device = nvml_get_device(); @@ -174,7 +186,9 @@ GPUUtilInfo get_gpu_utilization_info() { nvmlCheck(nvmlDeviceGetMaxClockInfo(device, NVML_CLOCK_SM, &info.max_clock)); nvmlCheck(nvmlDeviceGetPowerManagementLimit(device, &info.power_limit)); nvmlCheck(nvmlDeviceGetPowerUsage(device, &info.power)); - nvmlCheck(nvmlDeviceGetCurrentClocksThrottleReasons(device, &info.throttle)); + unsigned long long throttle; + nvmlCheck(nvmlDeviceGetCurrentClocksThrottleReasons(device, &throttle)); + info.throttle_reason = get_throttle_reason(throttle); nvmlCheck(nvmlDeviceGetFanSpeed(device, &info.fan)); // other potentially interesting functions // nvmlDeviceGetPcieThroughput diff --git a/train_gpt2.cu b/train_gpt2.cu index 6e1c9cd..9c806e3 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -1831,8 +1831,9 @@ int main(int argc, char *argv[]) { time_elapsed_ms, 100*mfu, bias_corrected_ema_tokens_per_second); if((step + 1) % 10 == 0) { GPUUtilInfo gpu_info = get_gpu_utilization_info(); - printf0(" compute %2.1f%% | memory: %2.1f%% | fan: %2d%% | clock: %4d MHz / %4d MHz | power %4d W / %4d W\n", - gpu_info.gpu_utilization, gpu_info.mem_utilization, gpu_info.fan, gpu_info.clock, gpu_info.max_clock, gpu_info.power / 1000, gpu_info.power_limit / 1000); + printf0(" compute %2.1f%% | memory: %2.1f%% | fan: %2d%% | %4d MHz / %4d MHz | %4d W / %4d W | %s\n", + gpu_info.gpu_utilization, gpu_info.mem_utilization, gpu_info.fan, gpu_info.clock, gpu_info.max_clock, gpu_info.power / 1000, gpu_info.power_limit / 1000, + gpu_info.throttle_reason); } logger_log_train(&logger, step, model.mean_loss, step_learning_rate, grad_norm); From bdff6727412d35107fbb6f1d0fa3f72eb435c075 Mon Sep 17 00:00:00 2001 From: Erik Schultheis Date: Sun, 28 Jul 2024 12:10:57 +0200 Subject: [PATCH 4/6] add temperature --- llmc/mfu.h | 20 +++++++++++++++----- train_gpt2.cu | 4 ++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/llmc/mfu.h b/llmc/mfu.h index c765342..0a2fb1f 100644 --- a/llmc/mfu.h +++ b/llmc/mfu.h @@ -144,6 +144,7 @@ float get_flops_promised(const char* device, int precision_mode) { return -1.0f; // ¯\_(ツ)_/¯ } +// lazily initialize nvml and generate a handle to the GPU nvmlDevice_t nvml_get_device() { static bool needs_init = true; static nvmlDevice_t device; @@ -161,12 +162,16 @@ struct GPUUtilInfo { unsigned int power; unsigned int power_limit; unsigned int fan; + unsigned int temperature; + unsigned int temp_slowdown; float gpu_utilization; float mem_utilization; const char* throttle_reason; }; +// convert throttle reason bitfield into a text reason. +// this is a lossy conversion; we just want to give some idea of what is happening const char* get_throttle_reason(unsigned long long bits) { if(bits & (nvmlClocksThrottleReasonSwPowerCap | nvmlClocksThrottleReasonHwPowerBrakeSlowdown)) { return "power cap"; @@ -179,22 +184,25 @@ const char* get_throttle_reason(unsigned long long bits) { } } +// gather data for a GPUUtilInfo object GPUUtilInfo get_gpu_utilization_info() { GPUUtilInfo info; nvmlDevice_t device = nvml_get_device(); + // query different infos directly nvmlCheck(nvmlDeviceGetClockInfo(device, NVML_CLOCK_SM, &info.clock)); nvmlCheck(nvmlDeviceGetMaxClockInfo(device, NVML_CLOCK_SM, &info.max_clock)); nvmlCheck(nvmlDeviceGetPowerManagementLimit(device, &info.power_limit)); nvmlCheck(nvmlDeviceGetPowerUsage(device, &info.power)); + nvmlCheck(nvmlDeviceGetTemperature(device, NVML_TEMPERATURE_GPU, &info.temperature)); + nvmlCheck(nvmlDeviceGetTemperatureThreshold(device, NVML_TEMPERATURE_THRESHOLD_SLOWDOWN, &info.temp_slowdown)); unsigned long long throttle; nvmlCheck(nvmlDeviceGetCurrentClocksThrottleReasons(device, &throttle)); info.throttle_reason = get_throttle_reason(throttle); nvmlCheck(nvmlDeviceGetFanSpeed(device, &info.fan)); - // other potentially interesting functions - // nvmlDeviceGetPcieThroughput - // nvmlDeviceGetPcieSpeed - // nvmlDeviceGetNumGpuCores - // nvmlDeviceGetMemoryBusWidth + + // for "utilization", we look at recorded samples. In principle, we could query the driver for how many samples + // to request, but then we'd need to dynamically allocate sufficient space. Let's just hard-code a limit of 128, + // and have no memory management required constexpr const int BUFFER_LIMIT = 128; nvmlSample_t buffer[BUFFER_LIMIT]; nvmlValueType_t v_type; @@ -205,6 +213,8 @@ GPUUtilInfo get_gpu_utilization_info() { gpu_utilization += (float)buffer[i].sampleValue.uiVal; } gpu_utilization /= (float)sample_count; + + // sample count may have been modified by the query above; reset back to buffer size sample_count = BUFFER_LIMIT; nvmlCheck(nvmlDeviceGetSamples(device, NVML_MEMORY_UTILIZATION_SAMPLES, 0, &v_type, &sample_count, buffer)); float mem_utilization = 0.f; diff --git a/train_gpt2.cu b/train_gpt2.cu index 9c806e3..0226fe1 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -1831,9 +1831,9 @@ int main(int argc, char *argv[]) { time_elapsed_ms, 100*mfu, bias_corrected_ema_tokens_per_second); if((step + 1) % 10 == 0) { GPUUtilInfo gpu_info = get_gpu_utilization_info(); - printf0(" compute %2.1f%% | memory: %2.1f%% | fan: %2d%% | %4d MHz / %4d MHz | %4d W / %4d W | %s\n", + printf0(" compute %2.1f%% | memory: %2.1f%% | fan: %2d%% | %4d MHz / %4d MHz | %3d W / %3d W | %d°C / %d°C | %s\n", gpu_info.gpu_utilization, gpu_info.mem_utilization, gpu_info.fan, gpu_info.clock, gpu_info.max_clock, gpu_info.power / 1000, gpu_info.power_limit / 1000, - gpu_info.throttle_reason); + gpu_info.temperature, gpu_info.temp_slowdown, gpu_info.throttle_reason); } logger_log_train(&logger, step, model.mean_loss, step_learning_rate, grad_norm); From bb9467dcca9ff84fa5d58778a92ee8371949af4d Mon Sep 17 00:00:00 2001 From: Erik Schultheis Date: Sun, 28 Jul 2024 12:35:19 +0200 Subject: [PATCH 5/6] make gpu log configurable --- train_gpt2.cu | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/train_gpt2.cu b/train_gpt2.cu index 0226fe1..efddf86 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -1331,6 +1331,7 @@ void error_usage() { fprintf(stderr, " -j val data filename pattern (default = dev/data/tinyshakespeare/tiny_shakespeare_val.bin)\n"); fprintf(stderr, " -e input .bin filename or descriptor, see code comments as docs. (default = gpt2_124M_bf16.bin)\n"); fprintf(stderr, " -o output log dir (default = NULL, no logging)\n"); + fprintf(stderr, " -lg log gpu info every x steps (default = -1; disabled)\n"); fprintf(stderr, " -n write optimization checkpoints every how many steps? (default 0, don't)\n"); fprintf(stderr, " -nk max number of checkpoints to keep in the directory, removing old ones (0 = disable, default)\n"); fprintf(stderr, " -nm every how many step checkpoints are considered major? major checkpoints never get deleted.\n"); @@ -1391,6 +1392,7 @@ int main(int argc, char *argv[]) { int T = 1024; // sequence length max int total_batch_size = -1; // will be calculated down below later, if not provided float learning_rate = 3e-4f; + int log_gpu_every = -1; int warmup_iterations = 0; float final_learning_rate_frac = 1.0f; // final fraction of learning rate, at end of training float weight_decay = 0.0f; @@ -1429,7 +1431,8 @@ int main(int argc, char *argv[]) { else if (argv[i][1] == 'b') { B = atoi(argv[i+1]); } // Per-GPU (micro) batch size else if (argv[i][1] == 't') { T = atoi(argv[i+1]); } else if (argv[i][1] == 'd') { total_batch_size = atoi(argv[i+1]); } - else if (argv[i][1] == 'l') { learning_rate = atof(argv[i+1]); } + else if (argv[i][1] == 'l' && argv[i][2] == '\0') { learning_rate = atof(argv[i+1]); } + else if (argv[i][1] == 'l' && argv[i][2] == 'g') { log_gpu_every = atoi(argv[i+1]); } else if (argv[i][1] == 'u') { warmup_iterations = atoi(argv[i+1]); } else if (argv[i][1] == 'q') { final_learning_rate_frac = atof(argv[i+1]); } else if (argv[i][1] == 'c') { weight_decay = atof(argv[i+1]); } @@ -1829,7 +1832,7 @@ int main(int argc, char *argv[]) { printf0("step %4d/%d | loss %7.6f (%+.2fz)| norm %6.4f (%+.2fz)| lr %.2e | %.2f ms | %.1f%% bf16 MFU | %.0f tok/s\n", step + 1, train_num_batches, model.mean_loss, zloss, grad_norm, zgrad, step_learning_rate, time_elapsed_ms, 100*mfu, bias_corrected_ema_tokens_per_second); - if((step + 1) % 10 == 0) { + if(log_gpu_every > 0 && (step + 1) % log_gpu_every == 0) { GPUUtilInfo gpu_info = get_gpu_utilization_info(); printf0(" compute %2.1f%% | memory: %2.1f%% | fan: %2d%% | %4d MHz / %4d MHz | %3d W / %3d W | %d°C / %d°C | %s\n", gpu_info.gpu_utilization, gpu_info.mem_utilization, gpu_info.fan, gpu_info.clock, gpu_info.max_clock, gpu_info.power / 1000, gpu_info.power_limit / 1000, From e4b5e5775f91fcf61e21784d3b18b39d4200de81 Mon Sep 17 00:00:00 2001 From: Erik Schultheis Date: Mon, 29 Jul 2024 10:00:44 +0200 Subject: [PATCH 6/6] disable if header is not found --- Makefile | 3 ++- llmc/mfu.h | 39 ++++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index cba1a37..f2cce2d 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ CUDA_OUTPUT_FILE = -o $@ # NVCC flags # -t=0 is short for --threads, 0 = number of CPUs on the machine NVCC_FLAGS = -O3 -t=0 --use_fast_math -std=c++17 -NVCC_LDFLAGS = -lcublas -lcublasLt -lnvidia-ml +NVCC_LDFLAGS = -lcublas -lcublasLt NVCC_INCLUDES = NVCC_LDLIBS = NCLL_INCUDES = @@ -62,6 +62,7 @@ $(info ---------------------------------------------) ifneq ($(OS), Windows_NT) NVCC := $(shell which nvcc 2>/dev/null) + NVCC_LDFLAGS += -lnvidia-ml # Function to test if the compiler accepts a given flag. define check_and_add_flag diff --git a/llmc/mfu.h b/llmc/mfu.h index 0a2fb1f..1c40b7b 100644 --- a/llmc/mfu.h +++ b/llmc/mfu.h @@ -4,13 +4,19 @@ #include #include #include +#if __has_include() +#define USE_NVML 1 #include +#else +#define USE_NVML 0 +#endif // tied to enum PrecisionMode, in a future refactor make them the same #define MFUH_PRECISION_FP32 0 #define MFUH_PRECISION_FP16 1 #define MFUH_PRECISION_BF16 2 +#if USE_NVML inline void nvml_check(nvmlReturn_t status, const char *file, int line) { if (status != NVML_SUCCESS) { printf("[NVML ERROR] at file %s:%d:\n%s\n", file, line, nvmlErrorString(status)); @@ -18,6 +24,7 @@ inline void nvml_check(nvmlReturn_t status, const char *file, int line) { } }; #define nvmlCheck(err) (nvml_check(err, __FILE__, __LINE__)) +#endif typedef struct { @@ -144,18 +151,6 @@ float get_flops_promised(const char* device, int precision_mode) { return -1.0f; // ¯\_(ツ)_/¯ } -// lazily initialize nvml and generate a handle to the GPU -nvmlDevice_t nvml_get_device() { - static bool needs_init = true; - static nvmlDevice_t device; - if(needs_init) { - needs_init = false; - nvmlCheck(nvmlInit()); - nvmlCheck(nvmlDeviceGetHandleByIndex_v2(0, &device)); - } - return device; -} - struct GPUUtilInfo { unsigned int clock; unsigned int max_clock; @@ -170,6 +165,19 @@ struct GPUUtilInfo { const char* throttle_reason; }; +// lazily initialize nvml and generate a handle to the GPU +#if USE_NVML +nvmlDevice_t nvml_get_device() { + static bool needs_init = true; + static nvmlDevice_t device; + if(needs_init) { + needs_init = false; + nvmlCheck(nvmlInit()); + nvmlCheck(nvmlDeviceGetHandleByIndex_v2(0, &device)); + } + return device; +} + // convert throttle reason bitfield into a text reason. // this is a lossy conversion; we just want to give some idea of what is happening const char* get_throttle_reason(unsigned long long bits) { @@ -227,5 +235,10 @@ GPUUtilInfo get_gpu_utilization_info() { info.mem_utilization = mem_utilization; return info; } - +#else +GPUUtilInfo get_gpu_utilization_info() { + fprintf(stderr, "Error: Compiled without nvml support. Cannot perform additional GPU state tracking."); + exit(EXIT_FAILURE); +} +#endif #endif // MFU_H