From 086ce2ff4de9752b34b3f098c648230f04684e1e Mon Sep 17 00:00:00 2001 From: chinthysl Date: Mon, 13 May 2024 06:48:22 +0000 Subject: [PATCH] Remove unsused template specializations and refactor --- train_gpt2.cu | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/train_gpt2.cu b/train_gpt2.cu index ad4f951..8a743ad 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -1282,16 +1282,6 @@ __device__ float cast_value(half val) { return __half2float(val); } -template<> -__device__ half cast_value(float val) { - return __float2half(val); -} - -template<> -__device__ __nv_bfloat16 cast_value<__nv_bfloat16, float>(float val) { - return __float2bfloat16(val); -} - template<> __device__ float cast_value(__nv_bfloat16 val) { return __bfloat162float(val); @@ -2302,7 +2292,6 @@ void gpt2_backward(GPT2 *model) { // Compute a mean of a single CPU value across all GPU processes. No-op when multi-GPU is disabled. float multi_gpu_cpu_float_mean(float value, const MultiGpuConfig* multi_gpu_config) { #ifdef MULTI_GPU - if (multi_gpu_config->num_processes == 1) return value; // MPI doesn't support all reduce with mean, so we sum up, then divide. float result; mpiCheck(MPI_Allreduce(&value, &result, 1, MPI_FLOAT, MPI_SUM, MPI_COMM_WORLD)); @@ -2315,11 +2304,11 @@ float multi_gpu_cpu_float_mean(float value, const MultiGpuConfig* multi_gpu_conf // Averages out the loss and gradients across all GPUs. No-op when multi-GPU is disabled. // todo - this version only works if all the parameters are the same size (floatX) void gpt2_multi_gpu_accumulate(GPT2* model, MultiGpuConfig* multi_gpu_config) { +#ifdef MULTI_GPU NVTX_RANGE_FN(); + if (multi_gpu_config->num_processes == 1) return; // Average all losses. model->accumulated_mean_loss = multi_gpu_cpu_float_mean(model->mean_loss, multi_gpu_config); -#ifdef MULTI_GPU - if (multi_gpu_config->num_processes == 1) return; // Average all gradients. ncclCheck(ncclAllReduce(model->grads_memory, model->grads_memory, model->num_parameters,