aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use NCCL stream

Browse source
This commit is contained in:
Aleksa Gordic 2024-06-23 10:39:55 +00:00
parent 52db158367
commit ac72875c24
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
train_gpt2.cu
View file

@ -899,7 +899,7 @@ float multi_gpu_cpu_float_sum(float value, MultiGpuConfig* multi_gpu_config) {
float* unified_buffer = multi_gpu_config->unified_buffer;
*unified_buffer = value;
ncclCheck(ncclAllReduce(unified_buffer, unified_buffer, sizeof(float), ncclFloat, ncclSum, multi_gpu_config->nccl_comm, main_stream));
ncclCheck(ncclAllReduce(unified_buffer, unified_buffer, sizeof(float), ncclFloat, ncclSum, multi_gpu_config->nccl_comm, multi_gpu_config->nccl_stream));
cudaCheck(cudaDeviceSynchronize());
return *unified_buffer;
#else