aparler/PyNorch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix allreduce distributed not working after 2nd iteration

Browse source
This commit is contained in:
lucasdelimanogueira 2024-06-03 15:15:35 -03:00
parent c42d334599
commit 7691cbc2dc
1 changed files with 0 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
norch/nn/parallel.py
View file

@ -32,7 +32,6 @@ class DistributedDataParallel(Module):
if isinstance(grad, norch.Tensor):
dist.allreduce_sum_tensor(grad)
avg_grad = grad / dist.get_world_size()
avg_grad = 0*grad + 5
return avg_grad
def register_grads_hooks(self):