aparler/PyNorch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

softmax and small fixes

Browse source
This commit is contained in:
lucasdelimanogueira 2024-05-21 02:03:24 -03:00
parent a39bd59a3f
commit a57298dfcb
4 changed files with 80 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
norch/autograd/__pycache__/functions.cpython-38.pyc
View file

Binary file not shown.

34
norch/autograd/functions.py
View file

@ -137,13 +137,14 @@ class SumBackward:
# If axis is None, sum reduces the tensor to a scalar.
grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like()
else:
if not self.keepdim:
# Remove dimensions of size 1 from the gradient tensor.
input_shape = [s for i, s in enumerate(input_shape) if i != self.axis]
if self.keepdim:
input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:]
else:
input_shape = input_shape[:self.axis] + input_shape[self.axis+1:]
# Broadcast the gradient to the input shape along the specified axis.
grad_output_shape = list(input_shape)
grad_output_shape.insert(self.axis, 1)
grad_output = gradient.reshape(grad_output_shape)
grad_output = grad_output + self.input[0].zeros_like()
@ -216,14 +217,15 @@ class MaxBackward:
grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0]
else:
if not self.keepdim:
# Remove dimensions of size 1 from the gradient tensor.
input_shape = [s for i, s in enumerate(input_shape) if i != self.axis]
if self.keepdim:
input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:]
else:
input_shape = input_shape[:self.axis] + input_shape[self.axis+1:]
# Broadcast the gradient to the input shape along the specified axis.
grad_output_shape = list(input_shape)
grad_output_shape.insert(self.axis, 1)
grad_output = gradient.reshape(grad_output_shape)
grad_output = grad_output + self.input[0].zeros_like()
max_values = self.input[0].max(axis=self.axis, keepdim=True)
@ -251,13 +253,13 @@ class MinBackward:
grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0]
else:
if not self.keepdim:
# Remove dimensions of size 1 from the gradient tensor.
input_shape = [s for i, s in enumerate(input_shape) if i != self.axis]
if self.keepdim:
input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:]
else:
input_shape = input_shape[:self.axis] + input_shape[self.axis+1:]
# Broadcast the gradient to the input shape along the specified axis.
grad_output_shape = list(input_shape)
grad_output_shape.insert(self.axis, 1)
grad_output = gradient.reshape(grad_output_shape)
grad_output = grad_output + self.input[0].zeros_like()
max_values = self.input[0].min(axis=self.axis, keepdim=True)

10
test.py
View file

@ -6,10 +6,12 @@ import norch.optim as optim
import random
random.seed(1)
torch_tensor = norch.Tensor([[[2, 2], [-1, -1]], [[1., 2], [3, 3]]], requires_grad=True)#.to(self.device)
torch_tensor = norch.Tensor([[[2, 2], [2, 2]], [[2, 2.], [2, 2]]])
b = norch.nn.functional.softmax(torch_tensor)
print(b)
torch_tensor = norch.Tensor([[[1, 5], [2, -1]], [[5, 2.], [2, 2]]], requires_grad=True)#.to(self.device)
b = norch.nn.functional.softmax(torch_tensor, dim=0)
soma = b.sum()
print(soma)
soma.backward()
print(torch_tensor.grad)
"""a = norch.Tensor([[[4.186502456665039]]])
b = norch.Tensor([[[2.0, 2.0,],[-1.0, -1.0,]],[[1.0, 2.0,],[3.0, 3.0,]]])

56
tests/test_autograd.py
View file

@ -583,6 +583,62 @@ class TestTensorAutograd(unittest.TestCase):
torch_expected_cos_tensor_grad = torch_cos_tensor.grad
self.assertTrue(utils.compare_torch(torch_result_cos_tensor_grad, torch_expected_cos_tensor_grad))
def test_sigmoid(self):
"""
Test autograd from sigmoid
"""
norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device)
norch_sigmoid = norch.sigmoid(norch_tensor)
norch_result = norch_sigmoid.sum()
norch_result.backward()
norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device)
torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device)
torch_sigmoid = torch.sigmoid(torch_tensor)
torch_result = torch_sigmoid.sum()
torch_result.backward()
torch_tensor_grad = torch_tensor.grad
self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad))
def test_softmax(self):
"""
Test autograd from softmax
"""
norch_tensor = norch.Tensor([[[-5, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device)
norch_softmax = norch.softmax(norch_tensor, dim=1)
norch_result = norch_softmax.sum()
norch_result.backward()
norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device)
torch_tensor = torch.tensor([[[-5, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device)
torch_softmax = torch.softmax(torch_tensor, dim=1)
torch_result = torch_softmax.sum()
torch_result.backward()
torch_tensor_grad = torch_tensor.grad
self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad))
norch_tensor = norch.Tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device)
norch_softmax = norch.softmax(norch_tensor, dim=2)
norch_result = norch_softmax.sum()
norch_result.backward()
norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device)
torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device)
torch_softmax = torch.softmax(torch_tensor, dim=2)
torch_result = torch_softmax.sum()
torch_result.backward()
torch_tensor_grad = torch_tensor.grad
self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad))
def test_reshape(self):
"""