From 098ede5a6ba1dbe4173388eb4ec93c18577618a5 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Mon, 20 May 2024 15:24:55 -0300 Subject: [PATCH] min max autograd --- norch/__pycache__/tensor.cpython-38.pyc | Bin 17577 -> 17692 bytes .../__pycache__/functions.cpython-38.pyc | Bin 8929 -> 9732 bytes norch/autograd/functions.py | 42 ++++++- norch/tensor.py | 13 ++- test.py | 7 +- tests/test_autograd.py | 105 +++++++++++++++++- tests/test_operations.py | 2 +- 7 files changed, 158 insertions(+), 11 deletions(-) diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 983cfe6016a3d31faae8b3a71fef51ec2b6b18f5..4d65981655e17c69dd8e981cc0fa199a12daf864 100644 GIT binary patch delta 1116 zcmZvb%TH556o>DbTPXHPDV0Y>K!{Ml2a%T`4{Id|2_YdGs3;UmvE@Bf0WAUoDkut> zuv08(Od!!~(R z^+vanyO(e*;WLBF;%id2)~)gSd-C0iHrWZ_6Sej{=un328vvccl#v&Ao3b(ys~YYg z->(ExmYCi{emlV{tZ7-`dsCJMseTfCgaDy{P)JbM(W|)A6@We}tnVk83iEDt6HOU* zS+C}+2?ZpIg_Co}yBOnOk>;vek##M4h%(jPi1%3yk%O>guge;$Av|`Q*e%v6-*eXh z_8P35Y6ATjrTx=2@k}*~%=7#Z=pX(+FAV2AV>_q?r4R&RlJT4n1eh1P$dkhIGU^G^ zD4?$i!7?=NM-H~AIZ?rAAH%$+!=t^dQ3^=`EmqZb$-JA!!IO`b0d!2qOkhAN69q0{@0i%C$WeY4?ttE{@eKP1`4fzdd@>V!66VrMrL3eHq93E~ zu!;t&4Q9lv^UX=KC>(5g3$-1^TsnOqe!Z+2X6b*4l6(BUf3Rwm(taTgAVX|cU)&4&s12=@sO2;+na!X#meKqt!|5uOrg*Zi53f}A7wIpGCj dJEoua^>^^#2%i^UF5QF^;4 z3IvYmLW30DU?R~BJ2&ztkhm~i>(U+Wy=PhyHBK_W``z!J@7}q0?(0R^UI61&gF(x& zXFt(9T9y7}TsJ^gerFgHAdC9qCg{eC#lN5j_Zv*IuOv$LuGtME_|v>qbb|%On*pu!Angjh7Gmf~m^7n@K z0J#OWP7i=?mSTUbT7JgI=f}ln33N~X9!LFIe|njDDNceE%51d|rSMqt6hSz0P{(GUMxWtC?M`eT^IqseV^)u!3qerdDfaZeD2aNaXgbBUJe`OngqnHNKKSBTyCM3*bwK{cI(ZbC1CYQ#Q*a_R4o zN)SZCUBVQBu9i4WNE03rW(aeH$Ao#p0%4Kxl(0&mBM{fPYDOgaoUqQ2Elh~8JT+g1tLrU!_6 zL|(*Nb!zi!mS`j;Te2l-#F8?uEcvBNHvkpC3*vHf z^B$R@5iufdI1?3?uugkmo0O1nqn%PLB~uAHF(PLLN-UbGvJ`kRY;3>ElO;OrXZwFm zv?P&AJ7Y}5lKE5ZS{v(xVz3ZG{@`6`bpd>UWhwv)+$vc+Hejss(??p<)y0NDVK+iO zKu%mTR&#RnR?=8x9T>fU(1Xy6a1p_e;6u2Ca2eqWLcY)XzzRATo)U8-^aD(l!N%d+ z#$qy>la>r~p`Q&PH;7PZRkZSHSAz8(me~YAXcbkdn^p-;f=^V08qo=!N~;Bo)eCDH ze$dyzaql#U-@z(2sVcMSV#}Q1oVLFLba&&LY|XE*Wje7RNrNUzqv zfp6l`loVIydw~vb7^0t%FN``=G0fYWJCEY;g5iIF;lMK-Y=)!Ea8eo89)#~RQDRDb zerX9%C`u5C1CcKRnNeiFSwt+Ck=ql*W(5)AljYfECa(~85cURgX8>_A3y|Po?P*qzM nP`DL=f}jZG*CLP)i$Fdp0y(q@WNlHz(R`mosbo diff --git a/norch/autograd/functions.py b/norch/autograd/functions.py index b862b73..e5b3c20 100644 --- a/norch/autograd/functions.py +++ b/norch/autograd/functions.py @@ -206,8 +206,34 @@ class MaxBackward: self.keepdim = keepdim def backward(self, gradient): - pass + input_shape = self.input[0].shape.copy() + if self.axis == -1: + max_value = self.input[0].max() + mask = self.input[0].equal(max_value) + grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + + grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0] + + else: + if not self.keepdim: + # Remove dimensions of size 1 from the gradient tensor. + input_shape = [s for i, s in enumerate(input_shape) if i != self.axis] + + # Broadcast the gradient to the input shape along the specified axis. + grad_output_shape = list(input_shape) + grad_output_shape.insert(self.axis, 1) + grad_output = gradient.reshape(grad_output_shape) + grad_output = grad_output + self.input[0].zeros_like() + + max_value = self.input[0].max() + mask = self.input[0].equal(max_value) + + grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0] + + return [grad_output] + + class MinBackward: def __init__(self, x, axis=None, keepdim=False): self.input = [x] @@ -215,6 +241,18 @@ class MinBackward: self.keepdim = keepdim def backward(self, gradient): - pass + input_shape = self.input[0].shape.copy() + if self.axis == -1: + min_value = self.input[0].min() + mask = self.input[0].equal(min_value) + + grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + + grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0] + + else: + pass + + return [grad_output] diff --git a/norch/tensor.py b/norch/tensor.py index 677231b..e8963e8 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -643,8 +643,19 @@ class Tensor: def equal(self, other): + if isinstance(other, Tensor) and other.numel == 1: + # other is a single value tensor + other = self.zeros_like() + other + return self.equal(other) + if not isinstance(other, Tensor): - return False + # other is a single value + if isinstance(other, (int, float)): + other = self.zeros_like() + other + + return self.equal(other) + else: + return False if self.shape != other.shape: return False diff --git a/test.py b/test.py index 9cd3b38..8dae5e0 100644 --- a/test.py +++ b/test.py @@ -7,8 +7,11 @@ import random random.seed(1) a = norch.Tensor([1, 2, 3]) -b = norch.Tensor([1, 2, 4]) -print(a == b) +b = 3 +for i in range(a.numel): + print(a.tensor.contents.data[i]) + +print(a) """ diff --git a/tests/test_autograd.py b/tests/test_autograd.py index aa31ca8..16e09bd 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -56,39 +56,134 @@ class TestTensorAutograd(unittest.TestCase): self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + norch_result = (norch_tensor1 + norch_tensor2).sum(axis=1).sum() + + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + + torch_result = (torch_tensor1 + torch_tensor2).sum(axis=1).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + + def test_max(self): + """ + Test autograd from max + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result = norch_tensor.max() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_result = torch_tensor.max() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) def test_max_axis(self): """ Test autograd from max specifying axis """ norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) - norch_result = norch_tensor.max(axis=1).sum() + norch_max_axis = norch_tensor.max(axis=1) + norch_result = norch_max_axis.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_max_axis, _ = torch_tensor.max(axis=1) + torch_result = torch_max_axis.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_max_axis = norch_tensor.max(axis=2) + norch_result = norch_max_axis.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_max_axis, _ = torch_tensor.max(axis=2) + torch_result = torch_max_axis.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + print(norch_tensor_grad, '\n\n', torch_tensor_grad) + + + def test_min(self): + """ + Test autograd from min + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result = norch_tensor.min() norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) - torch_result = torch_tensor.max(axis=1).sum() + torch_result = torch_tensor.min() torch_result.backward() torch_tensor_grad = torch_tensor.grad self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) - def test_min_axis(self): """ Test autograd from min specifying axis """ norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) - norch_result = norch_tensor.min(axis=1).sum() + norch_min = norch_tensor.min(axis=1) + norch_result = norch_min.sum() norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) - torch_result = torch_tensor.min(axis=1).sum() + torch_min = torch_tensor.min(axis=1) + torch_result = torch_min.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_min = norch_tensor.min(axis=2) + norch_result = norch_min.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_min = torch_tensor.min(axis=2) + torch_result = torch_min.sum() torch_result.backward() torch_tensor_grad = torch_tensor.grad diff --git a/tests/test_operations.py b/tests/test_operations.py index ab65a64..008136d 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -533,7 +533,7 @@ class TestTensorOperations(unittest.TestCase): def test_equal(self): """ - Test equal two tensors: tensor1 == tensor2 + Test equal two tensors: tensor1.equal(tensor2) """ norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 1], [7, 8]]]).to(self.device) norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device)