From d530f79a82a0b7b1dcb1392b05b4e5f5e8a1604d Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Tue, 21 May 2024 12:16:12 -0300 Subject: [PATCH] cross entropy loss --- norch/nn/__pycache__/loss.cpython-38.pyc | Bin 1420 -> 2620 bytes norch/nn/functional.py | 17 ++++- norch/nn/loss.py | 70 +++++++++++++++++++-- test.py | 19 ++++-- tests/test_nn.py | 77 +++++++++++++++++++++-- 5 files changed, 168 insertions(+), 15 deletions(-) diff --git a/norch/nn/__pycache__/loss.cpython-38.pyc b/norch/nn/__pycache__/loss.cpython-38.pyc index a39202c7182c730dfc171bf4b535752507f08ad9..df6fd5a931db4a8f313c204e140bf1b4c3b58460 100644 GIT binary patch literal 2620 zcmcIlPj4GV6rX>)-Z)Mg8Yn4+cFP|XOQ`OJ9;&LMv{hB9jSwjud|7RFCW)K%u9+DN z;;b)8q#`6xKZ1P>XD)mT&U@w53mCN2F(x$LLvIn2l^%=zul?<6>xefVTz zuE9W&HkAZW&^CWcHYlPx2--pl=^c*PLM&6GQzuj!xMx_E#a$IeWj%_z zj#w~mMA0|>IGt!}Q6%#uikwTGY5f+QSuRYjD^X}>=ARp&)%fEY0|T4$YCr{R@PIpM z0@7p8^g_&sAf7VTu@(3VY~_+m0W1}cr5>={u62}-O)gd zdzsaFZ|8sD$R&h5NiFA3+B%&q$hj9}`5tI$fdH01Vf)Nnv>eDHtXI%|I)=7KLCYrE zo1kS*3nZ$vKpkK5ZKi(%f6@OdmTO)Uvs$lNnB2f zX#L6Q{B(qc4h3Uv{EmbPAtGGK39q>9dL``kjSbgT z_*i8o*PpmJ+#L?CziL|*Q?J^mF-d7mc~D)r>~sg$9$*%C0bOlQgOGr3mvAB z2BeXyuTy**Y>}R2LbwFu#y)Sb2CHL(_2)phCBA46C2csJnvxLN;S(7nT}- E19Fl?YybcN delta 297 zcmX|5Jxjw-6n*!7Bx#zYSQP}nKoEQ&E`mCU4k87yE=7 class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + logits = norch.softmax(input, dim=0) + cost = -(logits.log() * target).sum() + + return cost + + + elif input.ndim == 2: + # batched + if target.ndim == 1: + # target -> Ground truth class indices: + num_classes = input.shape[1] + + target = norch.one_hot_encode(target, num_classes) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + cost = -(logits.log() * target).sum() / batch_size + + return cost + + else: + # target -> class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + cost = -(logits.log() * target).sum() / batch_size + + return cost + + + diff --git a/test.py b/test.py index d61d329..54b67fb 100644 --- a/test.py +++ b/test.py @@ -6,12 +6,19 @@ import norch.optim as optim import random random.seed(1) -torch_tensor = norch.Tensor([[[1, 5], [2, -1]], [[5, 2.], [2, 2]]], requires_grad=True)#.to(self.device) -b = norch.nn.functional.softmax(torch_tensor, dim=0) -soma = b.sum() -print(soma) -soma.backward() -print(torch_tensor.grad) +"""one_hot_target = norch.one_hot_encode(norch.Tensor([5]), num_classes=10) +print(one_hot_target)""" + +logits = norch.Tensor([[2.0, 1.0, 0.1, 0.1], [2.0, 1.0, 0.1, 0.1], [2.0, 1.0, 0.1, 0.1]], requires_grad=True) + +# One-hot encoded target with shape (batch_size, num_classes) +one_hot_target = norch.Tensor([0, 1, 1]) + +criterion = nn.CrossEntropyLoss() + +loss = criterion(logits, one_hot_target) +print(loss) + """a = norch.Tensor([[[4.186502456665039]]]) b = norch.Tensor([[[2.0, 2.0,],[-1.0, -1.0,]],[[1.0, 2.0,],[3.0, 3.0,]]]) diff --git a/tests/test_nn.py b/tests/test_nn.py index 1010c82..cd623f3 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -21,13 +21,13 @@ class TestNNModuleLoss(unittest.TestCase): loss_fn_torch = torch.nn.MSELoss() # Test case 1: Predictions and labels are equal - predictions_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device) - labels_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device) + predictions_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device) + labels_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device) loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device) - labels_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device) + predictions_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device) + labels_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) @@ -44,6 +44,75 @@ class TestNNModuleLoss(unittest.TestCase): self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + def test_cross_entropy_loss(self): + """ + Test the CrossEntropyLoss + """ + loss_fn_norch = norch.nn.CrossEntropyLoss() + loss_fn_torch = torch.nn.CrossEntropyLoss() + + # Test case 1: Single class, single sample + predictions_norch = norch.Tensor([2.0, 1.0, 0.1]).to(self.device) + labels_norch = norch.Tensor([0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([2.0, 1.0, 0.1]).to(self.device) + labels_torch = torch.tensor(0).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 2: Multiple classes, multiple samples + predictions_norch = norch.Tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device) + labels_norch = norch.Tensor([2, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device) + labels_torch = torch.tensor([2, 1]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 3: Edge case - all predictions are zero + predictions_norch = norch.Tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device) + labels_norch = norch.Tensor([1, 2]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device) + labels_torch = torch.tensor([1, 2]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Class probabilities instead of class index + predictions_norch = norch.Tensor([0.5, 0.2, 0.1]).to(self.device) + labels_norch = norch.Tensor([1., 0, 0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([0.5, 0.2, 0.1]).to(self.device) + labels_torch = torch.tensor([1., 0, 0]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Batched class probabilities instead of class index + predictions_norch = norch.Tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device) + labels_norch = norch.Tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + class TestNNModuleActivationFn(unittest.TestCase):