diff --git a/norch/nn/__pycache__/loss.cpython-38.pyc b/norch/nn/__pycache__/loss.cpython-38.pyc index a39202c..df6fd5a 100644 Binary files a/norch/nn/__pycache__/loss.cpython-38.pyc and b/norch/nn/__pycache__/loss.cpython-38.pyc differ diff --git a/norch/nn/functional.py b/norch/nn/functional.py index 75654cf..9493591 100644 --- a/norch/nn/functional.py +++ b/norch/nn/functional.py @@ -1,4 +1,6 @@ import math +import norch +import numpy as np def sigmoid(x): return 1.0 / (1.0 + (math.e) ** (-x)) @@ -15,4 +17,17 @@ def softmax(x, dim=None): return exp_x / sum_exp_x else: sum_exp_x = exp_x.sum() - return exp_x / sum_exp_x \ No newline at end of file + return exp_x / sum_exp_x + +def one_hot_encode(x, num_classes): + one_hot = [[0] * num_classes for _ in range(x.numel)] + + # Set the appropriate elements to 1 + for i in range(x.numel): + target_idx = int(x.tensor.contents.data[i]) + one_hot[i][target_idx] = 1 + + if x.numel < 2: + one_hot = one_hot[0] + + return norch.Tensor(one_hot) \ No newline at end of file diff --git a/norch/nn/loss.py b/norch/nn/loss.py index 789fab6..bfc4390 100644 --- a/norch/nn/loss.py +++ b/norch/nn/loss.py @@ -1,4 +1,5 @@ from .module import Module +import norch from abc import ABC class Loss(Module, ABC): @@ -18,8 +19,69 @@ class MSELoss(Loss): def __init__(self): super().__init__() - def forward(self, predictions, labels): - assert labels.shape == predictions.shape, \ - "Labels and predictions shape does not match: {} and {}".format(labels.shape, predictions.shape) + def forward(self, predictions, target): + assert target.shape == predictions.shape, \ + "Labels and predictions shape does not match: {} and {}".format(target.shape, predictions.shape) - return ((predictions - labels) ** 2).sum() / predictions.numel \ No newline at end of file + cost = ((predictions - target) ** 2).sum() / predictions.numel + return cost + + +class CrossEntropyLoss(Loss): + def __init__(self): + super().__init__() + + def forward(self, input, target): + assert isinstance(input, norch.Tensor), \ + "Cross entropy argument 'input' must be Tensor, not {}".format(type(input)) + + assert isinstance(target, norch.Tensor), \ + "Cross entropy argument 'target' must be Tensor, not {}".format(type(target)) + + if input.ndim == 1: + if target.numel == 1: + num_classes = input.shape[0] + target = norch.one_hot_encode(target, num_classes) + + logits = norch.softmax(input, dim=0) + cost = -(logits.log() * target).sum() + + return cost + + else: + # target -> class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + logits = norch.softmax(input, dim=0) + cost = -(logits.log() * target).sum() + + return cost + + + elif input.ndim == 2: + # batched + if target.ndim == 1: + # target -> Ground truth class indices: + num_classes = input.shape[1] + + target = norch.one_hot_encode(target, num_classes) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + cost = -(logits.log() * target).sum() / batch_size + + return cost + + else: + # target -> class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + cost = -(logits.log() * target).sum() / batch_size + + return cost + + + diff --git a/test.py b/test.py index d61d329..54b67fb 100644 --- a/test.py +++ b/test.py @@ -6,12 +6,19 @@ import norch.optim as optim import random random.seed(1) -torch_tensor = norch.Tensor([[[1, 5], [2, -1]], [[5, 2.], [2, 2]]], requires_grad=True)#.to(self.device) -b = norch.nn.functional.softmax(torch_tensor, dim=0) -soma = b.sum() -print(soma) -soma.backward() -print(torch_tensor.grad) +"""one_hot_target = norch.one_hot_encode(norch.Tensor([5]), num_classes=10) +print(one_hot_target)""" + +logits = norch.Tensor([[2.0, 1.0, 0.1, 0.1], [2.0, 1.0, 0.1, 0.1], [2.0, 1.0, 0.1, 0.1]], requires_grad=True) + +# One-hot encoded target with shape (batch_size, num_classes) +one_hot_target = norch.Tensor([0, 1, 1]) + +criterion = nn.CrossEntropyLoss() + +loss = criterion(logits, one_hot_target) +print(loss) + """a = norch.Tensor([[[4.186502456665039]]]) b = norch.Tensor([[[2.0, 2.0,],[-1.0, -1.0,]],[[1.0, 2.0,],[3.0, 3.0,]]]) diff --git a/tests/test_nn.py b/tests/test_nn.py index 1010c82..cd623f3 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -21,13 +21,13 @@ class TestNNModuleLoss(unittest.TestCase): loss_fn_torch = torch.nn.MSELoss() # Test case 1: Predictions and labels are equal - predictions_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device) - labels_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device) + predictions_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device) + labels_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device) loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device) - labels_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device) + predictions_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device) + labels_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) @@ -44,6 +44,75 @@ class TestNNModuleLoss(unittest.TestCase): self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + def test_cross_entropy_loss(self): + """ + Test the CrossEntropyLoss + """ + loss_fn_norch = norch.nn.CrossEntropyLoss() + loss_fn_torch = torch.nn.CrossEntropyLoss() + + # Test case 1: Single class, single sample + predictions_norch = norch.Tensor([2.0, 1.0, 0.1]).to(self.device) + labels_norch = norch.Tensor([0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([2.0, 1.0, 0.1]).to(self.device) + labels_torch = torch.tensor(0).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 2: Multiple classes, multiple samples + predictions_norch = norch.Tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device) + labels_norch = norch.Tensor([2, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device) + labels_torch = torch.tensor([2, 1]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 3: Edge case - all predictions are zero + predictions_norch = norch.Tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device) + labels_norch = norch.Tensor([1, 2]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device) + labels_torch = torch.tensor([1, 2]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Class probabilities instead of class index + predictions_norch = norch.Tensor([0.5, 0.2, 0.1]).to(self.device) + labels_norch = norch.Tensor([1., 0, 0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([0.5, 0.2, 0.1]).to(self.device) + labels_torch = torch.tensor([1., 0, 0]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Batched class probabilities instead of class index + predictions_norch = norch.Tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device) + labels_norch = norch.Tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + class TestNNModuleActivationFn(unittest.TestCase):