aparler/PyNorch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cross entropy loss

Browse source
This commit is contained in:
lucasdelimanogueira 2024-05-21 12:16:12 -03:00
parent a57298dfcb
commit d530f79a82
5 changed files with 168 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
norch/nn/__pycache__/loss.cpython-38.pyc
View file

Binary file not shown.

17
norch/nn/functional.py
View file

@ -1,4 +1,6 @@
import math
import norch
import numpy as np
def sigmoid(x):
return 1.0 / (1.0 + (math.e) ** (-x))
@ -15,4 +17,17 @@ def softmax(x, dim=None):
return exp_x / sum_exp_x
else:
sum_exp_x = exp_x.sum()
return exp_x / sum_exp_x
return exp_x / sum_exp_x
def one_hot_encode(x, num_classes):
one_hot = [[0] * num_classes for _ in range(x.numel)]
# Set the appropriate elements to 1
for i in range(x.numel):
target_idx = int(x.tensor.contents.data[i])
one_hot[i][target_idx] = 1
if x.numel < 2:
one_hot = one_hot[0]
return norch.Tensor(one_hot)

70
norch/nn/loss.py
View file

@ -1,4 +1,5 @@
from .module import Module
import norch
from abc import ABC
class Loss(Module, ABC):
@ -18,8 +19,69 @@ class MSELoss(Loss):
def __init__(self):
super().__init__()
def forward(self, predictions, labels):
assert labels.shape == predictions.shape, \
"Labels and predictions shape does not match: {} and {}".format(labels.shape, predictions.shape)
def forward(self, predictions, target):
assert target.shape == predictions.shape, \
"Labels and predictions shape does not match: {} and {}".format(target.shape, predictions.shape)
return ((predictions - labels) ** 2).sum() / predictions.numel
cost = ((predictions - target) ** 2).sum() / predictions.numel
return cost
class CrossEntropyLoss(Loss):
def __init__(self):
super().__init__()
def forward(self, input, target):
assert isinstance(input, norch.Tensor), \
"Cross entropy argument 'input' must be Tensor, not {}".format(type(input))
assert isinstance(target, norch.Tensor), \
"Cross entropy argument 'target' must be Tensor, not {}".format(type(target))
if input.ndim == 1:
if target.numel == 1:
num_classes = input.shape[0]
target = norch.one_hot_encode(target, num_classes)
logits = norch.softmax(input, dim=0)
cost = -(logits.log() * target).sum()
return cost
else:
# target -> class probabilities (one-hot encoded)
assert target.shape == input.shape, \
"Input and target shape does not match: {} and {}".format(input.shape, target.shape)
logits = norch.softmax(input, dim=0)
cost = -(logits.log() * target).sum()
return cost
elif input.ndim == 2:
# batched
if target.ndim == 1:
# target -> Ground truth class indices:
num_classes = input.shape[1]
target = norch.one_hot_encode(target, num_classes)
batch_size = input.shape[0]
logits = norch.softmax(input, dim=1)
cost = -(logits.log() * target).sum() / batch_size
return cost
else:
# target -> class probabilities (one-hot encoded)
assert target.shape == input.shape, \
"Input and target shape does not match: {} and {}".format(input.shape, target.shape)
batch_size = input.shape[0]
logits = norch.softmax(input, dim=1)
cost = -(logits.log() * target).sum() / batch_size
return cost

19
test.py
View file

@ -6,12 +6,19 @@ import norch.optim as optim
import random
random.seed(1)
torch_tensor = norch.Tensor([[[1, 5], [2, -1]], [[5, 2.], [2, 2]]], requires_grad=True)#.to(self.device)
b = norch.nn.functional.softmax(torch_tensor, dim=0)
soma = b.sum()
print(soma)
soma.backward()
print(torch_tensor.grad)
"""one_hot_target = norch.one_hot_encode(norch.Tensor([5]), num_classes=10)
print(one_hot_target)"""
logits = norch.Tensor([[2.0, 1.0, 0.1, 0.1], [2.0, 1.0, 0.1, 0.1], [2.0, 1.0, 0.1, 0.1]], requires_grad=True)
# One-hot encoded target with shape (batch_size, num_classes)
one_hot_target = norch.Tensor([0, 1, 1])
criterion = nn.CrossEntropyLoss()
loss = criterion(logits, one_hot_target)
print(loss)
"""a = norch.Tensor([[[4.186502456665039]]])
b = norch.Tensor([[[2.0, 2.0,],[-1.0, -1.0,]],[[1.0, 2.0,],[3.0, 3.0,]]])

77
tests/test_nn.py
View file

@ -21,13 +21,13 @@ class TestNNModuleLoss(unittest.TestCase):
loss_fn_torch = torch.nn.MSELoss()
# Test case 1: Predictions and labels are equal
predictions_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device)
labels_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device)
predictions_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device)
labels_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device)
loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch)
loss_torch_result = utils.to_torch(loss_norch).to(self.device)
predictions_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device)
labels_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device)
predictions_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device)
labels_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device)
loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch)
self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected))
@ -44,6 +44,75 @@ class TestNNModuleLoss(unittest.TestCase):
self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected))
def test_cross_entropy_loss(self):
"""
Test the CrossEntropyLoss
"""
loss_fn_norch = norch.nn.CrossEntropyLoss()
loss_fn_torch = torch.nn.CrossEntropyLoss()
# Test case 1: Single class, single sample
predictions_norch = norch.Tensor([2.0, 1.0, 0.1]).to(self.device)
labels_norch = norch.Tensor([0]).to(self.device)
loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch)
loss_torch_result = utils.to_torch(loss_norch).to(self.device)
predictions_torch = torch.tensor([2.0, 1.0, 0.1]).to(self.device)
labels_torch = torch.tensor(0).to(self.device)
loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch)
self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected))
# Test case 2: Multiple classes, multiple samples
predictions_norch = norch.Tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device)
labels_norch = norch.Tensor([2, 1]).to(self.device)
loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch)
loss_torch_result = utils.to_torch(loss_norch).to(self.device)
predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device)
labels_torch = torch.tensor([2, 1]).to(self.device)
loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch)
self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected))
# Test case 3: Edge case - all predictions are zero
predictions_norch = norch.Tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device)
labels_norch = norch.Tensor([1, 2]).to(self.device)
loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch)
loss_torch_result = utils.to_torch(loss_norch).to(self.device)
predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device)
labels_torch = torch.tensor([1, 2]).to(self.device)
loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch)
self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected))
# Test case 4: Class probabilities instead of class index
predictions_norch = norch.Tensor([0.5, 0.2, 0.1]).to(self.device)
labels_norch = norch.Tensor([1., 0, 0]).to(self.device)
loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch)
loss_torch_result = utils.to_torch(loss_norch).to(self.device)
predictions_torch = torch.tensor([0.5, 0.2, 0.1]).to(self.device)
labels_torch = torch.tensor([1., 0, 0]).to(self.device)
loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch)
self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected))
# Test case 4: Batched class probabilities instead of class index
predictions_norch = norch.Tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device)
labels_norch = norch.Tensor([[1., 0, 0], [0, 1, 0]]).to(self.device)
loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch)
loss_torch_result = utils.to_torch(loss_norch).to(self.device)
predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device)
labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]]).to(self.device)
loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch)
self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected))
class TestNNModuleActivationFn(unittest.TestCase):