diff --git a/build/tensor.o b/build/tensor.o index bcc09bc..d12bc52 100644 Binary files a/build/tensor.o and b/build/tensor.o differ diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 0e09a81..1192401 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/csrc/tensor.cpp b/norch/csrc/tensor.cpp index 570aefa..a340c73 100644 --- a/norch/csrc/tensor.cpp +++ b/norch/csrc/tensor.cpp @@ -225,14 +225,20 @@ extern "C" { if (keepdim) { shape = (int*) malloc((tensor->ndim) * sizeof(int)); for (int i = 0; i < tensor->ndim; i++) { - shape[i] = tensor->shape[i]; + if (axis == -1) { + shape[i] = 1; + } else { + shape[i] = tensor->shape[i]; + } } shape[axis] = 1; ndim = tensor->ndim; - + Tensor* new_tensor = create_tensor(result_data, shape, ndim, device); + return reshape_tensor(new_tensor, shape, ndim); } return create_tensor(result_data, shape, ndim, device); + } } @@ -289,11 +295,16 @@ extern "C" { if (keepdim) { shape = (int*) malloc((tensor->ndim) * sizeof(int)); for (int i = 0; i < tensor->ndim; i++) { - shape[i] = tensor->shape[i]; + if (axis == -1) { + shape[i] = 1; + } else { + shape[i] = tensor->shape[i]; + } } shape[axis] = 1; ndim = tensor->ndim; - + Tensor* new_tensor = create_tensor(result_data, shape, ndim, device); + return reshape_tensor(new_tensor, shape, ndim); } return create_tensor(result_data, shape, ndim, device); @@ -353,11 +364,16 @@ extern "C" { if (keepdim) { shape = (int*) malloc((tensor->ndim) * sizeof(int)); for (int i = 0; i < tensor->ndim; i++) { - shape[i] = tensor->shape[i]; + if (axis == -1) { + shape[i] = 1; + } else { + shape[i] = tensor->shape[i]; + } } shape[axis] = 1; ndim = tensor->ndim; - + Tensor* new_tensor = create_tensor(result_data, shape, ndim, device); + return reshape_tensor(new_tensor, shape, ndim); } return create_tensor(result_data, shape, ndim, device); diff --git a/norch/libtensor.so b/norch/libtensor.so index d1b1cd0..5d6e518 100755 Binary files a/norch/libtensor.so and b/norch/libtensor.so differ diff --git a/norch/nn/__pycache__/activation.cpython-38.pyc b/norch/nn/__pycache__/activation.cpython-38.pyc index ac065b4..0a2693b 100644 Binary files a/norch/nn/__pycache__/activation.cpython-38.pyc and b/norch/nn/__pycache__/activation.cpython-38.pyc differ diff --git a/norch/nn/activation.py b/norch/nn/activation.py index 5f93936..a1361f6 100644 --- a/norch/nn/activation.py +++ b/norch/nn/activation.py @@ -18,4 +18,13 @@ class Sigmoid(Activation): super().__init__() def forward(self, x): - return F.sigmoid(x) \ No newline at end of file + return F.sigmoid(x) + +class Softmax(Activation): + def __init__(self, dim): + super(Softmax, self).__init__() + + self.dim = dim + + def forward(self, x): + return F.softmax(x, self.dim) \ No newline at end of file diff --git a/norch/nn/functional.py b/norch/nn/functional.py index b865379..9b5c3ec 100644 --- a/norch/nn/functional.py +++ b/norch/nn/functional.py @@ -3,3 +3,16 @@ import math def sigmoid(x): return 1.0 / (1.0 + (math.e) ** (-x)) +def softmax(x, dim=None): + e = math.e ** x + s = e.sum(axis=dim, keepdim=True) + #l = s.log() + print('x', x) + print('\n\n') + #print('log', l) + + print('\n\n') + #print('x-log', x - l) + + + #return math.e ** (x - sum.log()) \ No newline at end of file diff --git a/norch/tensor.py b/norch/tensor.py index 7e60d7f..52a5558 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -697,7 +697,7 @@ class Tensor: elif self.ndim == other.ndim - 1: self = self.reshape([1] + self.shape) - + # Call equal_broadcasted_tensor if broadcasting is needed Tensor._C.equal_broadcasted_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] Tensor._C.equal_broadcasted_tensor.restype = ctypes.POINTER(CTensor) @@ -748,7 +748,9 @@ class Tensor: return result_data - def sum(self, axis=-1, keepdim=False): + def sum(self, axis=None, keepdim=False): + if axis == None: + axis = -1 Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.sum_tensor.restype = ctypes.POINTER(CTensor) @@ -783,10 +785,13 @@ class Tensor: return result_data - def max(self, axis=-1, keepdim=False): + def max(self, axis=None, keepdim=False): + if axis == None: + axis = -1 Tensor._C.max_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.max_tensor.restype = ctypes.POINTER(CTensor) + print(axis, keepdim) result_tensor_ptr = Tensor._C.max_tensor(self.tensor, axis, keepdim) result_data = Tensor() @@ -818,7 +823,9 @@ class Tensor: return result_data - def min(self, axis=-1, keepdim=False): + def min(self, axis=None, keepdim=False): + if axis == None: + axis = -1 Tensor._C.min_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.min_tensor.restype = ctypes.POINTER(CTensor) diff --git a/test.py b/test.py index 2306f6a..f6125ae 100644 --- a/test.py +++ b/test.py @@ -6,11 +6,16 @@ import norch.optim as optim import random random.seed(1) -torch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True)#.to(self.device) -torch_tensor2 = norch.Tensor([[[10.0,], [-4.0,]],[[6.0,], [8.0,]]]) -torch_tensor3 = torch_tensor.max(axis=2, keepdim=True) -print(torch_tensor3) -print(torch_tensor.equal(torch_tensor3)) +torch_tensor = norch.Tensor([[[2, 2], [-1, -1]], [[1., 2], [3, 3]]], requires_grad=True)#.to(self.device) +b = norch.nn.functional.softmax(torch_tensor) + + +"""a = norch.Tensor([[[4.186502456665039]]]) +b = norch.Tensor([[[2.0, 2.0,],[-1.0, -1.0,]],[[1.0, 2.0,],[3.0, 3.0,]]]) + +print(b) +print(a.shape) +print(b-a)""" """print(torch_tensor.shape) print('\n\n') diff --git a/tests/test_nn.py b/tests/test_nn.py index e790fa9..daaac21 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -60,11 +60,11 @@ class TestNNModuleActivationFn(unittest.TestCase): sigmoid_fn_torch = torch.nn.Sigmoid() # Test case 1: Positive input - x = norch.Tensor([1, 2, 3]).to(self.device) + x = norch.Tensor([[1, 2, 3]]).to(self.device) sigmoid_norch = sigmoid_fn_norch.forward(x) sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) - x = torch.tensor([1, 2, 3]).to(self.device) + x = torch.tensor([[1, 2, 3]]).to(self.device) sigmoid_torch_expected = sigmoid_fn_torch.forward(x) self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) @@ -87,4 +87,41 @@ class TestNNModuleActivationFn(unittest.TestCase): x = torch.tensor([0, 0, 0]).to(self.device) sigmoid_torch_expected = sigmoid_fn_torch.forward(x) - self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) \ No newline at end of file + self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) + + def test_softmax_activation(self): + """ + Test Softmax activation function + """ + + # Test different axes + axes = [None, 0, 1, -1] + + # Define the input tensors for different test cases + test_cases = [ + (norch.Tensor([[1., 2, 3], [4, 5, 6]]), torch.tensor([[1., 2, 3], [4, 5, 6]])), + (norch.Tensor([[1., -1, 0], [2, -2, 0]]), torch.tensor([[1., -1, 0], [2, -2, 0]])), + (norch.Tensor([[0., 0, 0], [0, 0, 0]]), torch.tensor([[0., 0, 0], [0, 0, 0]])) + ] + + for dim in axes: + softmax_fn_norch = norch.nn.Softmax(dim=dim) + softmax_fn_torch = torch.nn.Softmax(dim=dim) + + for norch_input, torch_input in test_cases: + # Move tensors to the correct device + norch_input = norch_input.to(self.device) + torch_input = torch_input.to(self.device) + + # Forward pass using norch + softmax_norch = softmax_fn_norch.forward(norch_input) + softmax_torch_result = utils.to_torch(softmax_norch).to(self.device) + + # Forward pass using torch + softmax_torch_expected = softmax_fn_torch.forward(torch_input) + + # Compare the results + print(softmax_torch_result) + print(softmax_torch_expected) + self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected)) +