diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 3d69ca0..3af7ab1 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/autograd/__pycache__/functions.cpython-38.pyc b/norch/autograd/__pycache__/functions.cpython-38.pyc index 3ce2caf..503e242 100644 Binary files a/norch/autograd/__pycache__/functions.cpython-38.pyc and b/norch/autograd/__pycache__/functions.cpython-38.pyc differ diff --git a/norch/autograd/functions.py b/norch/autograd/functions.py index 0761d0d..89c1a8d 100644 --- a/norch/autograd/functions.py +++ b/norch/autograd/functions.py @@ -97,6 +97,14 @@ class TransposeBackward: def backward(self, gradient): return [gradient.transpose(self.axis2, self.axis1)] +class TBackward: + def __init__(self, x): + self.input = [x] + + def backward(self, gradient): + return [gradient.T] + + class DivisionBackward: def __init__(self, x, y): self.input = [x, y] diff --git a/norch/tensor.py b/norch/tensor.py index 8231dca..bf4e4d4 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -103,7 +103,7 @@ class Tensor: return result_data - def reshape(self, new_shape, requires_grad=None): + def reshape(self, new_shape): new_shape_ctype = (ctypes.c_int * len(new_shape))(*new_shape) new_ndim_ctype = ctypes.c_int(len(new_shape)) @@ -119,8 +119,8 @@ class Tensor: result_data.device = self.device result_data.requires_grad = self.requires_grad - if requires_grad: - self.grad_fn = ReshapeBackward(self) + if result_data.requires_grad: + result_data.grad_fn = ReshapeBackward(self) return result_data @@ -584,6 +584,8 @@ class Tensor: result_data.device = self.device result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = TBackward(self) return result_data diff --git a/tests/test_autograd.py b/tests/test_autograd.py new file mode 100644 index 0000000..9a5b0d7 --- /dev/null +++ b/tests/test_autograd.py @@ -0,0 +1,308 @@ +import unittest +import norch +from norch import utils +import torch + +class TestTensorAutograd(unittest.TestCase): + + def test_addition(self): + """ + Test autograd from addition two tensors: tensor1 + tensor2 + """ + norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) + norch_result = (norch_tensor1 + norch_tensor2).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad) + + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) + torch_result = (torch_tensor1 + torch_tensor2).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + + def test_subtraction(self): + """ + Test autograd from subtraction two tensors: tensor1 - tensor2 + """ + norch_tensor1_sub = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_tensor2_sub = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) + norch_result_sub = (norch_tensor1_sub - norch_tensor2_sub).sum() + norch_result_sub.backward() + norch_tensor1_grad_sub = utils.to_torch(norch_tensor1_sub.grad) + norch_tensor2_grad_sub = utils.to_torch(norch_tensor2_sub.grad) + + torch_tensor1_sub = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_tensor2_sub = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) + torch_result_sub = (torch_tensor1_sub - torch_tensor2_sub).sum() + torch_result_sub.backward() + torch_tensor1_grad_sub = torch_tensor1_sub.grad + torch_tensor2_grad_sub = torch_tensor2_sub.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_sub, torch_tensor1_grad_sub)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_sub, torch_tensor2_grad_sub)) + + def test_division(self): + """ + Test autograd from dividing two tensors: tensor1 / tensor2 + """ + norch_tensor1_div = norch.Tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True) + norch_tensor2_div = norch.Tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True) + norch_result_div = (norch_tensor1_div / norch_tensor2_div).sum() + norch_result_div.backward() + norch_tensor1_grad_div = utils.to_torch(norch_tensor1_div.grad) + norch_tensor2_grad_div = utils.to_torch(norch_tensor2_div.grad) + + torch_tensor1_div = torch.tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True) + torch_tensor2_div = torch.tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True) + torch_result_div = (torch_tensor1_div / torch_tensor2_div).sum() + torch_result_div.backward() + torch_tensor1_grad_div = torch_tensor1_div.grad + torch_tensor2_grad_div = torch_tensor2_div.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_div, torch_tensor1_grad_div)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_div, torch_tensor2_grad_div)) + + + def test_tensor_division_scalar(self): + """ + Test autograd from dividing tensor by scalar: tensor / scalar + """ + norch_tensor_div_scalar = norch.Tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True) + scalar = 2 + norch_result_div_scalar = (norch_tensor_div_scalar / scalar).sum() + norch_result_div_scalar.backward() + norch_tensor_grad_div_scalar = utils.to_torch(norch_tensor_div_scalar.grad) + + torch_tensor_div_scalar = torch.tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True) + torch_result_div_scalar = (torch_tensor_div_scalar / scalar).sum() + torch_result_div_scalar.backward() + torch_tensor_grad_div_scalar = torch_tensor_div_scalar.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_div_scalar, torch_tensor_grad_div_scalar)) + + + def test_scalar_division_tensor(self): + """ + Test autograd from dividing scalar by tensor: scalar / tensor + """ + scalar = 2 + norch_tensor_scalar_div = norch.Tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_result_scalar_div = (scalar / norch_tensor_scalar_div).sum() + norch_result_scalar_div.backward() + norch_tensor_grad_scalar_div = utils.to_torch(norch_tensor_scalar_div.grad) + + torch_tensor_scalar_div = torch.tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_result_scalar_div = (scalar / torch_tensor_scalar_div).sum() + torch_result_scalar_div.backward() + torch_tensor_grad_scalar_div = torch_tensor_scalar_div.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_scalar_div, torch_tensor_grad_scalar_div)) + + + def test_power_scalar_tensor(self): + """ + Test autograd from scalar raised to tensor: scalar ** tensor + """ + scalar = 2 + norch_tensor_power_st = norch.Tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) + norch_result_power_st = (scalar ** norch_tensor_power_st).sum() + norch_result_power_st.backward() + norch_tensor_grad_power_st = utils.to_torch(norch_tensor_power_st.grad) + + torch_tensor_power_st = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) + torch_result_power_st = (scalar ** torch_tensor_power_st).sum() + torch_result_power_st.backward() + torch_tensor_grad_power_st = torch_tensor_power_st.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_power_st, torch_tensor_grad_power_st)) + + + def test_power_tensor_scalar(self): + """ + Test autograd from tensor raised to scalar: tensor ** scalar + """ + scalar = 2 + norch_tensor_power_ts = norch.Tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) + norch_result_power_ts = (norch_tensor_power_ts ** scalar).sum() + norch_result_power_ts.backward() + norch_tensor_grad_power_ts = utils.to_torch(norch_tensor_power_ts.grad) + + torch_tensor_power_ts = torch.tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) + torch_result_power_ts = (torch_tensor_power_ts ** scalar).sum() + torch_result_power_ts.backward() + torch_tensor_grad_power_ts = torch_tensor_power_ts.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_power_ts, torch_tensor_grad_power_ts)) + + def test_matmul(self): + """ + Test autograd from matrix multiplication: matmul(tensor1, tensor2) + """ + norch_tensor1_matmul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_tensor2_matmul = norch.Tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True) + norch_result_matmul = (norch_tensor1_matmul @ norch_tensor2_matmul).sum() + norch_result_matmul.backward() + norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad) + norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad) + + torch_tensor1_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_tensor2_matmul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True) + torch_result_matmul = (torch_tensor1_matmul @ torch_tensor2_matmul).sum() + torch_result_matmul.backward() + torch_tensor1_grad_matmul = torch_tensor1_matmul.grad + torch_tensor2_grad_matmul = torch_tensor2_matmul.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul)) + + + def test_elementwise_mul_scalar(self): + """ + Test autograd from elementwise multiplication with scalar: scalar * tensor + """ + scalar = 2 + norch_tensor_elemwise_mul_scalar = norch.Tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_result_elemwise_mul_scalar = (scalar * norch_tensor_elemwise_mul_scalar).sum() + norch_result_elemwise_mul_scalar.backward() + norch_tensor_grad_elemwise_mul_scalar = utils.to_torch(norch_tensor_elemwise_mul_scalar.grad) + + torch_tensor_elemwise_mul_scalar = torch.tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_result_elemwise_mul_scalar = (scalar * torch_tensor_elemwise_mul_scalar).sum() + torch_result_elemwise_mul_scalar.backward() + torch_tensor_grad_elemwise_mul_scalar = torch_tensor_elemwise_mul_scalar.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_elemwise_mul_scalar, torch_tensor_grad_elemwise_mul_scalar)) + + + def test_elementwise_mul_tensor(self): + """ + Test autograd from elementwise multiplication between two tensors: tensor1 * tensor2 + """ + norch_tensor1_elemwise_mul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_tensor2_elemwise_mul = norch.Tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True) + norch_result_elemwise_mul = (norch_tensor1_elemwise_mul * norch_tensor2_elemwise_mul).sum() + norch_result_elemwise_mul.backward() + norch_tensor1_grad_elemwise_mul = utils.to_torch(norch_tensor1_elemwise_mul.grad) + norch_tensor2_grad_elemwise_mul = utils.to_torch(norch_tensor2_elemwise_mul.grad) + + torch_tensor1_elemwise_mul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_tensor2_elemwise_mul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True) + torch_result_elemwise_mul = (torch_tensor1_elemwise_mul * torch_tensor2_elemwise_mul).sum() + torch_result_elemwise_mul.backward() + torch_tensor1_grad_elemwise_mul = torch_tensor1_elemwise_mul.grad + torch_tensor2_grad_elemwise_mul = torch_tensor2_elemwise_mul.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_elemwise_mul, torch_tensor1_grad_elemwise_mul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_elemwise_mul, torch_tensor2_grad_elemwise_mul)) + + def test_reshape(self): + """ + Test autograd from reshaping a tensor: tensor.reshape(shape) + """ + norch_tensor_reshape = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + new_shape = [2, 4] + norch_result_reshape = norch_tensor_reshape.reshape(new_shape).sum() + norch_result_reshape.backward() + norch_tensor_grad_reshape = utils.to_torch(norch_tensor_reshape.grad) + + torch_tensor_reshape = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_result_reshape = torch_tensor_reshape.reshape(new_shape).sum() + torch_result_reshape.backward() + torch_tensor_grad_reshape = torch_tensor_reshape.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape, torch_tensor_grad_reshape)) + + + def test_transpose_axes(self): + """ + Test autograd from transposing a tensor with specific axes: tensor.transpose(axis1, axis2) + """ + norch_tensor_transpose = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + axis1, axis2 = 0, 2 + norch_result_transpose = norch_tensor_transpose.transpose(axis1, axis2).sum() + norch_result_transpose.backward() + norch_tensor_grad_transpose = utils.to_torch(norch_tensor_transpose.grad) + + torch_tensor_transpose = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_result_transpose = torch_tensor_transpose.transpose(axis1, axis2).sum() + torch_result_transpose.backward() + torch_tensor_grad_transpose = torch_tensor_transpose.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose, torch_tensor_grad_transpose)) + + + def test_T(self): + """ + Test autograd from transposing a tensor using .T attribute + """ + norch_tensor_T = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_result_T = norch_tensor_T.T.sum() + norch_result_T.backward() + norch_tensor_grad_T = utils.to_torch(norch_tensor_T.grad) + + torch_tensor_T = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + torch_result_T = torch_tensor_T.T.sum() + torch_result_T.backward() + torch_tensor_grad_T = torch_tensor_T.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_T, torch_tensor_grad_T)) + +def test_reshape_then_matmul(self): + """ + Test autograd from reshaping a tensor then performing matrix multiplication: matmul(tensor1.reshape(shape), tensor2) + """ + norch_tensor_reshape_matmul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + new_shape = (2, 4) + norch_result_reshape_matmul = norch.matmul(norch_tensor_reshape_matmul.reshape(new_shape), norch_tensor_reshape_matmul).sum() + norch_result_reshape_matmul.backward() + norch_tensor_grad_reshape_matmul = utils.to_torch(norch_tensor_reshape_matmul.grad) + + torch_tensor_reshape_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], dtype=torch.float32, requires_grad=True) + torch_result_reshape_matmul = torch.matmul(torch_tensor_reshape_matmul.reshape(new_shape), torch_tensor_reshape_matmul).sum() + torch_result_reshape_matmul.backward() + torch_tensor_grad_reshape_matmul = torch_tensor_reshape_matmul.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul, torch_tensor_grad_reshape_matmul)) + + +def test_T_then_matmul(self): + """ + Test autograd from transposing a tensor then performing matrix multiplication: matmul(tensor.T, tensor) + """ + norch_tensor_T_matmul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + norch_result_T_matmul = norch.matmul(norch_tensor_T_matmul.T, norch_tensor_T_matmul).sum() + norch_result_T_matmul.backward() + norch_tensor_grad_T_matmul = utils.to_torch(norch_tensor_T_matmul.grad) + + torch_tensor_T_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], dtype=torch.float32, requires_grad=True) + torch_result_T_matmul = torch.matmul(torch_tensor_T_matmul.T, torch_tensor_T_matmul).sum() + torch_result_T_matmul.backward() + torch_tensor_grad_T_matmul = torch_tensor_T_matmul.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_T_matmul, torch_tensor_grad_T_matmul)) + + +def test_transpose_axes_then_matmul(self): + """ + Test autograd from transposing a tensor with specific axes then performing matrix multiplication: matmul(tensor.transpose(axis1, axis2), tensor) + """ + norch_tensor_transpose_matmul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) + axis1, axis2 = 0, 2 + norch_result_transpose_matmul = norch.matmul(norch_tensor_transpose_matmul.transpose(axis1, axis2), norch_tensor_transpose_matmul).sum() + norch_result_transpose_matmul.backward() + norch_tensor_grad_transpose_matmul = utils.to_torch(norch_tensor_transpose_matmul.grad) + + torch_tensor_transpose_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], dtype=torch.float32, requires_grad=True) + torch_result_transpose_matmul = torch.matmul(torch_tensor_transpose_matmul.transpose(axis1, axis2), torch_tensor_transpose_matmul).sum() + torch_result_transpose_matmul.backward() + torch_tensor_grad_transpose_matmul = torch_tensor_transpose_matmul.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose_matmul, torch_tensor_grad_transpose_matmul)) + diff --git a/tests/test_operations.py b/tests/test_operations.py index 5d46c62..65e399b 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -181,6 +181,57 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_reshape_then_matmul(self): + """ + Test reshaping a tensor followed by matrix multiplication: (tensor.reshape(shape) @ other_tensor) + """ + norch_tensor = norch.Tensor([[1, 2], [3, -4], [5, 6], [7, 8]]) + new_shape = [2, 4] + norch_reshaped = norch_tensor.reshape(new_shape) + + norch_result = norch_reshaped @ norch_tensor + torch_result = utils.to_torch(norch_result) + + torch_tensor = torch.tensor([[1, 2], [3, -4], [5, 6], [7, 8]]) + torch_expected = torch_tensor.reshape(new_shape) @ torch_tensor + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_transpose_then_matmul(self): + """ + Test transposing a tensor followed by matrix multiplication: (tensor.transpose(dim1, dim2) @ other_tensor) + """ + norch_tensor = norch.Tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + dim1, dim2 = 0, 2 + norch_result = norch_tensor.transpose(dim1, dim2) @ norch_tensor + torch_result = utils.to_torch(norch_result) + + torch_tensor = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + torch_expected = torch_tensor.transpose(dim1, dim2) @ torch_tensor + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_add_div_matmul_then_reshape(self): + """ + Test a combination of operations: (tensor.sum() + other_tensor) / scalar @ another_tensor followed by reshape + """ + norch_tensor1 = norch.Tensor([[[1., 2], [3, -4]], [[5, 6], [7, 8]]]) + norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]) + scalar = 2 + new_shape = [2, 4] + norch_result = ((norch_tensor1 + norch_tensor2) / scalar) @ norch_tensor1 + norch_result = norch_result.reshape(new_shape) + torch_result = utils.to_torch(norch_result) + + torch_tensor1 = torch.tensor([[[1., 2], [3, -4]], [[5, 6], [7, 8]]]) + torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]) + torch_expected = ((torch_tensor1 + torch_tensor2) / scalar) @ torch_tensor1 + torch_expected = torch_expected.reshape(new_shape) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + if __name__ == '__main__': unittest.main()