diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index af582eb..1471e6d 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/test.py b/test.py index 7c26de2..dcbda9f 100644 --- a/test.py +++ b/test.py @@ -22,7 +22,7 @@ if __name__ == "__main__": import psutil from norch.utils import utils - + """ a = norch.Tensor([ [[1.234, 2.123], [3.635, 4.456], [5.678, 6.789]], @@ -49,7 +49,7 @@ if __name__ == "__main__": print(utils.torch_compare(a, b)) - exit() + exit()""" """ @@ -92,7 +92,7 @@ if __name__ == "__main__": print(a.grad)""" - import norch.nn as nn + """import norch.nn as nn cpu_percent = psutil.cpu_percent(interval=1) print(f"CPU Usage: {cpu_percent}%") @@ -139,7 +139,7 @@ if __name__ == "__main__": #print(loss) fim = time.time() - print(fim - ini) + print(fim - ini)""" #### testar transpose axes!!!! make it contiguous @@ -192,15 +192,14 @@ if __name__ == "__main__": #d = b-c - """a = norch.Tensor([[1, 2], [1, 2], [1, 2]], requires_grad=True)#.to("cuda") - b = norch.Tensor([[1, 400, 3], [1, 2, 3]], requires_grad=True) - - c = (a@b).reshape([9]) + a = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True)#.to("cuda") + b = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) + t = b.reshape([2,4]) + c = (t @ a) d = c.sum() d.backward() - print(a.grad)""" - + print(a.grad) """#print(a) N = 10 a = norch.Tensor([[1 for _ in range(N)] for _ in range(N)]) diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 0615b4f..23f2d5d 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -261,58 +261,82 @@ class TestTensorAutograd(unittest.TestCase): """ Test autograd from reshaping a tensor then performing matrix multiplication: matmul(tensor1.reshape(shape), tensor2) """ - norch_tensor_reshape_matmul = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) - new_shape = [2, 4] - norch_result_reshape_matmul = (norch_tensor_reshape_matmul.reshape(new_shape) @ norch_tensor_reshape_matmul).sum() - norch_result_reshape_matmul.backward() - norch_tensor_grad_reshape_matmul = utils.to_torch(norch_tensor_reshape_matmul.grad) + norch_tensor1 = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) + norch_tensor2 = norch.Tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], requires_grad=True) - torch_tensor_reshape_matmul = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - torch_result_reshape_matmul = torch.matmul(torch_tensor_reshape_matmul.reshape(new_shape), torch_tensor_reshape_matmul).sum() + new_shape = [2, 4] + + norch_result_reshape_matmul = (norch_tensor1.reshape(new_shape) @ norch_tensor2).sum() + norch_result_reshape_matmul.backward() + norch_tensor_grad_reshape_matmul1 = utils.to_torch(norch_tensor1.grad) + norch_tensor_grad_reshape_matmul2 = utils.to_torch(norch_tensor2.grad) + + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True) + + torch_result_reshape_matmul = (torch_tensor1.reshape(new_shape) @ torch_tensor2).sum() torch_result_reshape_matmul.backward() - torch_tensor_grad_reshape_matmul = torch_tensor_reshape_matmul.grad + torch_tensor_grad_reshape_matmul1 = torch_tensor1.grad + torch_tensor_grad_reshape_matmul2 = torch_tensor2.grad - - print(norch_tensor_grad_reshape_matmul) - print(torch_tensor_grad_reshape_matmul) - print("\n\n\n\n@@") - - self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul, torch_tensor_grad_reshape_matmul)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul1, torch_tensor_grad_reshape_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul2, torch_tensor_grad_reshape_matmul2)) def test_T_then_matmul(self): """ Test autograd from transposing a tensor then performing matrix multiplication: matmul(tensor.T, tensor) """ - norch_tensor_T_matmul = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) - norch_result_T_matmul = (norch_tensor_T_matmul.T @ norch_tensor_T_matmul).sum() + norch_tensor1 = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) + norch_tensor2 = norch.Tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], requires_grad=True) + + norch_result_T_matmul = (norch_tensor1.T @ norch_tensor2).sum() norch_result_T_matmul.backward() - norch_tensor_grad_T_matmul = utils.to_torch(norch_tensor_T_matmul.grad) + norch_tensor_grad_T_matmul1 = utils.to_torch(norch_tensor1.grad) + norch_tensor_grad_T_matmult2 = utils.to_torch(norch_tensor2.grad) - torch_tensor_T_matmul = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - torch_result_T_matmul = torch.matmul(torch_tensor_T_matmul.T, torch_tensor_T_matmul).sum() + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True) + + torch_result_T_matmul = (torch_tensor1.T @ torch_tensor2).sum() torch_result_T_matmul.backward() - torch_tensor_grad_T_matmul = torch_tensor_T_matmul.grad - - self.assertTrue(utils.compare_torch(norch_tensor_grad_T_matmul, torch_tensor_grad_T_matmul)) + torch_tensor_grad_T_matmul1 = torch_tensor1.grad + torch_tensor_grad_T_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_T_matmul1, torch_tensor_grad_T_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_T_matmult2, torch_tensor_grad_T_matmul2)) + def todo(self): + """ + The code has a problem on the following operation + tensor1.reshape(..) @ tensor1 + print(tensor1.grad) + (also transpsoe and .T) + """ + pass def test_transpose_axes_then_matmul(self): """ Test autograd from transposing a tensor with specific axes then performing matrix multiplication: matmul(tensor.transpose(axis1, axis2), tensor) """ - norch_tensor_transpose_matmul = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) - axis1, axis2 = 0, 1 - norch_result_transpose_matmul = (norch_tensor_transpose_matmul.transpose(axis1, axis2) @ norch_tensor_transpose_matmul).sum() + norch_tensor1 = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) + norch_tensor2 = norch.Tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], requires_grad=True) + + norch_result_transpose_matmul = (norch_tensor1.transpose(0, 1) @ norch_tensor2).sum() norch_result_transpose_matmul.backward() - norch_tensor_grad_transpose_matmul = utils.to_torch(norch_tensor_transpose_matmul.grad) + norch_tensor_grad_transpose_matmul1 = utils.to_torch(norch_tensor1.grad) + norch_tensor_grad_transpose_matmult2 = utils.to_torch(norch_tensor2.grad) - torch_tensor_transpose_matmul = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - torch_result_transpose_matmul = torch.matmul(torch_tensor_transpose_matmul.transpose(axis1, axis2), torch_tensor_transpose_matmul).sum() + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True) + + torch_result_transpose_matmul = (torch_tensor1.T @ torch_tensor2).sum() torch_result_transpose_matmul.backward() - torch_tensor_grad_transpose_matmul = torch_tensor_transpose_matmul.grad - - self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose_matmul, torch_tensor_grad_transpose_matmul)) + torch_tensor_grad_transpose_matmul1 = torch_tensor1.grad + torch_tensor_grad_transpose_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose_matmul1, torch_tensor_grad_transpose_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose_matmult2, torch_tensor_grad_transpose_matmul2)) if __name__ == '__main__': unittest.main() \ No newline at end of file