Merge pull request #39 from lucasdelimanogueira/tmp

Tmp

norch/autograd/functions.py

@@ -97,6 +97,14 @@ class TransposeBackward:
     def backward(self, gradient):
         return [gradient.transpose(self.axis2, self.axis1)]
     
+class TBackward:
+    def __init__(self, x):
+        self.input = [x]
+
+    def backward(self, gradient):
+        return [gradient.T]
+
+    
 class DivisionBackward:
     def __init__(self, x, y):
         self.input = [x, y]

norch/tensor.py

@@ -103,7 +103,7 @@ class Tensor:
         
         return result_data
     
-    def reshape(self, new_shape, requires_grad=None):
+    def reshape(self, new_shape):
 
         new_shape_ctype = (ctypes.c_int * len(new_shape))(*new_shape)
         new_ndim_ctype = ctypes.c_int(len(new_shape))
@@ -119,8 +119,8 @@ class Tensor:
         result_data.device = self.device
 
         result_data.requires_grad = self.requires_grad
-        if requires_grad:
-            self.grad_fn = ReshapeBackward(self)
+        if result_data.requires_grad:
+            result_data.grad_fn = ReshapeBackward(self)
 
         return result_data
     
@@ -584,6 +584,8 @@ class Tensor:
         result_data.device = self.device
         
         result_data.requires_grad = self.requires_grad
+        if result_data.requires_grad:
+            result_data.grad_fn = TBackward(self)
 
         return result_data
     
@@ -591,4 +593,4 @@ class Tensor:
         self.grad = None
         self.grad_fn = None
 
-        return self
+        return self        

norch/tests.ipynb

@@ -1,98 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[0, 1, 2, 0, 2, 2] [2, 3, 3]\n",
-      "Tensor shape: [2, 3, 3]\n",
-      "Tensor created successfully\n",
-      "Tensor information:\n",
-      "Number of dimensions: 3\n",
-      "Shape: [2, 3, 3]\n",
-      "Data:\n",
-      "[0.00, 1.00, 2.00, 0.00, 2.00, 2.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]\n"
-     ]
-    }
-   ],
-   "source": [
-    "from tensor import Tensor\n",
-    "\n",
-    "data = [[0, 1, 2], [0, 2, 2], [0, 1, 2]]\n",
-    "tensor = Tensor(data)\n",
-    "print(\"Tensor shape:\", tensor.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Tensor created successfully\n",
-      "Tensor information:\n",
-      "Number of dimensions: 2\n",
-      "Shape: [3, 3]\n",
-      "Data:\n",
-      "[0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90]\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "0.10000000149011612 0.20000000298023224 0.30000001192092896 \n",
-       "0.4000000059604645 0.5 0.6000000238418579 \n",
-       "0.699999988079071 0.800000011920929 0.8999999761581421"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "\n",
-    "data = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
-    "shape = [3, 3]\n",
-    "tensor = Tensor(data, shape)\n",
-    "tensor\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

norch/utils/__init__.py

@@ -0,0 +1 @@
+from .utils import *

norch/utils/utils.py

@@ -1,5 +1,5 @@
 import random
-import numpy as np
+import torch
 
 
 def generate_random_list(shape):
@@ -14,4 +14,27 @@ def generate_random_list(shape):
             return [random.uniform(-1, 1) for _ in range(shape[0])]
         else:
             return [generate_random_list(inner_shape) for _ in range(shape[0])]
-        
+        
+
+def to_torch(custom_tensor):
+    shape = custom_tensor.shape
+    pytorch_tensor = torch.zeros(shape)
+    
+    def _iterate_indices(shape):
+        if len(shape) == 0:
+            yield ()
+        else:
+            for index in range(shape[0]):
+                for sub_indices in _iterate_indices(shape[1:]):
+                    yield (index,) + sub_indices
+    
+    # Iterate over all elements using the custom tensor's __getitem__ method
+    for indices in _iterate_indices(shape):
+        value = custom_tensor[indices]
+        pytorch_tensor[tuple(indices)] = value
+    
+    return pytorch_tensor
+
+def compare_torch(tensor1, tensor2, epsilon=1e-5):
+    diff = torch.abs(tensor1 - tensor2)    
+    return torch.all(diff < epsilon)

test.py

@@ -20,8 +20,11 @@ if __name__ == "__main__":
     import random
     import numpy as np
     import psutil
+    from norch.utils import utils
 
-    """a = norch.Tensor([
+
+
+    a = norch.Tensor([
         [[1.234, 2.123], [3.635, 4.456], [5.678, 6.789]],
         [[7.890, 8.901], [9.012, 1.234], [2.345, 3.456]],
         [[4.567, 5.678], [6.789, 7.890], [8.901, 9.012]],
@@ -29,6 +32,27 @@ if __name__ == "__main__":
         [[7.890, 8.901], [9.012, 1.234], [2.345, 3.456]]
     ], requires_grad=True)
 
+    print(a)
+    print(a[0, 2,0])
+    
+    a = utils.to_torch(a)
+
+    import torch
+
+    b = torch.tensor([
+        [[1.234, 2.123], [3.635, 4.456], [5.678, 6.789]],
+        [[7.890, 8.901], [9.012, 1.234], [2.345, 3.456]],
+        [[4.567, 5.678], [6.789, 7.890], [8.901, 9.012]],
+        [[1.234, 2.345], [3.456, 4.567], [5.678, 6.789]],
+        [[7.890, 8.901], [9.012, 1.234], [2.345, 3.456]]
+    ])
+
+    print(utils.torch_compare(a, b))
+
+    exit()
+
+    """
+
     b = norch.Tensor([[
         [1.234, 2.123, 1.5],
         [5.678, 6.789, 1.293],

tests/__init__.py

@@ -0,0 +1 @@
+from .test_operations import *

tests/test_autograd.py

@@ -0,0 +1,318 @@
+import unittest
+import norch
+from norch import utils
+import torch
+import os
+
+class TestTensorAutograd(unittest.TestCase):
+    def setUp(self):
+        self.device = os.environ.get('device', 'cpu')
+        
+    def test_addition(self):
+        """
+        Test autograd from addition two tensors: tensor1 + tensor2
+        """
+        norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True)
+        norch_result = (norch_tensor1 + norch_tensor2).sum()
+        norch_result.backward()
+        norch_tensor1_grad = utils.to_torch(norch_tensor1.grad)
+        norch_tensor2_grad = utils.to_torch(norch_tensor2.grad)
+
+        torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True)
+        torch_result = (torch_tensor1 + torch_tensor2).sum()
+        torch_result.backward()
+        torch_tensor1_grad = torch_tensor1.grad
+        torch_tensor2_grad = torch_tensor2.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad))
+        self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad))
+
+    
+    def test_subtraction(self):
+        """
+        Test autograd from subtraction two tensors: tensor1 - tensor2
+        """
+        norch_tensor1_sub = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        norch_tensor2_sub = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True)
+        norch_result_sub = (norch_tensor1_sub - norch_tensor2_sub).sum()
+        norch_result_sub.backward()
+        norch_tensor1_grad_sub = utils.to_torch(norch_tensor1_sub.grad)
+        norch_tensor2_grad_sub = utils.to_torch(norch_tensor2_sub.grad)
+
+        torch_tensor1_sub = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_tensor2_sub = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True)
+        torch_result_sub = (torch_tensor1_sub - torch_tensor2_sub).sum()
+        torch_result_sub.backward()
+        torch_tensor1_grad_sub = torch_tensor1_sub.grad
+        torch_tensor2_grad_sub = torch_tensor2_sub.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor1_grad_sub, torch_tensor1_grad_sub))
+        self.assertTrue(utils.compare_torch(norch_tensor2_grad_sub, torch_tensor2_grad_sub))
+        
+    def test_division(self):
+        """
+        Test autograd from dividing two tensors: tensor1 / tensor2
+        """
+        norch_tensor1_div = norch.Tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True)
+        norch_tensor2_div = norch.Tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True)
+        norch_result_div = (norch_tensor1_div / norch_tensor2_div).sum()
+        norch_result_div.backward()
+        norch_tensor1_grad_div = utils.to_torch(norch_tensor1_div.grad)
+        norch_tensor2_grad_div = utils.to_torch(norch_tensor2_div.grad)
+
+        torch_tensor1_div = torch.tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True)
+        torch_tensor2_div = torch.tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True)
+        torch_result_div = (torch_tensor1_div / torch_tensor2_div).sum()
+        torch_result_div.backward()
+        torch_tensor1_grad_div = torch_tensor1_div.grad
+        torch_tensor2_grad_div = torch_tensor2_div.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor1_grad_div, torch_tensor1_grad_div))
+        self.assertTrue(utils.compare_torch(norch_tensor2_grad_div, torch_tensor2_grad_div))
+    
+    
+    def test_tensor_division_scalar(self):
+        """
+        Test autograd from dividing tensor by scalar: tensor / scalar
+        """
+        norch_tensor_div_scalar = norch.Tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True)
+        scalar = 2
+        norch_result_div_scalar = (norch_tensor_div_scalar / scalar).sum()
+        norch_result_div_scalar.backward()
+        norch_tensor_grad_div_scalar = utils.to_torch(norch_tensor_div_scalar.grad)
+
+        torch_tensor_div_scalar = torch.tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True)
+        torch_result_div_scalar = (torch_tensor_div_scalar / scalar).sum()
+        torch_result_div_scalar.backward()
+        torch_tensor_grad_div_scalar = torch_tensor_div_scalar.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_div_scalar, torch_tensor_grad_div_scalar))
+    
+    
+    def test_scalar_division_tensor(self):
+        """
+        Test autograd from dividing scalar by tensor: scalar / tensor
+        """
+        scalar = 2
+        norch_tensor_scalar_div = norch.Tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True)
+        norch_result_scalar_div = (scalar / norch_tensor_scalar_div).sum()
+        norch_result_scalar_div.backward()
+        norch_tensor_grad_scalar_div = utils.to_torch(norch_tensor_scalar_div.grad)
+
+        torch_tensor_scalar_div = torch.tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_result_scalar_div = (scalar / torch_tensor_scalar_div).sum()
+        torch_result_scalar_div.backward()
+        torch_tensor_grad_scalar_div = torch_tensor_scalar_div.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_scalar_div, torch_tensor_grad_scalar_div))
+    
+    
+    def test_power_scalar_tensor(self):
+        """
+        Test autograd from scalar raised to tensor: scalar ** tensor
+        """
+        scalar = 2
+        norch_tensor_power_st = norch.Tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True)
+        norch_result_power_st = (scalar ** norch_tensor_power_st).sum()
+        norch_result_power_st.backward()
+        norch_tensor_grad_power_st = utils.to_torch(norch_tensor_power_st.grad)
+
+        torch_tensor_power_st = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True)
+        torch_result_power_st = (scalar ** torch_tensor_power_st).sum()
+        torch_result_power_st.backward()
+        torch_tensor_grad_power_st = torch_tensor_power_st.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_power_st, torch_tensor_grad_power_st))
+    
+    
+    def test_power_tensor_scalar(self):
+        """
+        Test autograd from tensor raised to scalar: tensor ** scalar
+        """
+        scalar = 2
+        norch_tensor_power_ts = norch.Tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True)
+        norch_result_power_ts = (norch_tensor_power_ts ** scalar).sum()
+        norch_result_power_ts.backward()
+        norch_tensor_grad_power_ts = utils.to_torch(norch_tensor_power_ts.grad)
+
+        torch_tensor_power_ts = torch.tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True)
+        torch_result_power_ts = (torch_tensor_power_ts ** scalar).sum()
+        torch_result_power_ts.backward()
+        torch_tensor_grad_power_ts = torch_tensor_power_ts.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_power_ts, torch_tensor_grad_power_ts))
+
+    def test_matmul(self):
+        """
+        Test autograd from matrix multiplication: matmul(tensor1, tensor2)
+        """
+        norch_tensor1_matmul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        norch_tensor2_matmul = norch.Tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True)
+        norch_result_matmul = (norch_tensor1_matmul @ norch_tensor2_matmul).sum()
+        norch_result_matmul.backward()
+        norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad)
+        norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad)
+
+        torch_tensor1_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_tensor2_matmul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True)
+        torch_result_matmul = (torch_tensor1_matmul @ torch_tensor2_matmul).sum()
+        torch_result_matmul.backward()
+        torch_tensor1_grad_matmul = torch_tensor1_matmul.grad
+        torch_tensor2_grad_matmul = torch_tensor2_matmul.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul))
+        self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul))
+    
+    
+    def test_elementwise_mul_scalar(self):
+        """
+        Test autograd from elementwise multiplication with scalar: scalar * tensor
+        """
+        scalar = 2
+        norch_tensor_elemwise_mul_scalar = norch.Tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        norch_result_elemwise_mul_scalar = (scalar * norch_tensor_elemwise_mul_scalar).sum()
+        norch_result_elemwise_mul_scalar.backward()
+        norch_tensor_grad_elemwise_mul_scalar = utils.to_torch(norch_tensor_elemwise_mul_scalar.grad)
+
+        torch_tensor_elemwise_mul_scalar = torch.tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_result_elemwise_mul_scalar = (scalar * torch_tensor_elemwise_mul_scalar).sum()
+        torch_result_elemwise_mul_scalar.backward()
+        torch_tensor_grad_elemwise_mul_scalar = torch_tensor_elemwise_mul_scalar.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_elemwise_mul_scalar, torch_tensor_grad_elemwise_mul_scalar))
+    
+    
+    def test_elementwise_mul_tensor(self):
+        """
+        Test autograd from elementwise multiplication between two tensors: tensor1 * tensor2
+        """
+        norch_tensor1_elemwise_mul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        norch_tensor2_elemwise_mul = norch.Tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True)
+        norch_result_elemwise_mul = (norch_tensor1_elemwise_mul * norch_tensor2_elemwise_mul).sum()
+        norch_result_elemwise_mul.backward()
+        norch_tensor1_grad_elemwise_mul = utils.to_torch(norch_tensor1_elemwise_mul.grad)
+        norch_tensor2_grad_elemwise_mul = utils.to_torch(norch_tensor2_elemwise_mul.grad)
+
+        torch_tensor1_elemwise_mul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_tensor2_elemwise_mul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True)
+        torch_result_elemwise_mul = (torch_tensor1_elemwise_mul * torch_tensor2_elemwise_mul).sum()
+        torch_result_elemwise_mul.backward()
+        torch_tensor1_grad_elemwise_mul = torch_tensor1_elemwise_mul.grad
+        torch_tensor2_grad_elemwise_mul = torch_tensor2_elemwise_mul.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor1_grad_elemwise_mul, torch_tensor1_grad_elemwise_mul))
+        self.assertTrue(utils.compare_torch(norch_tensor2_grad_elemwise_mul, torch_tensor2_grad_elemwise_mul))
+
+    def test_reshape(self):
+        """
+        Test autograd from reshaping a tensor: tensor.reshape(shape)
+        """
+        norch_tensor_reshape = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        new_shape = [2, 4]
+        norch_result_reshape = norch_tensor_reshape.reshape(new_shape).sum()
+        norch_result_reshape.backward()
+        norch_tensor_grad_reshape = utils.to_torch(norch_tensor_reshape.grad)
+
+        torch_tensor_reshape = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_result_reshape = torch_tensor_reshape.reshape(new_shape).sum()
+        torch_result_reshape.backward()
+        torch_tensor_grad_reshape = torch_tensor_reshape.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape, torch_tensor_grad_reshape))
+    
+    
+    def test_transpose_axes(self):
+        """
+        Test autograd from transposing a tensor with specific axes: tensor.transpose(axis1, axis2)
+        """
+        norch_tensor_transpose = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        axis1, axis2 = 0, 2
+        norch_result_transpose = norch_tensor_transpose.transpose(axis1, axis2).sum()
+        norch_result_transpose.backward()
+        norch_tensor_grad_transpose = utils.to_torch(norch_tensor_transpose.grad)
+
+        torch_tensor_transpose = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_result_transpose = torch_tensor_transpose.transpose(axis1, axis2).sum()
+        torch_result_transpose.backward()
+        torch_tensor_grad_transpose = torch_tensor_transpose.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose, torch_tensor_grad_transpose))
+    
+    
+    def test_T(self):
+        """
+        Test autograd from transposing a tensor using .T attribute
+        """
+        norch_tensor_T = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        norch_result_T = norch_tensor_T.T.sum()
+        norch_result_T.backward()
+        norch_tensor_grad_T = utils.to_torch(norch_tensor_T.grad)
+
+        torch_tensor_T = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True)
+        torch_result_T = torch_tensor_T.mT.sum()
+        torch_result_T.backward()
+        torch_tensor_grad_T = torch_tensor_T.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_T, torch_tensor_grad_T))
+
+    def test_reshape_then_matmul(self):
+        """
+        Test autograd from reshaping a tensor then performing matrix multiplication: matmul(tensor1.reshape(shape), tensor2)
+        """
+        norch_tensor_reshape_matmul = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True)
+        new_shape = [2, 4]
+        norch_result_reshape_matmul = (norch_tensor_reshape_matmul.reshape(new_shape) @ norch_tensor_reshape_matmul).sum()
+        norch_result_reshape_matmul.backward()
+        norch_tensor_grad_reshape_matmul = utils.to_torch(norch_tensor_reshape_matmul.grad)
+
+        torch_tensor_reshape_matmul = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True)
+        torch_result_reshape_matmul = torch.matmul(torch_tensor_reshape_matmul.reshape(new_shape), torch_tensor_reshape_matmul).sum()
+        torch_result_reshape_matmul.backward()
+        torch_tensor_grad_reshape_matmul = torch_tensor_reshape_matmul.grad
+        
+        
+        print(norch_tensor_grad_reshape_matmul)
+        print(torch_tensor_grad_reshape_matmul)
+        print("\n\n\n\n@@")
+        
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul, torch_tensor_grad_reshape_matmul))
+
+
+    def test_T_then_matmul(self):
+        """
+        Test autograd from transposing a tensor then performing matrix multiplication: matmul(tensor.T, tensor)
+        """
+        norch_tensor_T_matmul = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True)
+        norch_result_T_matmul = (norch_tensor_T_matmul.T @ norch_tensor_T_matmul).sum()
+        norch_result_T_matmul.backward()
+        norch_tensor_grad_T_matmul = utils.to_torch(norch_tensor_T_matmul.grad)
+
+        torch_tensor_T_matmul = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True)
+        torch_result_T_matmul = torch.matmul(torch_tensor_T_matmul.T, torch_tensor_T_matmul).sum()
+        torch_result_T_matmul.backward()
+        torch_tensor_grad_T_matmul = torch_tensor_T_matmul.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_T_matmul, torch_tensor_grad_T_matmul))
+
+
+    def test_transpose_axes_then_matmul(self):
+        """
+        Test autograd from transposing a tensor with specific axes then performing matrix multiplication: matmul(tensor.transpose(axis1, axis2), tensor)
+        """
+        norch_tensor_transpose_matmul = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True)
+        axis1, axis2 = 0, 1
+        norch_result_transpose_matmul = (norch_tensor_transpose_matmul.transpose(axis1, axis2) @ norch_tensor_transpose_matmul).sum()
+        norch_result_transpose_matmul.backward()
+        norch_tensor_grad_transpose_matmul = utils.to_torch(norch_tensor_transpose_matmul.grad)
+
+        torch_tensor_transpose_matmul = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True)
+        torch_result_transpose_matmul = torch.matmul(torch_tensor_transpose_matmul.transpose(axis1, axis2), torch_tensor_transpose_matmul).sum()
+        torch_result_transpose_matmul.backward()
+        torch_tensor_grad_transpose_matmul = torch_tensor_transpose_matmul.grad
+
+        self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose_matmul, torch_tensor_grad_transpose_matmul))
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_operations.py

@@ -0,0 +1,266 @@
+import unittest
+import norch
+from norch import utils
+import torch
+import sys
+
+class TestTensorOperations(unittest.TestCase):
+    def test_creation_and_conversion(self):
+        """
+        Test creation and convertion of norch tensor to pytorch
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_tensor = utils.to_torch(norch_tensor)
+        self.assertTrue(torch.is_tensor(torch_tensor))
+
+    def test_addition(self):
+        """
+        Test addition two tensors: tensor1 + tensor2
+        """
+        norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]])
+        norch_result = norch_tensor1 + norch_tensor2
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]])
+        torch_expected = torch_tensor1 + torch_tensor2
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_subtraction(self):
+        """
+        Test subtraction of two tensors: tensor1 - tensor2
+        """
+        norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]])
+        norch_result = norch_tensor1 - norch_tensor2
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]])
+        torch_expected = torch_tensor1 - torch_tensor2
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_division_by_scalar(self):
+        """
+        Test division of a tensor by a scalar: tensor / scalar
+        """
+        norch_tensor = norch.Tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]])
+        scalar = 2
+        norch_result = norch_tensor / scalar
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]])
+        torch_expected = torch_tensor / scalar
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_scalar_division_by_tensor(self):
+        """
+        Test scalar division by a tensor: scalar / tensor
+        """
+        scalar = 10
+        norch_tensor = norch.Tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]])
+        norch_result = scalar / norch_tensor
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]])
+        torch_expected = scalar / torch_tensor
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_matrix_multiplication(self):
+        """
+        Test matrix multiplication: tensor1 @ tensor2
+        """
+        norch_tensor1 = norch.Tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        norch_tensor2 = norch.Tensor([[[1, 0], [0, 1]], [[-1, 0], [0, -1]]])
+        norch_result = norch_tensor1 @ norch_tensor2
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor1 = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        torch_tensor2 = torch.tensor([[[1, 0], [0, 1]], [[-1, 0], [0, -1]]])
+        torch_expected = torch_tensor1 @ torch_tensor2
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_elementwise_multiplication_by_scalar(self):
+        """
+        Test elementwise multiplication of a tensor by a scalar: tensor * scalar
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        scalar = 2
+        norch_result = norch_tensor * scalar
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_expected = torch_tensor * scalar
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_elementwise_multiplication_by_tensor(self):
+        """
+        Test elementwise multiplication of two tensors: tensor1 * tensor2
+        """
+        norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        norch_tensor2 = norch.Tensor([[[2, 2], [2, 2]], [[2, 2], [2, 2]]])
+        norch_result = norch_tensor1 * norch_tensor2
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_tensor2 = torch.tensor([[[2, 2], [2, 2]], [[2, 2], [2, 2]]])
+        torch_expected = torch_tensor1 * torch_tensor2
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_reshape(self):
+        """
+        Test reshaping of a tensor: tensor.reshape(shape)
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        new_shape = [2, 4]
+        norch_result = norch_tensor.reshape(new_shape)
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_expected = torch_tensor.reshape(new_shape)
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_transpose(self):
+        """
+        Test transposition of a tensor: tensor.transpose(dim1, dim2)
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        dim1, dim2 = 0, 2
+        norch_result = norch_tensor.transpose(dim1, dim2)
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_expected = torch_tensor.transpose(dim1, dim2)
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_logarithm(self):
+        """
+        Test elementwise logarithm of a tensor: tensor.log()
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        norch_result = norch_tensor.log()
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        torch_expected = torch.log(torch_tensor)
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_sum(self):
+        """
+        Test summation of a tensor: tensor.sum()
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        norch_result = norch_tensor.sum()
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_expected = torch.sum(torch_tensor)
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_transpose_T(self):
+        """
+        Test transposition of a tensor: tensor.T
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        norch_result = norch_tensor.T
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_expected = torch.transpose(torch_tensor, 0, 2)
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_reshape_then_matmul(self):
+        """
+        Test reshaping a tensor followed by matrix multiplication: (tensor.reshape(shape) @ other_tensor)
+        """
+        norch_tensor = norch.Tensor([[1, 2], [3, -4], [5, 6], [7, 8]])
+        new_shape = [2, 4]
+        norch_reshaped = norch_tensor.reshape(new_shape)
+        
+        norch_result = norch_reshaped @ norch_tensor
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[1, 2], [3, -4], [5, 6], [7, 8]])
+        torch_expected = torch_tensor.reshape(new_shape) @ torch_tensor
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+
+    def test_transpose_then_matmul(self):
+        """
+        Test transposing a tensor followed by matrix multiplication: (tensor.transpose(dim1, dim2) @ other_tensor)
+        """
+        norch_tensor = norch.Tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        dim1, dim2 = 0, 2
+        norch_result = norch_tensor.transpose(dim1, dim2) @ norch_tensor
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        torch_expected = torch_tensor.transpose(dim1, dim2) @ torch_tensor
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_add_div_matmul_then_reshape(self):
+        """
+        Test a combination of operations: (tensor.sum() + other_tensor) / scalar @ another_tensor followed by reshape
+        """
+        norch_tensor1 = norch.Tensor([[[1., 2], [3, -4]], [[5, 6], [7, 8]]])
+        norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]])
+        scalar = 2
+        new_shape = [2, 4]
+        norch_result = ((norch_tensor1 + norch_tensor2) / scalar) @ norch_tensor1
+        norch_result = norch_result.reshape(new_shape)
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor1 = torch.tensor([[[1., 2], [3, -4]], [[5, 6], [7, 8]]])
+        torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]])
+        torch_expected = ((torch_tensor1 + torch_tensor2) / scalar) @ torch_tensor1
+        torch_expected = torch_expected.reshape(new_shape)
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_scalar_power_tensor(self):
+        """
+        Test scalar power of a tensor: scalar ** tensor
+        """
+        scalar = 3
+        norch_tensor = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]])
+        norch_result = scalar ** norch_tensor
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]])
+        torch_expected = scalar ** torch_tensor
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+    def test_tensor_power_scalar(self):
+        """
+        Test tensor power of a scalar: tensor ** scalar
+        """
+        scalar = 3
+        norch_tensor = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]])
+        norch_result = norch_tensor ** scalar
+        torch_result = utils.to_torch(norch_result)
+
+        torch_tensor = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]])
+        torch_expected = torch_tensor ** scalar
+
+        self.assertTrue(utils.compare_torch(torch_result, torch_expected))
+
+
+
+if __name__ == '__main__':
+    unittest.main()