Merge pull request #35 from lucasdelimanogueira/tmp

Tmp

norch/nn/module.py

@@ -47,13 +47,15 @@ class Module(ABC):
             yield module._grads
 
     def zero_grad(self):
-        for parameter in self.parameters():
+        for _, _, parameter in self.parameters():
             parameter.zero_grad()
 
     def to(self, device):
-        for parameter in self.parameters():
+        for _, _, parameter in self.parameters():
             parameter.to(device)
 
+        return self
+    
     def state_dict(self):
         state = OrderedDict()
         for i, param in enumerate(self.parameters()):

norch/optim/optimizers/sgd.py

@@ -9,13 +9,15 @@ class SGD(Optimizer):
         self._cache = {'velocity': [p.zeros_like() for (_, _, p) in self.parameters]}
 
     def step(self):
-        for i, (module, name, parameter) in enumerate(self.parameters):
+        for i, (module, name, _) in enumerate(self.parameters):
+            parameter = getattr(module, name)
+
             velocity = self._cache['velocity'][i]
 
             velocity = self.momentum * velocity - self.lr * parameter.grad
 
-            parameter += velocity
+            updated_parameter = parameter + velocity
 
-            setattr(module, name, parameter)
+            setattr(module, name, updated_parameter)
 
             self._cache['velocity'][i] = velocity

norch/tensor.py

@@ -137,24 +137,31 @@ class Tensor:
     def backward(self, gradient=None):
         if not self.requires_grad:
             return
-        
+                
         if gradient is None:
             if self.shape == [1]:
                 gradient = Tensor([1])
             else:
                 raise RuntimeError("Gradient argument must be specified for non-scalar tensors.")
 
-        if self.grad is None:
-            self.grad = gradient
+        stack = [(self, gradient)]
+        visited = set()
+    
+        while stack:
+            tensor, grad = stack.pop()
+            
+            if tensor.grad is None:
+                tensor.grad = grad
+            else:
+                tensor.grad += grad
 
-        else:
-            self.grad += gradient
-
-        if self.grad_fn is not None: # not a leaf
-            grads = self.grad_fn.backward(gradient)
-            for tensor, grad in zip(self.grad_fn.input, grads):
-                if isinstance(tensor, Tensor):
-                    tensor.backward(grad)
+            # Propagate gradients to inputs if not a leaf tensor
+            if tensor.grad_fn is not None:
+                grads = tensor.grad_fn.backward(grad)
+                for tensor, grad in zip(tensor.grad_fn.input, grads):
+                    if isinstance(tensor, Tensor) and tensor not in visited:
+                        stack.append((tensor, grad))
+                        visited.add(tensor)
 
     def zero_grad(self):
         self.grad = None
@@ -570,5 +577,13 @@ class Tensor:
         result_data.shape = self.shape.copy()[::-1]
         result_data.ndim = self.ndim
         result_data.device = self.device
+        
+        result_data.requires_grad = self.requires_grad
 
-        return result_data
+        return result_data
+    
+    def detach(self):
+        self.grad = None
+        self.grad_fn = None
+
+        return self

norch/utils/utils.py

@@ -1,4 +1,6 @@
 import random
+import numpy as np
+
 
 def generate_random_list(shape):
     """
@@ -9,7 +11,7 @@ def generate_random_list(shape):
     else:
         inner_shape = shape[1:]
         if len(inner_shape) == 0:
-            return [random.random()] * shape[0]
+            return [random.uniform(-1, 1) for _ in range(shape[0])]
         else:
             return [generate_random_list(inner_shape) for _ in range(shape[0])]
         

test.py

@@ -19,6 +19,7 @@ if __name__ == "__main__":
     import time
     import random
     import numpy as np
+    import psutil
 
     """a = norch.Tensor([
         [[1.234, 2.123], [3.635, 4.456], [5.678, 6.789]],
@@ -69,11 +70,17 @@ if __name__ == "__main__":
 
     import norch.nn as nn
 
+    cpu_percent = psutil.cpu_percent(interval=1)
+    print(f"CPU Usage: {cpu_percent}%")
+    memory_usage = psutil.virtual_memory()
+    print(f"Memory Usage: {memory_usage.percent}%")  
+    
+
     class MeuModulo(nn.Module):
         def __init__(self):
             super(MeuModulo, self).__init__()
 
-            self.layer1 = nn.Linear(10, 2)
+            self.layer1 = nn.Linear(5, 2)
             #self.layer2 = nn.Linear(5, 2)
             self.sigmoid = nn.Sigmoid()
 
@@ -86,33 +93,24 @@ if __name__ == "__main__":
             return out
         
     modelo = MeuModulo()
-    input_list = [[0.05 for _ in range(10)]]
+    input_list = [[0.5 for _ in range(5)]]
     input = norch.Tensor(input_list).T
     criterion = nn.MSELoss()
-    optimizer = norch.optim.SGD(modelo.parameters(), lr=0.5)
+    optimizer = norch.optim.SGD(modelo.parameters(), lr=1)
 
-    target_list = [[0.1 for _ in range(2)]]
+    target_list = [[random.random() for _ in range(2)]]
     target = norch.Tensor(target_list).T
-    
-    for epoch in range(5):
+
+    for epoch in range(10):
         output = modelo(input)
         loss = criterion(output, target)
         optimizer.zero_grad()
-        #print('FORA ANTES: ', modelo.layer1.bias, '\n')
 
-        #print(modelo.layer1.weight.grad)        
         loss.backward()
-        #print(modelo.layer1.weight.grad)
+        #print('fora grad', modelo.layer1.weight.grad, "\n\n")
         optimizer.step()
-        #print('FORA DEPOIS: ', modelo.layer1.bias, '\n')
-
-        #print("\n\n")
-        #print(modelo.layer1.weight.grad)
-
-        #print(modelo.layer1.weight[0,0])
         print(loss)
 
-    exit()
 
     #### testar transpose axes!!!! make it contiguous
 
@@ -122,15 +120,20 @@ if __name__ == "__main__":
                         [[19, 20], [21, 22], [23, 24]],
                         [[25, 26], [27, 28], [29, 0.030]]], requires_grad=True)
     
-    op = nn.Sigmoid()
-    result = op(tensor1)
-    result = result.sum()
+    #op = nn.Sigmoid()
+    tensor2 = 
+    result = tensor1.sum()
+    
     result.backward()
     print(tensor1.grad)
     exit()"""
 
     # Reshape tensor1 to 2x3x5
-    reshaped_tensor = tensor1.transpose(1, 0)
+    """tensor1 = norch.Tensor([[[1, 2], [3, 4], [5, 6]],
+                        [[7, 8], [9, 10], [11, 12]],
+                        [[13, 14], [15, 16], [17, 18]],
+                        [[19, 20], [21, 22], [23, 24]],
+                        [[25, 26], [27, 28], [29, 0.030]]], requires_grad=True)
 
     # Create a 5x4 tensor
     tensor2 = norch.Tensor([[1, 2, 3],
@@ -138,16 +141,14 @@ if __name__ == "__main__":
                             [9, 10, 11],
                             [13, 14, 15],
                             [17, 18, 19]])
-
-    tensor2 = tensor2.transpose(1,0)
     
 
     # Multiply reshaped_tensor by tensor2
-    result = tensor2 @ reshaped_tensor
+    result = tensor2 @ tensor1
 
     result = result.sum()
     result.backward()
-    print(tensor1.grad)
+    print(tensor1.grad)"""
 
     #print(a.shape, b.shape, result.shape)
     #c = result.sum()