Fix grad update step! loss decreasing

norch/nn/module.py

@@ -47,7 +47,7 @@ class Module(ABC):
             yield module._grads
 
     def zero_grad(self):
-        for parameter in self.parameters():
+        for _, _, parameter in self.parameters():
             parameter.zero_grad()
 
     def to(self, device):

norch/optim/optimizers/sgd.py

@@ -9,13 +9,15 @@ class SGD(Optimizer):
         self._cache = {'velocity': [p.zeros_like() for (_, _, p) in self.parameters]}
 
     def step(self):
-        for i, (module, name, parameter) in enumerate(self.parameters):
+        for i, (module, name, _) in enumerate(self.parameters):
+            parameter = getattr(module, name)
+
             velocity = self._cache['velocity'][i]
 
             velocity = self.momentum * velocity - self.lr * parameter.grad
 
-            parameter += velocity
+            updated_parameter = parameter + velocity
 
-            setattr(module, name, parameter)
+            setattr(module, name, updated_parameter)
 
             self._cache['velocity'][i] = velocity

norch/tensor.py

@@ -164,9 +164,7 @@ class Tensor:
                         visited.add(tensor)
 
     def zero_grad(self):
-        tmp = self.zeros_like()
-        self.detach()
-        self.grad = tmp
+        self.grad = None
 
     def __getitem__(self, indices):
         if len(indices) != self.ndim:

norch/utils/utils.py

@@ -1,4 +1,6 @@
 import random
+import numpy as np
+
 
 def generate_random_list(shape):
     """
@@ -9,7 +11,7 @@ def generate_random_list(shape):
     else:
         inner_shape = shape[1:]
         if len(inner_shape) == 0:
-            return [random.random()] * shape[0]
+            return [random.uniform(-1, 1) for _ in range(shape[0])]
         else:
             return [generate_random_list(inner_shape) for _ in range(shape[0])]
         

test.py

@@ -93,24 +93,22 @@ if __name__ == "__main__":
             return out
         
     modelo = MeuModulo()
-    input_list = [[0.05 for _ in range(5)]]
+    input_list = [[0.5 for _ in range(5)]]
     input = norch.Tensor(input_list).T
     criterion = nn.MSELoss()
-    optimizer = norch.optim.SGD(modelo.parameters(), lr=0.1)
+    optimizer = norch.optim.SGD(modelo.parameters(), lr=1)
 
-    target_list = [[0.1 for _ in range(2)]]
+    target_list = [[random.random() for _ in range(2)]]
     target = norch.Tensor(target_list).T
-    
-    for epoch in range(50):
-        cpu_percent = psutil.cpu_percent(interval=1)
-        memory_usage = psutil.virtual_memory()
+
+    for epoch in range(10):
         output = modelo(input)
         loss = criterion(output, target)
         optimizer.zero_grad()
 
         loss.backward()
+        #print('fora grad', modelo.layer1.weight.grad, "\n\n")
         optimizer.step()
-
         print(loss)