diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 62b2210..6f19674 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/module.cpython-38.pyc b/norch/nn/__pycache__/module.cpython-38.pyc index 455e790..f960105 100644 Binary files a/norch/nn/__pycache__/module.cpython-38.pyc and b/norch/nn/__pycache__/module.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/parameter.cpython-38.pyc b/norch/nn/__pycache__/parameter.cpython-38.pyc index a54bfe0..10a5bf3 100644 Binary files a/norch/nn/__pycache__/parameter.cpython-38.pyc and b/norch/nn/__pycache__/parameter.cpython-38.pyc differ diff --git a/norch/nn/module.py b/norch/nn/module.py index d0ffaf1..eaabd9a 100644 --- a/norch/nn/module.py +++ b/norch/nn/module.py @@ -47,7 +47,7 @@ class Module(ABC): yield module._grads def zero_grad(self): - for parameter in self.parameters(): + for _, _, parameter in self.parameters(): parameter.zero_grad() def to(self, device): diff --git a/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc b/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc index 869d1b1..a46889f 100644 Binary files a/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc and b/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc differ diff --git a/norch/optim/optimizers/sgd.py b/norch/optim/optimizers/sgd.py index 4604f41..3539114 100644 --- a/norch/optim/optimizers/sgd.py +++ b/norch/optim/optimizers/sgd.py @@ -9,13 +9,15 @@ class SGD(Optimizer): self._cache = {'velocity': [p.zeros_like() for (_, _, p) in self.parameters]} def step(self): - for i, (module, name, parameter) in enumerate(self.parameters): + for i, (module, name, _) in enumerate(self.parameters): + parameter = getattr(module, name) + velocity = self._cache['velocity'][i] velocity = self.momentum * velocity - self.lr * parameter.grad - parameter += velocity + updated_parameter = parameter + velocity - setattr(module, name, parameter) + setattr(module, name, updated_parameter) self._cache['velocity'][i] = velocity diff --git a/norch/tensor.py b/norch/tensor.py index ac0d996..845ceb4 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -164,9 +164,7 @@ class Tensor: visited.add(tensor) def zero_grad(self): - tmp = self.zeros_like() - self.detach() - self.grad = tmp + self.grad = None def __getitem__(self, indices): if len(indices) != self.ndim: diff --git a/norch/utils/__pycache__/utils.cpython-38.pyc b/norch/utils/__pycache__/utils.cpython-38.pyc index 1abde9b..2285e44 100644 Binary files a/norch/utils/__pycache__/utils.cpython-38.pyc and b/norch/utils/__pycache__/utils.cpython-38.pyc differ diff --git a/norch/utils/utils.py b/norch/utils/utils.py index 2614a09..e977d6d 100644 --- a/norch/utils/utils.py +++ b/norch/utils/utils.py @@ -1,4 +1,6 @@ import random +import numpy as np + def generate_random_list(shape): """ @@ -9,7 +11,7 @@ def generate_random_list(shape): else: inner_shape = shape[1:] if len(inner_shape) == 0: - return [random.random()] * shape[0] + return [random.uniform(-1, 1) for _ in range(shape[0])] else: return [generate_random_list(inner_shape) for _ in range(shape[0])] \ No newline at end of file diff --git a/test.py b/test.py index 0b30999..b2af4cf 100644 --- a/test.py +++ b/test.py @@ -93,24 +93,22 @@ if __name__ == "__main__": return out modelo = MeuModulo() - input_list = [[0.05 for _ in range(5)]] + input_list = [[0.5 for _ in range(5)]] input = norch.Tensor(input_list).T criterion = nn.MSELoss() - optimizer = norch.optim.SGD(modelo.parameters(), lr=0.1) + optimizer = norch.optim.SGD(modelo.parameters(), lr=1) - target_list = [[0.1 for _ in range(2)]] + target_list = [[random.random() for _ in range(2)]] target = norch.Tensor(target_list).T - - for epoch in range(50): - cpu_percent = psutil.cpu_percent(interval=1) - memory_usage = psutil.virtual_memory() + + for epoch in range(10): output = modelo(input) loss = criterion(output, target) optimizer.zero_grad() loss.backward() + #print('fora grad', modelo.layer1.weight.grad, "\n\n") optimizer.step() - print(loss)