diff --git a/build/libtensor.so b/build/libtensor.so index dacb48e..856e985 100755 Binary files a/build/libtensor.so and b/build/libtensor.so differ diff --git a/build/tensor.o b/build/tensor.o index 549b305..1e68803 100644 Binary files a/build/tensor.o and b/build/tensor.o differ diff --git a/norch/__init__.py b/norch/__init__.py index 98156f7..e013fa7 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -1 +1,2 @@ -from norch.tensor import Tensor \ No newline at end of file +from norch.tensor import Tensor +from .optim import * \ No newline at end of file diff --git a/norch/__pycache__/__init__.cpython-38.pyc b/norch/__pycache__/__init__.cpython-38.pyc index bf1db31..60e1eea 100644 Binary files a/norch/__pycache__/__init__.cpython-38.pyc and b/norch/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index d22bb34..ce87ffb 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/csrc/tensor.cpp b/norch/csrc/tensor.cpp index 741662e..1c46245 100644 --- a/norch/csrc/tensor.cpp +++ b/norch/csrc/tensor.cpp @@ -11,7 +11,6 @@ extern "C" { Tensor* create_tensor(float* data, int* shape, int ndim, char* device) { - printf("Creating tensor\n"); Tensor* tensor = (Tensor*)malloc(sizeof(Tensor)); if (tensor == NULL) { fprintf(stderr, "Memory allocation failed\n"); @@ -44,39 +43,6 @@ extern "C" { tensor->strides[i] = stride; stride *= shape[i]; } - - printf("Tensor created successfully\n"); - printf("Tensor information:\n"); - printf("Number of dimensions: %d\n", tensor->ndim); - printf("Number size: %d\n", tensor->size); - printf("Device: %s\n", tensor->device); - - printf("Shape: ["); - for (int i = 0; i < ndim; i++) { - printf("%d", tensor->shape[i]); - if (i < ndim - 1) { - printf(", "); - } - } - printf("]\n"); - - printf("Strides: ["); - for (int i = 0; i < ndim; i++) { - printf("%d", tensor->strides[i]); - if (i < ndim - 1) { - printf(", "); - } - } - printf("]\n"); - - /*printf("Data:\n["); - for (int i = 0; i < stride; i++) { - printf("%.2f", tensor->data[i]); - if (i < stride - 1) { - printf(", "); - } - } - printf("]\n\n\n");*/ return tensor; } @@ -98,8 +64,6 @@ extern "C" { } void to_device(Tensor* tensor, char* target_device) { - printf("Sending tensor to device: %s\n", target_device); - if ((strcmp(target_device, "cuda") == 0) && (strcmp(tensor->device, "cpu") == 0)) { cpu_to_cuda(tensor); } diff --git a/norch/nn/__init__.py b/norch/nn/__init__.py index 12c8a92..dadface 100644 --- a/norch/nn/__init__.py +++ b/norch/nn/__init__.py @@ -1,2 +1,3 @@ from .modules import * -from .activation import * \ No newline at end of file +from .activation import * +from .loss import * \ No newline at end of file diff --git a/norch/nn/__pycache__/__init__.cpython-38.pyc b/norch/nn/__pycache__/__init__.cpython-38.pyc index 8726808..1f19ab6 100644 Binary files a/norch/nn/__pycache__/__init__.cpython-38.pyc and b/norch/nn/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/loss.cpython-38.pyc b/norch/nn/__pycache__/loss.cpython-38.pyc new file mode 100644 index 0000000..9f14563 Binary files /dev/null and b/norch/nn/__pycache__/loss.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/module.cpython-38.pyc b/norch/nn/__pycache__/module.cpython-38.pyc index 32c92f5..6321315 100644 Binary files a/norch/nn/__pycache__/module.cpython-38.pyc and b/norch/nn/__pycache__/module.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/parameter.cpython-38.pyc b/norch/nn/__pycache__/parameter.cpython-38.pyc index 7c686b8..a54bfe0 100644 Binary files a/norch/nn/__pycache__/parameter.cpython-38.pyc and b/norch/nn/__pycache__/parameter.cpython-38.pyc differ diff --git a/norch/nn/loss.py b/norch/nn/loss.py new file mode 100644 index 0000000..9b55af9 --- /dev/null +++ b/norch/nn/loss.py @@ -0,0 +1,25 @@ +from .module import Module +from abc import ABC + +class Loss(Module, ABC): + "Abstract class for loss functions" + + def __init__(self): + super().__init__() + + def forward(self, predictions, labels): + raise NotImplementedError + + def __call__(self, *inputs): + return self.forward(*inputs) + + +class MSELoss(Loss): + def __init__(self): + super().__init__() + + def forward(self, predictions, labels): + assert labels.shape == predictions.shape, \ + "Labels and predictions shape does not match: {} and {}".format(labels.shape, predictions.shape) + + return ((predictions - labels) **2).sum() \ No newline at end of file diff --git a/norch/nn/module.py b/norch/nn/module.py index 92cc0ca..c7d8a30 100644 --- a/norch/nn/module.py +++ b/norch/nn/module.py @@ -16,6 +16,9 @@ class Module(ABC): self._grads = OrderedDict() self.training = True + def forward(self, *inputs, **kwargs): + raise NotImplementedError + def __call__(self, *inputs, **kwargs): return self.forward(*inputs, **kwargs) @@ -36,6 +39,17 @@ class Module(ABC): elif isinstance(value, Module): yield from value.parameters() + def modules(self): + yield from self._modules.values() + + def gradients(self): + for module in self.modules(): + yield module._grads + + def zero_grad(self): + for parameter in self.parameters(): + parameter.zero_grad() + def to(self, device): for parameter in self.parameters(): parameter.to(device) diff --git a/norch/nn/modules/__pycache__/linear.cpython-38.pyc b/norch/nn/modules/__pycache__/linear.cpython-38.pyc index 3b211f6..356b57a 100644 Binary files a/norch/nn/modules/__pycache__/linear.cpython-38.pyc and b/norch/nn/modules/__pycache__/linear.cpython-38.pyc differ diff --git a/norch/nn/modules/linear.py b/norch/nn/modules/linear.py index 83ce02d..2d61023 100644 --- a/norch/nn/modules/linear.py +++ b/norch/nn/modules/linear.py @@ -10,7 +10,6 @@ class Linear(Module): self.bias = Parameter(shape=[self.output_dim, 1]) def forward(self, x): - print(self.weight.shape, x.shape, self.bias.shape, "@@@@@@\n\n\n\n") z = self.weight @ x + self.bias return z diff --git a/norch/nn/parameter.py b/norch/nn/parameter.py index 8203ba4..2324a1a 100644 --- a/norch/nn/parameter.py +++ b/norch/nn/parameter.py @@ -8,5 +8,4 @@ class Parameter(Tensor): """ def __init__(self, shape): data = utils.generate_random_list(shape=shape) - super().__init__(data, requires_grad=True) \ No newline at end of file diff --git a/norch/optim/__init__.py b/norch/optim/__init__.py new file mode 100644 index 0000000..c5baa22 --- /dev/null +++ b/norch/optim/__init__.py @@ -0,0 +1 @@ +from .optimizers import * \ No newline at end of file diff --git a/norch/optim/__pycache__/__init__.cpython-38.pyc b/norch/optim/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..34d28a8 Binary files /dev/null and b/norch/optim/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/optim/__pycache__/optimizer.cpython-38.pyc b/norch/optim/__pycache__/optimizer.cpython-38.pyc new file mode 100644 index 0000000..514d2be Binary files /dev/null and b/norch/optim/__pycache__/optimizer.cpython-38.pyc differ diff --git a/norch/optim/optimizer.py b/norch/optim/optimizer.py new file mode 100644 index 0000000..fabd468 --- /dev/null +++ b/norch/optim/optimizer.py @@ -0,0 +1,22 @@ +from abc import ABC +from norch.tensor import Tensor + +class Optimizer(ABC): + """ + Abstract class for optimizers + """ + + def __init__(self, parameters): + if isinstance(parameters, Tensor): + raise TypeError("parameters should be an iterable but got {}".format(type(parameters))) + elif isinstance(parameters, dict): + parameters = parameters.values() + + self.parameters = list(parameters) + + def step(self): + raise NotImplementedError + + def zero_grad(self): + for parameter in self.parameters: + parameter.zero_grad() \ No newline at end of file diff --git a/norch/optim/optimizers/__init__.py b/norch/optim/optimizers/__init__.py new file mode 100644 index 0000000..bd076ac --- /dev/null +++ b/norch/optim/optimizers/__init__.py @@ -0,0 +1 @@ +from .sgd import * \ No newline at end of file diff --git a/norch/optim/optimizers/__pycache__/__init__.cpython-38.pyc b/norch/optim/optimizers/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..4631190 Binary files /dev/null and b/norch/optim/optimizers/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc b/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc new file mode 100644 index 0000000..d1e4537 Binary files /dev/null and b/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc differ diff --git a/norch/optim/optimizers/sgd.py b/norch/optim/optimizers/sgd.py new file mode 100644 index 0000000..eb194db --- /dev/null +++ b/norch/optim/optimizers/sgd.py @@ -0,0 +1,19 @@ +from ..optimizer import Optimizer +from norch.tensor import Tensor + +class SGD(Optimizer): + def __init__(self, parameters, lr=1e-1, momentum=0): + super().__init__(parameters) + self.lr = lr + self.momentum = momentum + self._cache = {'velocity': [p.zeros_like() for p in self.parameters]} + + def step(self): + for i, parameter in enumerate(self.parameters): + velocity = self._cache['velocity'][i] + + velocity = self.momentum * velocity - self.lr * parameter.grad + + parameter += velocity + + self._cache['velocity'][i] = velocity diff --git a/test.py b/test.py index 706ab10..68948c6 100644 --- a/test.py +++ b/test.py @@ -73,22 +73,43 @@ if __name__ == "__main__": def __init__(self): super(MeuModulo, self).__init__() - self.layer1 = nn.Linear(10, 100) - self.layer2 = nn.Linear(100, 2) + self.layer1 = nn.Linear(10, 2) + #self.layer2 = nn.Linear(5, 2) self.sigmoid = nn.Sigmoid() def forward(self, x): out = self.layer1(x) - out = self.layer2(out) out = self.sigmoid(out) + #out = self.layer2(out) + #out = self.sigmoid(out) return out modelo = MeuModulo() - input_list = [[0.01 for _ in range(10)]] + input_list = [[0.05 for _ in range(10)]] input = norch.Tensor(input_list).T - output = modelo(input) - print(output) + criterion = nn.MSELoss() + optimizer = norch.optim.SGD(modelo.parameters(), lr=0.5) + + target_list = [[0.1 for _ in range(2)]] + target = norch.Tensor(target_list).T + + for epoch in range(2): + output = modelo(input) + loss = criterion(output, target) + #print('FORA ANTES: ', modelo.layer1.bias, '\n') + optimizer.zero_grad() + #print(modelo.layer1.weight.grad) + loss.backward() + #print(modelo.layer1.weight.grad) + optimizer.step() + #print('FORA DEPOIS: ', modelo.layer1.bias.data, '\n') + + #print("\n\n") + #print(modelo.layer1.weight.grad) + + #print(modelo.layer1.weight[0,0]) + print(loss) exit()