Merge pull request #34 from lucasdelimanogueira/tmp

Tmp

norch/__init__.py

@@ -1 +1,2 @@
-from norch.tensor import Tensor
+from norch.tensor import Tensor
+from .optim import *

norch/csrc/tensor.cpp

@@ -11,7 +11,6 @@ extern "C" {
 
     Tensor* create_tensor(float* data, int* shape, int ndim, char* device) {
         
-        printf("Creating tensor\n");
         Tensor* tensor = (Tensor*)malloc(sizeof(Tensor));
         if (tensor == NULL) {
             fprintf(stderr, "Memory allocation failed\n");
@@ -44,39 +43,6 @@ extern "C" {
             tensor->strides[i] = stride;
             stride *= shape[i];
         }
-
-        printf("Tensor created successfully\n");
-        printf("Tensor information:\n");
-        printf("Number of dimensions: %d\n", tensor->ndim);
-        printf("Number size: %d\n", tensor->size);
-        printf("Device: %s\n", tensor->device);
-
-        printf("Shape: [");
-        for (int i = 0; i < ndim; i++) {
-            printf("%d", tensor->shape[i]);
-            if (i < ndim - 1) {
-                printf(", ");
-            }
-        }
-        printf("]\n");
-
-        printf("Strides: [");
-        for (int i = 0; i < ndim; i++) {
-            printf("%d", tensor->strides[i]);
-            if (i < ndim - 1) {
-                printf(", ");
-            }
-        }
-        printf("]\n");
-
-        /*printf("Data:\n[");
-        for (int i = 0; i < stride; i++) {
-            printf("%.2f", tensor->data[i]);
-            if (i < stride - 1) {
-                printf(", ");
-            }
-        }
-        printf("]\n\n\n");*/
         
         return tensor;
     }
@@ -98,8 +64,6 @@ extern "C" {
     }
 
     void to_device(Tensor* tensor, char* target_device) {
-        printf("Sending tensor to device: %s\n", target_device);
-
         if ((strcmp(target_device, "cuda") == 0) && (strcmp(tensor->device, "cpu") == 0)) {
             cpu_to_cuda(tensor);
         }

norch/nn/__init__.py

@@ -1,2 +1,3 @@
 from .modules import *
-from .activations import *
+from .activation import *
+from .loss import *

norch/nn/activation.py

@@ -1,4 +1,4 @@
-from module import Module
+from .module import Module
 import math
 
 class Activation(Module):

norch/nn/loss.py

@@ -0,0 +1,25 @@
+from .module import Module
+from abc import ABC
+
+class Loss(Module, ABC):
+    "Abstract class for loss functions"
+
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, predictions, labels):
+        raise NotImplementedError
+    
+    def __call__(self, *inputs):
+        return self.forward(*inputs)
+    
+
+class MSELoss(Loss):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, predictions, labels):
+        assert labels.shape == predictions.shape, \
+            "Labels and predictions shape does not match: {} and {}".format(labels.shape, predictions.shape)
+        
+        return ((predictions - labels) **2).sum()

norch/nn/module.py

@@ -1,4 +1,4 @@
-from parameter import Parameter
+from .parameter import Parameter
 from collections import OrderedDict
 from abc import ABC
 import pickle
@@ -16,6 +16,9 @@ class Module(ABC):
         self._grads = OrderedDict()
         self.training = True
 
+    def forward(self, *inputs, **kwargs):
+        raise NotImplementedError
+
     def __call__(self, *inputs, **kwargs):
         return self.forward(*inputs, **kwargs)
 
@@ -32,10 +35,21 @@ class Module(ABC):
     def parameters(self):
         for name, value in inspect.getmembers(self):
             if isinstance(value, Parameter):
-                yield value
+                yield self, name, value
             elif isinstance(value, Module):
                 yield from value.parameters()
 
+    def modules(self):
+        yield from self._modules.values()
+
+    def gradients(self):
+        for module in self.modules():
+            yield module._grads
+
+    def zero_grad(self):
+        for parameter in self.parameters():
+            parameter.zero_grad()
+
     def to(self, device):
         for parameter in self.parameters():
             parameter.to(device)
@@ -78,4 +92,12 @@ class Module(ABC):
         return f'{string}\n)'
     
     def get_name(self):
-        return self.__class__.__name__
+        return self.__class__.__name__
+    
+    def __setattr__(self, key, value):
+        self.__dict__[key] = value
+
+        if isinstance(value, Module):
+            self._modules[key] = value
+        elif isinstance(value, Parameter):
+            self._params[key] = value

norch/nn/modules/__init__.py

@@ -0,0 +1 @@
+from .linear import *

norch/nn/modules/linear.py

@@ -1,12 +1,12 @@
-from module import Module
-from parameter import Parameter
+from ..module import Module
+from ..parameter import Parameter
 
 class Linear(Module):
     def __init__(self, input_dim, output_dim):
         super().__init__()
         self.input_dim = input_dim
         self.output_dim = output_dim
-        self.weight = Parameter(shape=[self.input_dim, self.output_dim])
+        self.weight = Parameter(shape=[self.output_dim, self.input_dim])
         self.bias = Parameter(shape=[self.output_dim, 1])
 
     def forward(self, x):

norch/nn/parameter.py

@@ -1,4 +1,5 @@
-from tensor import Tensor
+from norch.tensor import Tensor
+from norch.utils import utils
 import random
 
 class Parameter(Tensor):
@@ -6,9 +7,5 @@ class Parameter(Tensor):
     A parameter is a trainable tensor.
     """
     def __init__(self, shape):
-        data = []
-        for dim_size in reversed(shape):
-            random_dim = [random.random() for _ in range(dim_size)]
-            data.insert(0, random_dim)
-
+        data = utils.generate_random_list(shape=shape)
         super().__init__(data, requires_grad=True)

norch/optim/__init__.py

@@ -0,0 +1 @@
+from .optimizers import *

norch/optim/optimizer.py

@@ -0,0 +1,22 @@
+from abc import ABC
+from norch.tensor import Tensor
+
+class Optimizer(ABC):
+    """
+    Abstract class for optimizers
+    """
+
+    def __init__(self, parameters):
+        if isinstance(parameters, Tensor):
+            raise TypeError("parameters should be an iterable but got {}".format(type(parameters)))
+        elif isinstance(parameters, dict):
+            parameters = parameters.values()
+
+        self.parameters = list(parameters)
+
+    def step(self):
+        raise NotImplementedError
+    
+    def zero_grad(self):
+        for module, name, parameter in self.parameters:
+            parameter.zero_grad()

norch/optim/optimizers/__init__.py

@@ -0,0 +1 @@
+from .sgd import *

norch/optim/optimizers/sgd.py

@@ -0,0 +1,21 @@
+from ..optimizer import Optimizer
+from norch.tensor import Tensor
+
+class SGD(Optimizer):
+    def __init__(self, parameters, lr=1e-1, momentum=0):
+        super().__init__(parameters)
+        self.lr = lr
+        self.momentum = momentum
+        self._cache = {'velocity': [p.zeros_like() for (_, _, p) in self.parameters]}
+
+    def step(self):
+        for i, (module, name, parameter) in enumerate(self.parameters):
+            velocity = self._cache['velocity'][i]
+
+            velocity = self.momentum * velocity - self.lr * parameter.grad
+
+            parameter += velocity
+
+            setattr(module, name, parameter)
+
+            self._cache['velocity'][i] = velocity

norch/tensor.py

@@ -571,4 +571,4 @@ class Tensor:
         result_data.ndim = self.ndim
         result_data.device = self.device
 
-        return result_data
+        return result_data

norch/utils/utils.py

@@ -0,0 +1,15 @@
+import random
+
+def generate_random_list(shape):
+    """
+    Generate a list with random numbers and shape 'shape'
+    """
+    if len(shape) == 0:
+        return []
+    else:
+        inner_shape = shape[1:]
+        if len(inner_shape) == 0:
+            return [random.random()] * shape[0]
+        else:
+            return [generate_random_list(inner_shape) for _ in range(shape[0])]
+        

test.py

@@ -67,19 +67,67 @@ if __name__ == "__main__":
 
     print(a.grad)"""
 
+    import norch.nn as nn
+
+    class MeuModulo(nn.Module):
+        def __init__(self):
+            super(MeuModulo, self).__init__()
+
+            self.layer1 = nn.Linear(10, 2)
+            #self.layer2 = nn.Linear(5, 2)
+            self.sigmoid = nn.Sigmoid()
+
+        def forward(self, x):
+            out = self.layer1(x)
+            out = self.sigmoid(out)
+            #out = self.layer2(out)
+            #out = self.sigmoid(out)
+
+            return out
+        
+    modelo = MeuModulo()
+    input_list = [[0.05 for _ in range(10)]]
+    input = norch.Tensor(input_list).T
+    criterion = nn.MSELoss()
+    optimizer = norch.optim.SGD(modelo.parameters(), lr=0.5)
+
+    target_list = [[0.1 for _ in range(2)]]
+    target = norch.Tensor(target_list).T
+    
+    for epoch in range(5):
+        output = modelo(input)
+        loss = criterion(output, target)
+        optimizer.zero_grad()
+        #print('FORA ANTES: ', modelo.layer1.bias, '\n')
+
+        #print(modelo.layer1.weight.grad)        
+        loss.backward()
+        #print(modelo.layer1.weight.grad)
+        optimizer.step()
+        #print('FORA DEPOIS: ', modelo.layer1.bias, '\n')
+
+        #print("\n\n")
+        #print(modelo.layer1.weight.grad)
+
+        #print(modelo.layer1.weight[0,0])
+        print(loss)
+
+    exit()
+
     #### testar transpose axes!!!! make it contiguous
 
-    tensor1 = norch.Tensor([[[1, 2], [3, 4], [5, 6]],
+    """tensor1 = norch.Tensor([[[1, 2], [3, 4], [5, 6]],
                         [[7, 8], [9, 10], [11, 12]],
                         [[13, 14], [15, 16], [17, 18]],
                         [[19, 20], [21, 22], [23, 24]],
-                        [[25, 26], [27, 28], [29, 30]]], requires_grad=True)
+                        [[25, 26], [27, 28], [29, 0.030]]], requires_grad=True)
     
-    result =  (-10) - tensor1
+    op = nn.Sigmoid()
+    result = op(tensor1)
     result = result.sum()
     result.backward()
     print(tensor1.grad)
-    exit()
+    exit()"""
 
     # Reshape tensor1 to 2x3x5
     reshaped_tensor = tensor1.transpose(1, 0)