nn module v1

norch/__init__.py

@@ -1 +1,2 @@
-from norch.tensor import Tensor
+from norch.tensor import Tensor
+from .optim import *

norch/csrc/tensor.cpp

@@ -11,7 +11,6 @@ extern "C" {
 
     Tensor* create_tensor(float* data, int* shape, int ndim, char* device) {
         
-        printf("Creating tensor\n");
         Tensor* tensor = (Tensor*)malloc(sizeof(Tensor));
         if (tensor == NULL) {
             fprintf(stderr, "Memory allocation failed\n");
@@ -44,39 +43,6 @@ extern "C" {
             tensor->strides[i] = stride;
             stride *= shape[i];
         }
-
-        printf("Tensor created successfully\n");
-        printf("Tensor information:\n");
-        printf("Number of dimensions: %d\n", tensor->ndim);
-        printf("Number size: %d\n", tensor->size);
-        printf("Device: %s\n", tensor->device);
-
-        printf("Shape: [");
-        for (int i = 0; i < ndim; i++) {
-            printf("%d", tensor->shape[i]);
-            if (i < ndim - 1) {
-                printf(", ");
-            }
-        }
-        printf("]\n");
-
-        printf("Strides: [");
-        for (int i = 0; i < ndim; i++) {
-            printf("%d", tensor->strides[i]);
-            if (i < ndim - 1) {
-                printf(", ");
-            }
-        }
-        printf("]\n");
-
-        /*printf("Data:\n[");
-        for (int i = 0; i < stride; i++) {
-            printf("%.2f", tensor->data[i]);
-            if (i < stride - 1) {
-                printf(", ");
-            }
-        }
-        printf("]\n\n\n");*/
         
         return tensor;
     }
@@ -98,8 +64,6 @@ extern "C" {
     }
 
     void to_device(Tensor* tensor, char* target_device) {
-        printf("Sending tensor to device: %s\n", target_device);
-
         if ((strcmp(target_device, "cuda") == 0) && (strcmp(tensor->device, "cpu") == 0)) {
             cpu_to_cuda(tensor);
         }

norch/nn/__init__.py

@@ -1,2 +1,3 @@
 from .modules import *
-from .activation import *
+from .activation import *
+from .loss import *

norch/nn/loss.py

@@ -0,0 +1,25 @@
+from .module import Module
+from abc import ABC
+
+class Loss(Module, ABC):
+    "Abstract class for loss functions"
+
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, predictions, labels):
+        raise NotImplementedError
+    
+    def __call__(self, *inputs):
+        return self.forward(*inputs)
+    
+
+class MSELoss(Loss):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, predictions, labels):
+        assert labels.shape == predictions.shape, \
+            "Labels and predictions shape does not match: {} and {}".format(labels.shape, predictions.shape)
+        
+        return ((predictions - labels) **2).sum()

norch/nn/module.py

@@ -16,6 +16,9 @@ class Module(ABC):
         self._grads = OrderedDict()
         self.training = True
 
+    def forward(self, *inputs, **kwargs):
+        raise NotImplementedError
+
     def __call__(self, *inputs, **kwargs):
         return self.forward(*inputs, **kwargs)
 
@@ -36,6 +39,17 @@ class Module(ABC):
             elif isinstance(value, Module):
                 yield from value.parameters()
 
+    def modules(self):
+        yield from self._modules.values()
+
+    def gradients(self):
+        for module in self.modules():
+            yield module._grads
+
+    def zero_grad(self):
+        for parameter in self.parameters():
+            parameter.zero_grad()
+
     def to(self, device):
         for parameter in self.parameters():
             parameter.to(device)

norch/nn/modules/linear.py

@@ -10,7 +10,6 @@ class Linear(Module):
         self.bias = Parameter(shape=[self.output_dim, 1])
 
     def forward(self, x):
-        print(self.weight.shape, x.shape, self.bias.shape, "@@@@@@\n\n\n\n")
         z = self.weight @ x + self.bias
         return z
 

norch/nn/parameter.py

@@ -8,5 +8,4 @@ class Parameter(Tensor):
     """
     def __init__(self, shape):
         data = utils.generate_random_list(shape=shape)
-
         super().__init__(data, requires_grad=True)

norch/optim/__init__.py

@@ -0,0 +1 @@
+from .optimizers import *

norch/optim/optimizer.py

@@ -0,0 +1,22 @@
+from abc import ABC
+from norch.tensor import Tensor
+
+class Optimizer(ABC):
+    """
+    Abstract class for optimizers
+    """
+
+    def __init__(self, parameters):
+        if isinstance(parameters, Tensor):
+            raise TypeError("parameters should be an iterable but got {}".format(type(parameters)))
+        elif isinstance(parameters, dict):
+            parameters = parameters.values()
+
+        self.parameters = list(parameters)
+
+    def step(self):
+        raise NotImplementedError
+    
+    def zero_grad(self):
+        for parameter in self.parameters:
+            parameter.zero_grad()

norch/optim/optimizers/__init__.py

@@ -0,0 +1 @@
+from .sgd import *

norch/optim/optimizers/sgd.py

@@ -0,0 +1,19 @@
+from ..optimizer import Optimizer
+from norch.tensor import Tensor
+
+class SGD(Optimizer):
+    def __init__(self, parameters, lr=1e-1, momentum=0):
+        super().__init__(parameters)
+        self.lr = lr
+        self.momentum = momentum
+        self._cache = {'velocity': [p.zeros_like() for p in self.parameters]}
+
+    def step(self):
+        for i, parameter in enumerate(self.parameters):
+            velocity = self._cache['velocity'][i]
+
+            velocity = self.momentum * velocity - self.lr * parameter.grad
+
+            parameter += velocity
+
+            self._cache['velocity'][i] = velocity

test.py

@@ -73,22 +73,43 @@ if __name__ == "__main__":
         def __init__(self):
             super(MeuModulo, self).__init__()
 
-            self.layer1 = nn.Linear(10, 100)
-            self.layer2 = nn.Linear(100, 2)
+            self.layer1 = nn.Linear(10, 2)
+            #self.layer2 = nn.Linear(5, 2)
             self.sigmoid = nn.Sigmoid()
 
         def forward(self, x):
             out = self.layer1(x)
-            out = self.layer2(out)
             out = self.sigmoid(out)
+            #out = self.layer2(out)
+            #out = self.sigmoid(out)
 
             return out
         
     modelo = MeuModulo()
-    input_list = [[0.01 for _ in range(10)]]
+    input_list = [[0.05 for _ in range(10)]]
     input = norch.Tensor(input_list).T
-    output = modelo(input)
-    print(output)
+    criterion = nn.MSELoss()
+    optimizer = norch.optim.SGD(modelo.parameters(), lr=0.5)
+
+    target_list = [[0.1 for _ in range(2)]]
+    target = norch.Tensor(target_list).T
+
+    for epoch in range(2):
+        output = modelo(input)
+        loss = criterion(output, target)
+        #print('FORA ANTES: ', modelo.layer1.bias, '\n')
+        optimizer.zero_grad()
+        #print(modelo.layer1.weight.grad)        
+        loss.backward()
+        #print(modelo.layer1.weight.grad)
+        optimizer.step()
+        #print('FORA DEPOIS: ', modelo.layer1.bias.data, '\n')
+
+        #print("\n\n")
+        #print(modelo.layer1.weight.grad)
+
+        #print(modelo.layer1.weight[0,0])
+        print(loss)
 
     exit()