Merge pull request #60 from lucasdelimanogueira/tmp

Tmp

README.md

@@ -62,12 +62,88 @@ class MyModel(nn.Module):
         return out
 ```
 
+### 3.3 - Example training
+```python
+import norch
+from norch.utils.data.dataloader import Dataloader
+from norch.norchvision import transforms
+import norch
+import norch.nn as nn
+import norch.optim as optim
+import random
+random.seed(1)
+
+BATCH_SIZE = 32
+device = "cpu"
+epochs = 10
+
+transform = transforms.Sequential(
+    [
+        transforms.ToTensor(),
+        transforms.Reshape([-1, 784, 1])
+    ]
+)
+
+target_transform = transforms.Sequential(
+    [
+        transforms.ToTensor()
+    ]
+)
+
+train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform)
+train_loader = Dataloader(train_data, batch_size = BATCH_SIZE)
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+        self.fc1 = nn.Linear(784, 30)
+        self.sigmoid1 = nn.Sigmoid()
+        self.fc2 = nn.Linear(30, 10)
+        self.sigmoid2 = nn.Sigmoid()
+
+    def forward(self, x):
+        out = self.fc1(x)
+        out = self.sigmoid1(out)
+        out = self.fc2(out)
+        out = self.sigmoid2(out)
+        
+        return out
+
+model = MyModel().to(device)
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.SGD(model.parameters(), lr=0.01)
+loss_list = []
+
+for epoch in range(epochs):    
+    for idx, batch in enumerate(train_loader):
+
+        inputs, target = batch
+
+        inputs = inputs.to(device)
+        target = target.to(device)
+
+        outputs = model(inputs)
+        
+        loss = criterion(outputs, target)
+        
+        optimizer.zero_grad()
+        
+        loss.backward()
+
+        optimizer.step()
+
+    print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}')
+    loss_list.append(loss[0])
+
+```
+
+
 # 4 - Progress
 
 | Development                  | Status      | Feature                                                                |
 | ---------------------------- | ----------- | ---------------------------------------------------------------------- |
-| Operations                   | in progress | <ul><li>[X] GPU Support</li><li>[X] Autograd</li><li>[ ] Broadcasting</li></ul>                 |
-| Loss                         | in progress | <ul><li>[x] MSE</li><li>[ ] Cross Entropy</li></ul>    |
-| Data                         | in progress    | <ul><li>[ ] Dataset</li><li>[ ] Batch</li><li>[ ] Iterator</li></ul>   |
+| Operations                   | in progress | <ul><li>[X] GPU Support</li><li>[X] Autograd</li><li>[X] Broadcasting</li></ul>                 |
+| Loss                         | in progress | <ul><li>[x] MSE</li><li>[X] Cross Entropy</li></ul>    |
+| Data                         | in progress    | <ul><li>[X] Dataset</li><li>[X] Batch</li><li>[X] Iterator</li></ul>   |
 | Convolutional Neural Network | in progress    | <ul><li>[ ] Conv2d</li><li>[ ] MaxPool2d</li><li>[ ] Dropout</li></ul> |
 | Distributed                  | in progress | <ul><li>[ ] Distributed Data Parallel</li></ul>             

norch/__init__.py

@@ -2,7 +2,7 @@ from norch.tensor import Tensor
 from .nn import *
 from .optim import *
 from .utils import *
-from .datasets import *
+from .norchvision import *
 
 __version__ = "0.0.1"
 __author__ = 'Lucas de Lima Nogueira'

norch/autograd/functions.py

@@ -28,7 +28,7 @@ class AddBroadcastedBackward:
         for i in range(len(shape)):
             if shape[i] == 1:
                 gradient = gradient.sum(axis=i, keepdim=True)
-
+        
         return gradient
     
 class SubBackward:
@@ -183,6 +183,7 @@ class DivisionBackward:
         x, y = self.input
         grad_x = gradient / y
         grad_y = -1 * gradient * (x / (y * y))
+
         return [grad_x, grad_y]
     
 class SinBackward:
@@ -291,4 +292,13 @@ class CrossEntropyLossBackward:
 
         return [grad_logits, None]  # targets do not have a gradient
 
+class SigmoidBackward:
+    def __init__(self, input):
+        self.input = [input]
+
+    def backward(self, gradient):
+        sigmoid_x = self.input[0].sigmoid()
+        grad_input = gradient * sigmoid_x * (1 - sigmoid_x)
+
+        return [grad_input]
 

norch/csrc/cpu.cpp

@@ -405,6 +405,22 @@ void sin_tensor_cpu(Tensor* tensor, float* result_data) {
     }
 }
 
+void sigmoid_tensor_cpu(Tensor* tensor, float* result_data) {
+    for (int i = 0; i < tensor->size; i++) {
+        // avoid overflow
+        if (tensor->data[i] >= 0) {
+
+            float z = expf(-tensor->data[i]);
+            result_data[i] = 1 / (1 + z);
+
+        } else {
+
+            float z = expf(tensor->data[i]);
+            result_data[i] = z / (1 + z);
+        }
+    }
+}
+
 void cos_tensor_cpu(Tensor* tensor, float* result_data) {
     for (int i = 0; i < tensor->size; i++) {
         result_data[i] = cosf(tensor->data[i]);

norch/csrc/cpu.h

@@ -32,5 +32,6 @@ void transpose_axes_cpu(Tensor* tensor, float* result_data, int axis1, int axis2
 void assign_tensor_cpu(Tensor* tensor, float* result_data);
 void sin_tensor_cpu(Tensor* tensor, float* result_data);
 void cos_tensor_cpu(Tensor* tensor, float* result_data);
+void sigmoid_tensor_cpu(Tensor* tensor, float* result_data);
 
 #endif /* CPU_H */

norch/csrc/cuda.cu

@@ -764,6 +764,39 @@ __host__ void cos_tensor_cuda(Tensor* tensor, float* result_data) {
     cudaDeviceSynchronize();
 }
 
+__global__ void sigmoid_tensor_cuda_kernel(float* data, float* result_data, int size) {
+    
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (i < size) {
+        // avoid overflow
+        if (data[i] >= 0) {
+
+            float z = expf(-data[i]);
+            result_data[i] = 1 / (1 + z);
+
+        } else {
+
+            float z = expf(data[i]);
+            result_data[i] = z / (1 + z);
+        }
+    }
+}
+
+__host__ void sigmoid_tensor_cuda(Tensor* tensor, float* result_data) {
+    
+    int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK;
+    sigmoid_tensor_cuda_kernel<<<number_of_blocks, THREADS_PER_BLOCK>>>(tensor->data, result_data, tensor->size);
+
+    cudaError_t error = cudaGetLastError();
+    if (error != cudaSuccess) {
+        printf("CUDA error: %s\n", cudaGetErrorString(error));
+        exit(-1);
+    }
+
+    cudaDeviceSynchronize();
+}
+
 
 
 

norch/csrc/cuda.h

@@ -73,6 +73,9 @@
      __global__ void cos_tensor_cuda_kernel(float* data, float* result_data, int size);
     __host__ void cos_tensor_cuda(Tensor* tensor, float* result_data);
 
+    __global__ void sigmoid_tensor_cuda_kernel(float* data, float* result_data, int size);
+    __host__ void sigmoid_tensor_cuda(Tensor* tensor, float* result_data);
+
 
 
 

norch/csrc/tensor.cpp

@@ -1290,6 +1290,43 @@ extern "C" {
         }
     }
 
+    Tensor* sigmoid_tensor(Tensor* tensor) {
+        char* device = (char*)malloc(strlen(tensor->device) + 1);
+        if (device != NULL) {
+            strcpy(device, tensor->device);
+        } else {
+            fprintf(stderr, "Memory allocation failed\n");
+            exit(-1);
+        }
+        int ndim = tensor->ndim;
+        int* shape = (int*)malloc(ndim * sizeof(int));
+        if (shape == NULL) {
+            fprintf(stderr, "Memory allocation failed\n");
+            exit(1);
+        }
+
+        for (int i = 0; i < ndim; i++) {
+            shape[i] = tensor->shape[i];
+        }
+
+        if (strcmp(tensor->device, "cuda") == 0) {
+
+            float* result_data;
+            cudaMalloc((void **)&result_data, tensor->size * sizeof(float));
+            sigmoid_tensor_cuda(tensor, result_data);
+            return create_tensor(result_data, shape, ndim, device);
+        } 
+        else {
+            float* result_data = (float*)malloc(tensor->size * sizeof(float));
+            if (result_data == NULL) {
+                fprintf(stderr, "Memory allocation failed\n");
+                exit(1);
+            }
+            sigmoid_tensor_cpu(tensor, result_data);
+            return create_tensor(result_data, shape, ndim, device);
+        }
+    }
+
     Tensor* transpose_tensor(Tensor* tensor) {
         char* device = (char*)malloc(strlen(tensor->device) + 1);
         if (device != NULL) {

norch/nn/functional.py

@@ -1,9 +1,11 @@
 import math
 import norch
 import numpy as np
+from norch.autograd.functions import *
 
 def sigmoid(x):
-    return 1.0 / (1.0 + (math.e) ** (-x)) 
+    z = x.sigmoid()
+    return z
 
 def softmax(x, dim=None):
     if dim is not None and dim < 0:
@@ -27,7 +29,4 @@ def one_hot_encode(x, num_classes):
         target_idx = int(x.tensor.contents.data[i])
         one_hot[i][target_idx] = 1
 
-    if x.numel < 2:
-        one_hot = one_hot[0]
-
     return norch.Tensor(one_hot)

norch/nn/loss.py

@@ -38,13 +38,16 @@ class CrossEntropyLoss(Loss):
         
         assert isinstance(target, norch.Tensor), \
             "Cross entropy argument 'target' must be Tensor, not {}".format(type(target))
-        
+        if input.ndim > 2:
+            input = input.squeeze(-1)
+
         if input.ndim == 1:
             if target.numel == 1:
                 num_classes = input.shape[0]
                 target = norch.one_hot_encode(target, num_classes)
                 
                 logits = norch.softmax(input, dim=0)
+                target = target.reshape(logits.shape)
                 cost = -(logits.log() * target).sum()
                 
             else:
@@ -52,10 +55,13 @@ class CrossEntropyLoss(Loss):
                 assert target.shape == input.shape, \
                     "Input and target shape does not match: {} and {}".format(input.shape, target.shape)
                 logits = norch.softmax(input, dim=0)
+                target = target.reshape(logits.shape)
                 cost = -(logits.log() * target).sum()
 
 
         elif input.ndim == 2:
+            if target.ndim > 1:
+                target = target.squeeze(-1)
             # batched 
             if target.ndim == 1:
                 # target -> Ground truth class indices:
@@ -65,6 +71,7 @@ class CrossEntropyLoss(Loss):
                 
                 batch_size = input.shape[0]
                 logits = norch.softmax(input, dim=1)
+                target = target.reshape(logits.shape)
                 cost = -(logits.log() * target).sum() / batch_size
 
             else:
@@ -74,6 +81,7 @@ class CrossEntropyLoss(Loss):
                 
                 batch_size = input.shape[0]
                 logits = norch.softmax(input, dim=1)
+                target = target.reshape(logits.shape)
                 cost = -(logits.log() * target).sum() / batch_size
 
         if input.requires_grad:

norch/norchvision/__init__.py

@@ -0,0 +1 @@
+from .datasets import *

norch/norchvision/transforms.py

@@ -0,0 +1,22 @@
+import norch
+
+class ToTensor:
+    def __call__(self, x):
+        return norch.Tensor(x)
+
+class Reshape:
+    def __init__(self, shape):
+        self.shape = shape
+
+    def __call__(self, x):
+        return x.reshape(self.shape)
+    
+class Sequential:
+    def __init__(self, transforms):
+        self.transforms = transforms
+
+    def __call__(self, x):
+        for transform in self.transforms:
+            x = transform(x)
+        return x
+

norch/tensor.py

@@ -178,7 +178,8 @@ class Tensor:
             
             # Only squeeze the specified dimension if its size is 1
             if self.shape[dim] != 1:
-                raise ValueError("Dimension {0} does not have size 1 and cannot be squeezed".format(dim))
+                return self
+                #raise ValueError("Dimension {0} does not have size 1 and cannot be squeezed".format(dim))
             
             # Create the new shape without the specified dimension
             new_shape = self.shape[:dim] + self.shape[dim+1:]
@@ -947,6 +948,25 @@ class Tensor:
         
         return result_data
     
+    def sigmoid(self):
+        Tensor._C.sigmoid_tensor.argtypes = [ctypes.POINTER(CTensor)]
+        Tensor._C.sigmoid_tensor.restype = ctypes.POINTER(CTensor)
+
+        result_tensor_ptr = Tensor._C.sigmoid_tensor(self.tensor)
+
+        result_data = Tensor()
+        result_data.tensor = result_tensor_ptr
+        result_data.shape = self.shape.copy()
+        result_data.ndim = self.ndim
+        result_data.device = self.device
+        result_data.numel = self.numel
+
+        result_data.requires_grad = self.requires_grad
+        if result_data.requires_grad:
+            result_data.grad_fn = SigmoidBackward(self)
+        
+        return result_data
+    
 
     
     def transpose(self, axis1, axis2):

test.py

@@ -1,75 +0,0 @@
-import norch
-from norch.utils.data.dataloader import Dataloader
-import norch
-import norch.nn as nn
-import norch.optim as optim
-import random
-random.seed(1)
-
-
-to_tensor = lambda x: norch.Tensor(x)
-reshape = lambda x: x.reshape([-1, 784])
-transform = lambda x: reshape(to_tensor(x))
-target_transform = lambda x: to_tensor(x)
-
-train_data, test_data = norch.datasets.MNIST.splits(transform=transform, target_transform=target_transform)
-sample, _ = train_data[0]
-
-BATCH_SIZE = 100
-
-train_loader = Dataloader(train_data, batch_size = BATCH_SIZE)
-
-class MyModel(nn.Module):
-    def __init__(self):
-        super(MyModel, self).__init__()
-        self.fc1 = nn.Linear(784, 5)
-        self.sigmoid = nn.Sigmoid()
-        self.fc2 = nn.Linear(5, 10)
-
-    def forward(self, x):
-        out = self.fc1(x)
-        out = self.sigmoid(out)
-        out = self.fc2(out)
-        
-        return out
-
-device = "cpu"
-epochs = 10
-
-model = MyModel().to(device)
-criterion = nn.CrossEntropyLoss()
-optimizer = optim.SGD(model.parameters(), lr=0.001)
-loss_list = []
-
-for epoch in range(epochs):    
-    for idx, batch in enumerate(train_loader):
-        x, target = batch
-
-        x = x.unsqueeze(-1)
-        target = target
-
-        x = x.to(device)
-        target = target.to(device).unsqueeze(-1)
-
-        outputs = model(x)
-        print(outputs.shape, target.shape, x.shape)
-        loss = criterion(outputs, target)
-        
-        optimizer.zero_grad()
-        loss.backward()
-        print('loss_antes', loss)
-
-        print('f1 antes', model.fc1.bias)
-        print('f1 grad_antes', model.fc1.bias.grad)
-        print('f2 antes', model.fc2.bias)
-        print('f2 grad_antes', model.fc2.bias.grad)
-
-        optimizer.step()
-        print('\n')
-
-        print('f1 depois', model.fc1.bias)
-        print('f2 depois', model.fc2.bias)
-        print('\n\n')
-
-    print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}')
-    loss_list.append(loss[0])

tests/test_autograd.py

@@ -601,7 +601,7 @@ class TestTensorAutograd(unittest.TestCase):
 
         torch_result.backward()
         torch_tensor_grad = torch_tensor.grad
-        
+
         self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad))
 
     def test_mse_loss_autograd(self):

tests/test_operations.py

@@ -258,10 +258,6 @@ class TestTensorOperations(unittest.TestCase):
         torch_expected_0 = torch_tensor.squeeze(0)
         self.assertTrue(utils.compare_torch(torch_squeeze_0, torch_expected_0))
 
-        # Squeeze at dim=2 (should raise an error because size is not 1)
-        with self.assertRaises(ValueError):
-            norch_tensor.squeeze(2)
-
         # Create a tensor with a dimension of size 1 in the middle
         norch_tensor_middle_1 = norch.Tensor([[[1, 2]], [[3, 4]]]).to(self.device)  # shape [2, 1, 2]