unit test dataloader and distributedsampler

README.md

@@ -65,7 +65,7 @@ class MyModel(nn.Module):
 ### 3.3 - Example training
 ```python
 import norch
-from norch.utils.data.dataloader import Dataloader
+from norch.utils.data.dataloader import DataLoader
 from norch.norchvision import transforms
 import norch
 import norch.nn as nn
@@ -77,21 +77,21 @@ BATCH_SIZE = 32
 device = "cuda" #cpu
 epochs = 10
 
-transform = transforms.Sequential(
+transform = transforms.Compose(
     [
         transforms.ToTensor(),
         transforms.Reshape([-1, 784, 1])
     ]
 )
 
-target_transform = transforms.Sequential(
+target_transform = transforms.Compose(
     [
         transforms.ToTensor()
     ]
 )
 
 train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform)
-train_loader = Dataloader(train_data, batch_size = BATCH_SIZE)
+train_loader = DataLoader(train_data, batch_size = BATCH_SIZE)
 
 class MyModel(nn.Module):
     def __init__(self):
@@ -146,4 +146,4 @@ for epoch in range(epochs):
 | Loss                         | in progress | <ul><li>[x] MSE</li><li>[X] Cross Entropy</li></ul>    |
 | Data                         | in progress    | <ul><li>[X] Dataset</li><li>[X] Batch</li><li>[X] Iterator</li></ul>   |
 | Convolutional Neural Network | in progress    | <ul><li>[ ] Conv2d</li><li>[ ] MaxPool2d</li><li>[ ] Dropout</li></ul> |
-| Distributed                  | in progress | <ul><li>[ ] Distributed Data Parallel</li></ul>             
+| Distributed                  | in progress | <ul>><li>[ ] All reduce</li><li>[ ] DistributedDataParallel</li>><li>[ ] DistributedSampler</li></ul>             

examples/train.ipynb

@@ -33,7 +33,7 @@
     "import norch\n",
     "import norch.nn as nn\n",
     "import norch.optim as optim\n",
-    "from norch.utils.data.dataloader import Dataloader\n",
+    "from norch.utils.data.dataloader import DataLoader\n",
     "from norch.norchvision import transforms\n",
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
@@ -102,7 +102,7 @@
     "device = \"cuda\" #cpu\n",
     "epochs = 10\n",
     "\n",
-    "transform = transforms.Sequential(\n",
+    "transform = transforms.Compose(\n",
     "    [\n",
     "        transforms.ToTensor(),\n",
     "        transforms.Reshape([-1, 784, 1])\n",
@@ -116,7 +116,7 @@
     ")\n",
     "\n",
     "train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform)\n",
-    "train_loader = Dataloader(train_data, batch_size = BATCH_SIZE)\n",
+    "train_loader = DataLoader(train_data, batch_size = BATCH_SIZE)\n",
     "\n",
     "class MyModel(nn.Module):\n",
     "    def __init__(self):\n",

norch/norchvision/transforms.py

@@ -11,7 +11,7 @@ class Reshape:
     def __call__(self, x):
         return x.reshape(self.shape)
     
-class Sequential:
+class Compose:
     def __init__(self, transforms):
         self.transforms = transforms
 

norch/utils/data/dataloader.py

@@ -2,7 +2,7 @@ import numpy as np
 from .batch import Batch
 
 
-class Dataloader:
+class DataLoader:
     
     def __init__(self, dataset, batch_size=32, sampler=None):
         self.dataset = dataset

tests/test_dataset.py

@@ -0,0 +1,93 @@
+import unittest
+import norch
+from norch.utils import utils_unittests as utils
+import torch
+import torchvision
+import numpy as np
+from norch.norchvision import transforms as norch_transforms
+from torchvision import transforms as torch_transforms
+
+class TestDataset(unittest.TestCase):
+    def test_dataloader_batchsize_1(self):
+        transforms = norch_transforms.Compose(
+            [
+                norch_transforms.ToTensor(),
+            ]
+        )
+        train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms)
+        train_loader = norch.utils.data.DataLoader(train_data, batch_size = 1)
+
+        labels_norch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_norch.append(utils.to_torch(label))
+
+            if i > 10:
+                break
+
+        transforms = torch_transforms.Compose(
+            [
+                torch_transforms.ToTensor(),
+            ]
+        )
+
+        train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms)
+        train_loader = torch.utils.data.DataLoader(train_data, batch_size = 1)
+
+        labels_torch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_torch.append(label)
+
+            if i > 10:
+                break
+
+        for label_norch, label_torch in zip(labels_norch, labels_torch):
+            self.assertTrue(utils.compare_torch(label_norch, label_torch))
+
+    
+    def test_dataloader_batchsize_32(self):
+        transforms = norch_transforms.Compose(
+            [
+                norch_transforms.ToTensor(),
+            ]
+        )
+        train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms)
+        train_loader = norch.utils.data.DataLoader(train_data, batch_size = 32)
+
+        labels_norch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_norch.append(utils.to_torch(label))
+
+            if i > 10:
+                break
+
+        transforms = torch_transforms.Compose(
+            [
+                torch_transforms.ToTensor(),
+            ]
+        )
+
+        train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms)
+        train_loader = torch.utils.data.DataLoader(train_data, batch_size = 32)
+
+        labels_torch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_torch.append(label)
+
+            if i > 10:
+                break
+
+        for label_norch, label_torch in zip(labels_norch, labels_torch):
+            self.assertTrue(utils.compare_torch(label_norch, label_torch))
+
+
+        
+
+            

tests/test_distributed.py

@@ -0,0 +1,98 @@
+import unittest
+import norch
+from norch.utils import utils_unittests as utils
+import torch
+import torchvision
+import numpy as np
+from norch.norchvision import transforms as norch_transforms
+from torchvision import transforms as torch_transforms
+
+class TestDistributed(unittest.TestCase):
+    def test_distributed_sampler_batch_1(self):
+        transforms = norch_transforms.Compose(
+            [
+                norch_transforms.ToTensor(),
+            ]
+        )
+        train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms)
+        distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2)
+        train_loader = norch.utils.data.DataLoader(train_data, batch_size = 1, sampler=distributed_sampler)
+
+        labels_norch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_norch.append(utils.to_torch(label))
+
+            if i > 10:
+                break
+
+        transforms = torch_transforms.Compose(
+            [
+                torch_transforms.ToTensor(),
+            ]
+        )
+
+        train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms)
+        distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2, shuffle=False)
+        train_loader = torch.utils.data.DataLoader(train_data, batch_size = 1, sampler=distributed_sampler)
+
+        labels_torch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_torch.append(label)
+
+            if i > 10:
+                break
+
+        for label_norch, label_torch in zip(labels_norch, labels_torch):
+            self.assertTrue(utils.compare_torch(label_norch, label_torch))
+
+    
+    def test_distributed_sampler_batch_32(self):
+        transforms = norch_transforms.Compose(
+            [
+                norch_transforms.ToTensor(),
+            ]
+        )
+        train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms)
+        distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2)
+        train_loader = norch.utils.data.DataLoader(train_data, batch_size = 32, sampler=distributed_sampler)
+
+        labels_norch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_norch.append(utils.to_torch(label))
+
+            if i > 10:
+                break
+
+        transforms = torch_transforms.Compose(
+            [
+                torch_transforms.ToTensor(),
+            ]
+        )
+
+        train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms)
+        distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2, shuffle=False)
+        train_loader = torch.utils.data.DataLoader(train_data, batch_size = 32, sampler=distributed_sampler)
+
+        labels_torch = []
+        for i, batch in enumerate(train_loader): 
+
+            image, label = batch
+            labels_torch.append(label)
+
+            if i > 10:
+                break
+
+        for label_norch, label_torch in zip(labels_norch, labels_torch):
+            print(label_norch, label_torch)
+            self.assertTrue(utils.compare_torch(label_norch, label_torch))
+
+
+        
+
+            

tests/test_nn.py

@@ -200,5 +200,4 @@ class TestNNModuleActivationFn(unittest.TestCase):
                 softmax_torch_expected = softmax_fn_torch.forward(torch_input)
 
                 # Compare the results
-                self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected))
-
+                self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected))

train.py

@@ -35,8 +35,9 @@ import norch
 import matplotlib.pyplot as plt
 import numpy as np
 import random
+from norch.norchvision import transforms
 
-train_data, test_data = norch.norchvision.datasets.MNIST.splits()
+train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms.ToTensor())
 train_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=10, rank=2)
 train_loader = norch.utils.data.Dataloader(train_data, batch_size = 1, sampler=train_sampler)
 input_sample, target_sample = train_data[0]