dataset, dataloader and mnist

norch/__init__.py

@@ -2,6 +2,7 @@ from norch.tensor import Tensor
 from .nn import *
 from .optim import *
 from .utils import *
+from .datasets import *
 
 __version__ = "0.0.1"
 __author__ = 'Lucas de Lima Nogueira'

norch/datasets/__init__.py

@@ -0,0 +1 @@
+from .mnist import *

norch/datasets/mnist.py

@@ -0,0 +1,90 @@
+import gzip
+import os
+import norch
+from norch.utils.data import Dataset
+import numpy as np
+
+class MNIST(Dataset):
+    """
+    Loads training, validation, and test partitions of the mnist dataset
+    (http://yann.lecun.com/exdb/mnist/). If the data is not already contained in data_dir, it will
+    try to download it.
+
+    This dataset contains 60000 training examples, and 10000 test examples of handwritten digits
+    in {0, ..., 9} and corresponding labels. Each handwritten image has an "original" dimension of
+    28x28x1, and is stored row-wise as a string of 784x1 bytes. Pixel values are in range 0 to 255
+    (inclusive).
+
+    Args:
+        data_dir: String. Relative or absolute path of the dataset.
+        devel_size: Integer. Size of the development (validation) dataset partition.
+
+    Returns:
+        X_train: float64 numpy array with shape [784, 60000-devel_size] with values in [0, 1].
+        Y_train: uint8 numpy array with shape [60000-devel_size]. Labels.
+        X_devel: float64 numpy array with shape [784, devel_size] with values in [0, 1].
+        Y_devel: uint8 numpy array with shape [devel_size]. Labels.
+        X_test: float64 numpy array with shape [784, 10000] with values in [0, 1].
+        Y_test: uint8 numpy array with shape [10000]. Labels.
+    """
+
+    urls = ['https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz',
+            'https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz',
+            'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz',
+            'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz',]
+    name = 'mnist-data-py'
+    dirname = 'mnist'
+
+    def __init__(self, path_data, path_label, transform=None, target_transform=None):
+        self.data = self._load_mnist(path_data, header_size=16).reshape((-1, 28, 28))
+        self.labels = self._load_mnist(path_label, header_size=8)
+        self.transform = transform
+        self.target_transform = target_transform
+
+    def _load_mnist(self, path, header_size):
+        with gzip.open(path, 'rb') as f:
+            data = np.frombuffer(f.read(), np.uint8, offset=header_size)
+        return np.asarray(data, dtype=np.uint8)
+
+
+    @classmethod
+    def splits(cls, root='.data', train_data='train-images-idx3-ubyte.gz', train_label='train-labels-idx1-ubyte.gz',
+               test_data='t10k-images-idx3-ubyte.gz', test_label='t10k-labels-idx1-ubyte.gz', **kwargs):
+        r"""
+        Loads training and test partitions of the [mnist dataset](https://www.cs.toronto.edu/~kriz/cifar.html). If
+        the data is not already contained in the ``root`` folder, it will download it.
+
+        Args:
+            root (str): relative or absolute path of the dataset.
+
+        Returns:
+            tuple(Dataset): training and testing datasets
+        """
+        path = os.path.join(root, cls.dirname, cls.name)
+        if not os.path.isdir(path):
+            path = cls.download(root)
+        train_data = os.path.join(path, train_data)
+        train_label = os.path.join(path, train_label)
+        test_data = os.path.join(path, test_data)
+        test_label = os.path.join(path, test_label)
+        return MNIST(train_data, train_label, **kwargs), MNIST(test_data, test_label, **kwargs)
+
+    def __getitem__(self, item):
+        data = self.data[item].tolist()
+        label = self.labels[item].tolist()
+
+        if self.transform is not None:
+            data = self.transform(data)
+        
+        if self.target_transform is not None:
+            label = self.target_transform(label)
+            
+        
+
+        return data, label
+
+    def __setitem__(self, key, value):
+        self.data[key], self.labels[key] = value
+
+    def __len__(self):
+        return len(self.data)

norch/tensor.py

@@ -19,13 +19,18 @@ class Tensor:
     def __init__(self, data=None, device="cpu", requires_grad=False):
 
         if data != None:
+            if isinstance(data, (float, int)):
+                data = [data]
+
             data, shape = self.flatten(data)
-            self.data_ctype = (ctypes.c_float * len(data))(*data)
-            self.shape_ctype = (ctypes.c_int * len(shape))(*shape)
+            
+            self.shape = shape.copy()
+            
+            self.data_ctype = (ctypes.c_float * len(data))(*data.copy())
+            self.shape_ctype = (ctypes.c_int * len(shape))(*shape.copy())
             self.ndim_ctype = ctypes.c_int(len(shape))
             self.device_ctype = device.encode('utf-8')
 
-            self.shape = shape
             self.ndim = len(shape)
             self.device = device
 
@@ -109,6 +114,25 @@ class Tensor:
         return result_data
     
     def reshape(self, new_shape):
+        # Calculate the total number of elements in the tensor
+        total_elements = self.numel
+        
+        # Check for the presence of -1 in new_shape
+        if new_shape.count(-1) > 1:
+            raise ValueError("Only one dimension can be inferred (set to -1).")
+        
+        inferred_dim = None
+        known_dims_product = 1
+        for dim in new_shape:
+            if dim == -1:
+                inferred_dim = dim
+            else:
+                known_dims_product *= dim
+        
+        # Calculate the inferred dimension if -1 is present
+        if inferred_dim == -1:
+            inferred_dim_size = total_elements // known_dims_product
+            new_shape = [inferred_dim_size if dim == -1 else dim for dim in new_shape]
 
         new_shape_ctype = (ctypes.c_int * len(new_shape))(*new_shape)
         new_ndim_ctype = ctypes.c_int(len(new_shape))

norch/utils/__init__.py

@@ -1 +1,2 @@
-from .utils import *
+from .utils import *
+from .data import *

norch/utils/data/__init__.py

@@ -0,0 +1,4 @@
+from .dataset import *
+from .example import *
+from .dataloader import *
+from .batch import *

norch/utils/data/batch.py

@@ -0,0 +1,13 @@
+class Batch(object):
+    """
+    A ``Batch``depends on a ``Dataset`` and is made of ``batch_size`` ``Example``.
+    """
+    def __init__(self, example, batch_size):
+        self.example = example
+        self.batch_size = batch_size
+        
+    def __getitem__(self, item):
+        return self.example[item]
+
+    def __len__(self):
+        return self.batch_size

norch/utils/data/dataloader.py

@@ -0,0 +1,20 @@
+import numpy as np
+from .batch import Batch
+
+
+class Dataloader(object):
+
+    def __init__(self, dataset, batch_size=32):
+        self.dataset = dataset
+        self.batch_size = batch_size
+
+    def __iter__(self):
+        starts = np.arange(0, len(self.dataset), self.batch_size)
+
+        for start in starts:
+            end = start + self.batch_size
+            batch_size = min(end, len(self.dataset)) - start
+            yield Batch(self.dataset[start:end], batch_size)
+
+    def __len__(self):
+        return len(self.dataset) // self.batch_size

norch/utils/data/dataset.py

@@ -0,0 +1,74 @@
+from abc import ABC, abstractmethod
+import os
+from norch.utils import extract_to_dir, download_from_url
+from .example import Example
+import norch
+
+
+class Dataset(ABC):
+    r"""
+    Abstract Dataset class. All dataset for machine learning purposes can inherits from this architecture,
+    for convenience.
+    """
+    urls = []
+    name = ''
+    dirname = ''
+
+    def __init__(self, examples, fields):
+        self.examples = examples
+        self.fields = fields
+
+    @classmethod
+    def splits(cls, train=None, test=None, valid=None, root='.'):
+        raise NotImplementedError
+
+    @classmethod
+    def download(cls, root):
+        r"""Download and unzip a web archive (.zip, .gz, or .tgz).
+
+        Args:
+            root (str): Folder to download data to.
+
+        Returns:
+            string: Path to extracted dataset.
+        """
+        path_dirname = os.path.join(root, cls.dirname)
+        path_name = os.path.join(path_dirname, cls.name)
+        if not os.path.isdir(path_dirname):
+            for url in cls.urls:
+                filename = os.path.basename(url)
+                zpath = os.path.join(path_dirname, filename)
+                if not os.path.isfile(zpath):
+                    if not os.path.exists(os.path.dirname(zpath)):
+                        os.makedirs(os.path.dirname(zpath))
+                    print(f'Download {filename} from {url} to {zpath}')
+                    download_from_url(url, zpath)
+                extract_to_dir(zpath, path_name)
+
+        return path_name
+
+    def __repr__(self):
+        name = self.__class__.__name__
+        string = f"Dataset {name}("
+        tab = "   "
+        for (key, value) in self.__dict__.items():
+            if key[0] != "_":
+                if isinstance(value, Example):
+                    fields = self.fields
+                    for (name, field) in fields:
+                        string += f"\n{tab}({name}): {field.__class__.__name__}" \
+                                  f"(transform={True if field.transform is not None else None}, dtype={field.dtype})"
+                elif isinstance(value, norch.Tensor):
+                    string += f"\n{tab}({key}): {value.__class__.__name__}(shape={value.shape}, dtype={value.dtype})"
+                else:
+                    string += f"\n{tab}({key}): {value.__class__.__name__}"
+        return f'{string}\n)'
+
+    def __getitem__(self, item):
+        return self.examples[item]
+
+    def __setitem__(self, key, value):
+        self.examples[key] = value
+
+    def __len__(self):
+        return len(self.examples)

norch/utils/data/example.py

@@ -0,0 +1,65 @@
+# File: example.py
+# Creation: Wednesday August 19th 2020
+# Author: Arthur Dujardin
+# Contact: arthur.dujardin@ensg.eu
+#          arthurd@ifi.uio.no
+# --------
+# Copyright (c) 2020 Arthur Dujardin
+
+
+class Field(object):
+    r"""
+    A ``Field`` defines the data to process from a raw dataset. It will convert the data into a tensor. The data can
+    be a string, integers, float etc. The data is meant to be preprocessed and the attribute ``transform`` handles
+    the way the user want to process the raw data.
+    """
+
+    def __init__(self, transform=None, dtype=None):
+        self.transform = transform
+        self.dtype = dtype
+
+    def process(self, value):
+        """Applies the transformation and changes the data's type if necessary.
+
+        Args:
+            value (Tensor):
+
+        Returns:
+
+        """
+        if self.transform is not None:
+            value = self.transform(value)
+        if self.dtype is not None:
+            value = value.astype(self.dtype)
+        return value
+
+
+class Example(object):
+    r"""
+    Store a single training / testing example, and store it as an attribute.
+    Highly inspired from PyTorch `Example <https://github.com/pytorch/text/blob/master/torchtext/data/example.py>`__.
+    
+    """
+    @classmethod
+    def fromlist(cls, values, fields):
+        """
+        Add an example from a list of data with respect to the fields.
+
+        Args:
+            values: raw data
+            fields (tuple(string, Field)): fields to preprocess the data on
+
+        Returns:
+            None
+        """
+        example = cls()
+        for (value, field) in zip(values, fields):
+            assert len(field) == 2, f"expected a field template similar to \
+                                    ('name_field', Field()) but got {format(field)}"
+            assert isinstance(field[1], Field), f"expected a field template similar to \
+                                                ('name_field', Field()) but got {format(field)}"
+            name = field[0]
+            value = field[1].process(value)
+            setattr(example, name, value)
+
+        return example

norch/utils/utils.py

@@ -1,4 +1,9 @@
 import random
+import requests
+import tarfile
+import zipfile
+import shutil
+import os
 
 def generate_random_list(shape):
     """
@@ -11,4 +16,105 @@ def generate_random_list(shape):
         if len(inner_shape) == 0:
             return [random.uniform(-1, 1) for _ in range(shape[0])]
         else:
-            return [generate_random_list(inner_shape) for _ in range(shape[0])]
+            return [generate_random_list(inner_shape) for _ in range(shape[0])]
+
+
+def download_from_url(url, save_path, chunk_size=128):
+    """Download a file from an URL.
+
+    Original answer from https://stackoverflow.com/questions/9419162/download-returned-zip-file-from-url
+
+    Args:
+        url (str): path to the URL.
+        save_path (str): path to the saving directory.
+        chunk_size (int): download chunk.
+
+    Returns:
+        None
+    """
+    response = requests.get(url, stream=True)
+    total = response.headers.get('content-length')
+    with open(save_path, 'wb') as f:
+        if total is None:
+            f.write(response.content)
+        else:
+            downloaded = 0
+            total = int(total)
+            for data in response.iter_content(chunk_size=max(int(total / 1000), 1024 * 1024)):
+                downloaded += len(data)
+                f.write(data)
+                progress_bar(downloaded, total, "Downloading...")
+
+
+def extract_to_dir(filename, dirpath='.'):
+    # Does not create folder twice with the same name
+    name, ext = os.path.splitext(filename)
+    # if os.path.basename(name) == os.path.basename(dirpath):
+    #     dirpath = '.'
+    # Extract
+    print(dirpath)
+    print("Extracting...", end="")
+    if tarfile.is_tarfile(filename):
+        tarfile.open(filename, 'r').extractall(dirpath)
+    elif zipfile.is_zipfile(filename):
+        zipfile.ZipFile(filename, 'r').extractall(dirpath)
+    elif ext == '.gz':
+        if not os.path.exists(dirpath):
+            os.mkdir(dirpath)
+        shutil.move(filename, os.path.join(dirpath, os.path.basename(filename)))
+        print(f" | NOTE: gzip files are not extracted, and moved to {dirpath}", end="")
+    # Return the path where the file was extracted
+    print(" | Done !")
+    return os.path.abspath(dirpath)
+
+def progress_bar(current_index, max_index, prefix=None, suffix=None, start_time=None):
+    """Display a progress bar and duration.
+
+    Args:
+        current_index (int): current state index (or epoch number).
+        max_index (int): maximal numbers of state.
+        prefix (str, optional): prefix of the progress bar. The default is None.
+        suffix (str, optional): suffix of the progress bar. The default is None.
+        start_time (float, optional): starting time of the progress bar. If not None, it will display the time
+            spent from the beginning to the current state. The default is None.
+
+    Returns:
+        None. Display the progress bar in the console.
+    """
+    # Add a prefix to the progress bar
+    prefix = "" if prefix is None else str(prefix) + " "
+
+    # Get the percentage
+    percentage = current_index * 100 // max_index
+    loading = "[" + "=" * (percentage // 2) + " " * (50 - percentage // 2) + "]"
+    progress_display = "\r{0}{1:3d}% | {2}".format(prefix, percentage, loading)
+
+    # Add a suffix to the progress bar
+    progress_display += "" if suffix is None else " | " + str(suffix)
+
+    # Add a timer
+    if start_time is not None:
+        time_min, time_sec = get_time(start_time, time.time())
+        time_display = " | Time: {0}m {1}s".format(time_min, time_sec)
+        progress_display += time_display
+
+    # Print the progress bar
+    # TODO: return a string instead
+    print(progress_display, end="{}".format("" if current_index < max_index else " | Done !\n"))
+
+def get_time(start_time, end_time):
+    """Get ellapsed time in minutes and seconds.
+
+    Args:
+        start_time (float): strarting time
+        end_time (float): ending time
+
+    Returns:
+        elapsed_mins (float): elapsed time in minutes
+        elapsed_secs (float): elapsed time in seconds.
+    """
+    elapsed_time = end_time - start_time
+    elapsed_mins = int(elapsed_time / 60)
+    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
+
+    return elapsed_mins, elapsed_secs

test.py

@@ -0,0 +1,75 @@
+import norch
+from norch.utils.data.dataloader import Dataloader
+import norch
+import norch.nn as nn
+import norch.optim as optim
+import random
+random.seed(1)
+
+to_tensor = lambda x: norch.Tensor(x)
+reshape = lambda x: x.reshape([-1, 784])
+transform = lambda x: reshape(to_tensor(x))
+target_transform = lambda x: to_tensor(x)
+
+train_data, test_data = norch.datasets.MNIST.splits(transform=transform, target_transform=target_transform)
+sample, _ = train_data[0]
+
+BATCH_SIZE = 100
+
+train_loader = Dataloader(train_data, batch_size = BATCH_SIZE)
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+        self.fc1 = nn.Linear(784, 10)
+        self.sigmoid = nn.Sigmoid()
+        self.fc2 = nn.Linear(10, 1)
+
+    def forward(self, x):
+        out = self.fc1(x)
+        out = self.sigmoid(out)
+        out = self.fc2(out)
+        
+        return out
+
+device = "cpu"
+epochs = 10
+
+model = MyModel().to(device)
+criterion = nn.MSELoss()
+optimizer = optim.SGD(model.parameters(), lr=0.001)
+loss_list = []
+
+
+for epoch in range(epochs):    
+    for idx, batch in enumerate(train_loader):
+        x, target = batch
+
+        x = x.T
+        target = target.T 
+
+        x = x.to(device)
+        target = target.to(device)
+
+        outputs = model(x)
+
+        loss = criterion(outputs, target)
+        
+        optimizer.zero_grad()
+        loss.backward()
+        print('loss_antes', loss)
+
+        print('f1 antes', model.fc1.bias)
+        print('f1 grad_antes', model.fc1.bias.grad)
+        print('f2 antes', model.fc2.bias)
+        print('f2 grad_antes', model.fc2.bias.grad)
+
+        optimizer.step()
+        print('\n')
+
+        print('f1 depois', model.fc1.bias)
+        print('f2 depois', model.fc2.bias)
+        print('\n\n')
+
+    print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}')
+    loss_list.append(loss[0])