diff --git a/examples/train.ipynb b/examples/train.ipynb index 67d7c2d..b29a29d 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -26,7 +26,64 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Download train-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to .data/mnist/train-images-idx3-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download train-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to .data/mnist/train-labels-idx1-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download t10k-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to .data/mnist/t10k-labels-idx1-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download t10k-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to .data/mnist/t10k-images-idx3-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n" + ] + }, + { + "ename": "BadGzipFile", + "evalue": "Not a gzipped file (b' 6\u001b[0m train_data, test_data \u001b[38;5;241m=\u001b[39m \u001b[43mnorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdatasets\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mMNIST\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplits\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mreshape\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:75\u001b[0m, in \u001b[0;36mMNIST.splits\u001b[0;34m(cls, root, train_data, train_label, test_data, test_label, **kwargs)\u001b[0m\n\u001b[1;32m 73\u001b[0m test_data \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_data)\n\u001b[1;32m 74\u001b[0m test_label \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_label)\n\u001b[0;32m---> 75\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMNIST\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_label\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m, MNIST(test_data, test_label, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:39\u001b[0m, in \u001b[0;36mMNIST.__init__\u001b[0;34m(self, path_data, path_label, transform)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, path_data, path_label, transform\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m---> 39\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_load_mnist\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheader_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m16\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mreshape((\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m28\u001b[39m, \u001b[38;5;241m28\u001b[39m))\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m transform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 41\u001b[0m data \u001b[38;5;241m=\u001b[39m transform(data)\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:47\u001b[0m, in \u001b[0;36mMNIST._load_mnist\u001b[0;34m(self, path, header_size)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_load_mnist\u001b[39m(\u001b[38;5;28mself\u001b[39m, path, header_size):\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m gzip\u001b[38;5;241m.\u001b[39mopen(path, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m---> 47\u001b[0m \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43mheader_size\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Skip the header\u001b[39;00m\n\u001b[1;32m 48\u001b[0m data \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m 50\u001b[0m data_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(data)\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:384\u001b[0m, in \u001b[0;36mGzipFile.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 382\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;241m==\u001b[39m READ:\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_not_closed()\n\u001b[0;32m--> 384\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_buffer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwhence\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moffset\n", + "File \u001b[0;32m/usr/lib/python3.8/_compression.py:143\u001b[0m, in \u001b[0;36mDecompressReader.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;66;03m# Read and discard data until we reach the desired position.\u001b[39;00m\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m offset \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 143\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDEFAULT_BUFFER_SIZE\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m 145\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:479\u001b[0m, in \u001b[0;36m_GzipReader.read\u001b[0;34m(self, size)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_new_member:\n\u001b[1;32m 476\u001b[0m \u001b[38;5;66;03m# If the _new_member flag is set, we have to\u001b[39;00m\n\u001b[1;32m 477\u001b[0m \u001b[38;5;66;03m# jump to the next member, if there is one.\u001b[39;00m\n\u001b[1;32m 478\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_read()\n\u001b[0;32m--> 479\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_gzip_header\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 480\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pos\n\u001b[1;32m 481\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:427\u001b[0m, in \u001b[0;36m_GzipReader._read_gzip_header\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 424\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 426\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m magic \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\037\u001b[39;00m\u001b[38;5;130;01m\\213\u001b[39;00m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 427\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m BadGzipFile(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mNot a gzipped file (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m magic)\n\u001b[1;32m 429\u001b[0m (method, flag,\n\u001b[1;32m 430\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_last_mtime) \u001b[38;5;241m=\u001b[39m struct\u001b[38;5;241m.\u001b[39munpack(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m" ] diff --git a/norch/__init__.py b/norch/__init__.py index ca7be0b..3eb0b7f 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -2,6 +2,7 @@ from norch.tensor import Tensor from .nn import * from .optim import * from .utils import * +from .datasets import * __version__ = "0.0.1" __author__ = 'Lucas de Lima Nogueira' diff --git a/norch/__pycache__/__init__.cpython-38.pyc b/norch/__pycache__/__init__.cpython-38.pyc index d172276..8202d1d 100644 Binary files a/norch/__pycache__/__init__.cpython-38.pyc and b/norch/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index cc2bfbb..9764715 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/autograd/__pycache__/functions.cpython-38.pyc b/norch/autograd/__pycache__/functions.cpython-38.pyc index d56a797..0a7fe5b 100644 Binary files a/norch/autograd/__pycache__/functions.cpython-38.pyc and b/norch/autograd/__pycache__/functions.cpython-38.pyc differ diff --git a/norch/datasets/__init__.py b/norch/datasets/__init__.py new file mode 100644 index 0000000..fe34b11 --- /dev/null +++ b/norch/datasets/__init__.py @@ -0,0 +1 @@ +from .mnist import * \ No newline at end of file diff --git a/norch/datasets/mnist.py b/norch/datasets/mnist.py new file mode 100644 index 0000000..ff11edf --- /dev/null +++ b/norch/datasets/mnist.py @@ -0,0 +1,90 @@ +import gzip +import os +import norch +from norch.utils.data import Dataset +import numpy as np + +class MNIST(Dataset): + """ + Loads training, validation, and test partitions of the mnist dataset + (http://yann.lecun.com/exdb/mnist/). If the data is not already contained in data_dir, it will + try to download it. + + This dataset contains 60000 training examples, and 10000 test examples of handwritten digits + in {0, ..., 9} and corresponding labels. Each handwritten image has an "original" dimension of + 28x28x1, and is stored row-wise as a string of 784x1 bytes. Pixel values are in range 0 to 255 + (inclusive). + + Args: + data_dir: String. Relative or absolute path of the dataset. + devel_size: Integer. Size of the development (validation) dataset partition. + + Returns: + X_train: float64 numpy array with shape [784, 60000-devel_size] with values in [0, 1]. + Y_train: uint8 numpy array with shape [60000-devel_size]. Labels. + X_devel: float64 numpy array with shape [784, devel_size] with values in [0, 1]. + Y_devel: uint8 numpy array with shape [devel_size]. Labels. + X_test: float64 numpy array with shape [784, 10000] with values in [0, 1]. + Y_test: uint8 numpy array with shape [10000]. Labels. + """ + + urls = ['https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz',] + name = 'mnist-data-py' + dirname = 'mnist' + + def __init__(self, path_data, path_label, transform=None, target_transform=None): + self.data = self._load_mnist(path_data, header_size=16).reshape((-1, 28, 28)) + self.labels = self._load_mnist(path_label, header_size=8) + self.transform = transform + self.target_transform = target_transform + + def _load_mnist(self, path, header_size): + with gzip.open(path, 'rb') as f: + data = np.frombuffer(f.read(), np.uint8, offset=header_size) + return np.asarray(data, dtype=np.uint8) + + + @classmethod + def splits(cls, root='.data', train_data='train-images-idx3-ubyte.gz', train_label='train-labels-idx1-ubyte.gz', + test_data='t10k-images-idx3-ubyte.gz', test_label='t10k-labels-idx1-ubyte.gz', **kwargs): + r""" + Loads training and test partitions of the [mnist dataset](https://www.cs.toronto.edu/~kriz/cifar.html). If + the data is not already contained in the ``root`` folder, it will download it. + + Args: + root (str): relative or absolute path of the dataset. + + Returns: + tuple(Dataset): training and testing datasets + """ + path = os.path.join(root, cls.dirname, cls.name) + if not os.path.isdir(path): + path = cls.download(root) + train_data = os.path.join(path, train_data) + train_label = os.path.join(path, train_label) + test_data = os.path.join(path, test_data) + test_label = os.path.join(path, test_label) + return MNIST(train_data, train_label, **kwargs), MNIST(test_data, test_label, **kwargs) + + def __getitem__(self, item): + data = self.data[item].tolist() + label = self.labels[item].tolist() + + if self.transform is not None: + data = self.transform(data) + + if self.target_transform is not None: + label = self.target_transform(label) + + + + return data, label + + def __setitem__(self, key, value): + self.data[key], self.labels[key] = value + + def __len__(self): + return len(self.data) diff --git a/norch/tensor.py b/norch/tensor.py index 807c290..8f3ae12 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -19,13 +19,18 @@ class Tensor: def __init__(self, data=None, device="cpu", requires_grad=False): if data != None: + if isinstance(data, (float, int)): + data = [data] + data, shape = self.flatten(data) - self.data_ctype = (ctypes.c_float * len(data))(*data) - self.shape_ctype = (ctypes.c_int * len(shape))(*shape) + + self.shape = shape.copy() + + self.data_ctype = (ctypes.c_float * len(data))(*data.copy()) + self.shape_ctype = (ctypes.c_int * len(shape))(*shape.copy()) self.ndim_ctype = ctypes.c_int(len(shape)) self.device_ctype = device.encode('utf-8') - self.shape = shape self.ndim = len(shape) self.device = device @@ -109,6 +114,25 @@ class Tensor: return result_data def reshape(self, new_shape): + # Calculate the total number of elements in the tensor + total_elements = self.numel + + # Check for the presence of -1 in new_shape + if new_shape.count(-1) > 1: + raise ValueError("Only one dimension can be inferred (set to -1).") + + inferred_dim = None + known_dims_product = 1 + for dim in new_shape: + if dim == -1: + inferred_dim = dim + else: + known_dims_product *= dim + + # Calculate the inferred dimension if -1 is present + if inferred_dim == -1: + inferred_dim_size = total_elements // known_dims_product + new_shape = [inferred_dim_size if dim == -1 else dim for dim in new_shape] new_shape_ctype = (ctypes.c_int * len(new_shape))(*new_shape) new_ndim_ctype = ctypes.c_int(len(new_shape)) diff --git a/norch/utils/__init__.py b/norch/utils/__init__.py index 90f60fd..c6adfd1 100644 --- a/norch/utils/__init__.py +++ b/norch/utils/__init__.py @@ -1 +1,2 @@ -from .utils import * \ No newline at end of file +from .utils import * +from .data import * \ No newline at end of file diff --git a/norch/utils/__pycache__/utils.cpython-38.pyc b/norch/utils/__pycache__/utils.cpython-38.pyc index 95d3b45..d78c80a 100644 Binary files a/norch/utils/__pycache__/utils.cpython-38.pyc and b/norch/utils/__pycache__/utils.cpython-38.pyc differ diff --git a/norch/utils/data/__init__.py b/norch/utils/data/__init__.py new file mode 100644 index 0000000..2397554 --- /dev/null +++ b/norch/utils/data/__init__.py @@ -0,0 +1,4 @@ +from .dataset import * +from .example import * +from .dataloader import * +from .batch import * \ No newline at end of file diff --git a/norch/utils/data/batch.py b/norch/utils/data/batch.py new file mode 100644 index 0000000..3c9b92c --- /dev/null +++ b/norch/utils/data/batch.py @@ -0,0 +1,13 @@ +class Batch(object): + """ + A ``Batch``depends on a ``Dataset`` and is made of ``batch_size`` ``Example``. + """ + def __init__(self, example, batch_size): + self.example = example + self.batch_size = batch_size + + def __getitem__(self, item): + return self.example[item] + + def __len__(self): + return self.batch_size \ No newline at end of file diff --git a/norch/utils/data/dataloader.py b/norch/utils/data/dataloader.py new file mode 100644 index 0000000..01bd41e --- /dev/null +++ b/norch/utils/data/dataloader.py @@ -0,0 +1,20 @@ +import numpy as np +from .batch import Batch + + +class Dataloader(object): + + def __init__(self, dataset, batch_size=32): + self.dataset = dataset + self.batch_size = batch_size + + def __iter__(self): + starts = np.arange(0, len(self.dataset), self.batch_size) + + for start in starts: + end = start + self.batch_size + batch_size = min(end, len(self.dataset)) - start + yield Batch(self.dataset[start:end], batch_size) + + def __len__(self): + return len(self.dataset) // self.batch_size diff --git a/norch/utils/data/dataset.py b/norch/utils/data/dataset.py new file mode 100644 index 0000000..d38c1ea --- /dev/null +++ b/norch/utils/data/dataset.py @@ -0,0 +1,74 @@ +from abc import ABC, abstractmethod +import os +from norch.utils import extract_to_dir, download_from_url +from .example import Example +import norch + + +class Dataset(ABC): + r""" + Abstract Dataset class. All dataset for machine learning purposes can inherits from this architecture, + for convenience. + """ + urls = [] + name = '' + dirname = '' + + def __init__(self, examples, fields): + self.examples = examples + self.fields = fields + + @classmethod + def splits(cls, train=None, test=None, valid=None, root='.'): + raise NotImplementedError + + @classmethod + def download(cls, root): + r"""Download and unzip a web archive (.zip, .gz, or .tgz). + + Args: + root (str): Folder to download data to. + + Returns: + string: Path to extracted dataset. + """ + path_dirname = os.path.join(root, cls.dirname) + path_name = os.path.join(path_dirname, cls.name) + if not os.path.isdir(path_dirname): + for url in cls.urls: + filename = os.path.basename(url) + zpath = os.path.join(path_dirname, filename) + if not os.path.isfile(zpath): + if not os.path.exists(os.path.dirname(zpath)): + os.makedirs(os.path.dirname(zpath)) + print(f'Download {filename} from {url} to {zpath}') + download_from_url(url, zpath) + extract_to_dir(zpath, path_name) + + return path_name + + def __repr__(self): + name = self.__class__.__name__ + string = f"Dataset {name}(" + tab = " " + for (key, value) in self.__dict__.items(): + if key[0] != "_": + if isinstance(value, Example): + fields = self.fields + for (name, field) in fields: + string += f"\n{tab}({name}): {field.__class__.__name__}" \ + f"(transform={True if field.transform is not None else None}, dtype={field.dtype})" + elif isinstance(value, norch.Tensor): + string += f"\n{tab}({key}): {value.__class__.__name__}(shape={value.shape}, dtype={value.dtype})" + else: + string += f"\n{tab}({key}): {value.__class__.__name__}" + return f'{string}\n)' + + def __getitem__(self, item): + return self.examples[item] + + def __setitem__(self, key, value): + self.examples[key] = value + + def __len__(self): + return len(self.examples) diff --git a/norch/utils/data/example.py b/norch/utils/data/example.py new file mode 100644 index 0000000..a4f4ebf --- /dev/null +++ b/norch/utils/data/example.py @@ -0,0 +1,65 @@ +# File: example.py +# Creation: Wednesday August 19th 2020 +# Author: Arthur Dujardin +# Contact: arthur.dujardin@ensg.eu +# arthurd@ifi.uio.no +# -------- +# Copyright (c) 2020 Arthur Dujardin + + +class Field(object): + r""" + A ``Field`` defines the data to process from a raw dataset. It will convert the data into a tensor. The data can + be a string, integers, float etc. The data is meant to be preprocessed and the attribute ``transform`` handles + the way the user want to process the raw data. + """ + + def __init__(self, transform=None, dtype=None): + self.transform = transform + self.dtype = dtype + + def process(self, value): + """Applies the transformation and changes the data's type if necessary. + + Args: + value (Tensor): + + Returns: + + """ + if self.transform is not None: + value = self.transform(value) + if self.dtype is not None: + value = value.astype(self.dtype) + return value + + +class Example(object): + r""" + Store a single training / testing example, and store it as an attribute. + Highly inspired from PyTorch `Example `__. + + """ + @classmethod + def fromlist(cls, values, fields): + """ + Add an example from a list of data with respect to the fields. + + Args: + values: raw data + fields (tuple(string, Field)): fields to preprocess the data on + + Returns: + None + """ + example = cls() + for (value, field) in zip(values, fields): + assert len(field) == 2, f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + assert isinstance(field[1], Field), f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + name = field[0] + value = field[1].process(value) + setattr(example, name, value) + + return example diff --git a/norch/utils/utils.py b/norch/utils/utils.py index 1652e16..f8b9b30 100644 --- a/norch/utils/utils.py +++ b/norch/utils/utils.py @@ -1,4 +1,9 @@ import random +import requests +import tarfile +import zipfile +import shutil +import os def generate_random_list(shape): """ @@ -11,4 +16,105 @@ def generate_random_list(shape): if len(inner_shape) == 0: return [random.uniform(-1, 1) for _ in range(shape[0])] else: - return [generate_random_list(inner_shape) for _ in range(shape[0])] \ No newline at end of file + return [generate_random_list(inner_shape) for _ in range(shape[0])] + + +def download_from_url(url, save_path, chunk_size=128): + """Download a file from an URL. + + Original answer from https://stackoverflow.com/questions/9419162/download-returned-zip-file-from-url + + Args: + url (str): path to the URL. + save_path (str): path to the saving directory. + chunk_size (int): download chunk. + + Returns: + None + """ + response = requests.get(url, stream=True) + total = response.headers.get('content-length') + with open(save_path, 'wb') as f: + if total is None: + f.write(response.content) + else: + downloaded = 0 + total = int(total) + for data in response.iter_content(chunk_size=max(int(total / 1000), 1024 * 1024)): + downloaded += len(data) + f.write(data) + progress_bar(downloaded, total, "Downloading...") + + +def extract_to_dir(filename, dirpath='.'): + # Does not create folder twice with the same name + name, ext = os.path.splitext(filename) + # if os.path.basename(name) == os.path.basename(dirpath): + # dirpath = '.' + # Extract + print(dirpath) + print("Extracting...", end="") + if tarfile.is_tarfile(filename): + tarfile.open(filename, 'r').extractall(dirpath) + elif zipfile.is_zipfile(filename): + zipfile.ZipFile(filename, 'r').extractall(dirpath) + elif ext == '.gz': + if not os.path.exists(dirpath): + os.mkdir(dirpath) + shutil.move(filename, os.path.join(dirpath, os.path.basename(filename))) + print(f" | NOTE: gzip files are not extracted, and moved to {dirpath}", end="") + # Return the path where the file was extracted + print(" | Done !") + return os.path.abspath(dirpath) + +def progress_bar(current_index, max_index, prefix=None, suffix=None, start_time=None): + """Display a progress bar and duration. + + Args: + current_index (int): current state index (or epoch number). + max_index (int): maximal numbers of state. + prefix (str, optional): prefix of the progress bar. The default is None. + suffix (str, optional): suffix of the progress bar. The default is None. + start_time (float, optional): starting time of the progress bar. If not None, it will display the time + spent from the beginning to the current state. The default is None. + + Returns: + None. Display the progress bar in the console. + """ + # Add a prefix to the progress bar + prefix = "" if prefix is None else str(prefix) + " " + + # Get the percentage + percentage = current_index * 100 // max_index + loading = "[" + "=" * (percentage // 2) + " " * (50 - percentage // 2) + "]" + progress_display = "\r{0}{1:3d}% | {2}".format(prefix, percentage, loading) + + # Add a suffix to the progress bar + progress_display += "" if suffix is None else " | " + str(suffix) + + # Add a timer + if start_time is not None: + time_min, time_sec = get_time(start_time, time.time()) + time_display = " | Time: {0}m {1}s".format(time_min, time_sec) + progress_display += time_display + + # Print the progress bar + # TODO: return a string instead + print(progress_display, end="{}".format("" if current_index < max_index else " | Done !\n")) + +def get_time(start_time, end_time): + """Get ellapsed time in minutes and seconds. + + Args: + start_time (float): strarting time + end_time (float): ending time + + Returns: + elapsed_mins (float): elapsed time in minutes + elapsed_secs (float): elapsed time in seconds. + """ + elapsed_time = end_time - start_time + elapsed_mins = int(elapsed_time / 60) + elapsed_secs = int(elapsed_time - (elapsed_mins * 60)) + + return elapsed_mins, elapsed_secs diff --git a/test.py b/test.py new file mode 100644 index 0000000..dfe3aae --- /dev/null +++ b/test.py @@ -0,0 +1,75 @@ +import norch +from norch.utils.data.dataloader import Dataloader +import norch +import norch.nn as nn +import norch.optim as optim +import random +random.seed(1) + +to_tensor = lambda x: norch.Tensor(x) +reshape = lambda x: x.reshape([-1, 784]) +transform = lambda x: reshape(to_tensor(x)) +target_transform = lambda x: to_tensor(x) + +train_data, test_data = norch.datasets.MNIST.splits(transform=transform, target_transform=target_transform) +sample, _ = train_data[0] + +BATCH_SIZE = 100 + +train_loader = Dataloader(train_data, batch_size = BATCH_SIZE) + +class MyModel(nn.Module): + def __init__(self): + super(MyModel, self).__init__() + self.fc1 = nn.Linear(784, 10) + self.sigmoid = nn.Sigmoid() + self.fc2 = nn.Linear(10, 1) + + def forward(self, x): + out = self.fc1(x) + out = self.sigmoid(out) + out = self.fc2(out) + + return out + +device = "cpu" +epochs = 10 + +model = MyModel().to(device) +criterion = nn.MSELoss() +optimizer = optim.SGD(model.parameters(), lr=0.001) +loss_list = [] + + +for epoch in range(epochs): + for idx, batch in enumerate(train_loader): + x, target = batch + + x = x.T + target = target.T + + x = x.to(device) + target = target.to(device) + + outputs = model(x) + + loss = criterion(outputs, target) + + optimizer.zero_grad() + loss.backward() + print('loss_antes', loss) + + print('f1 antes', model.fc1.bias) + print('f1 grad_antes', model.fc1.bias.grad) + print('f2 antes', model.fc2.bias) + print('f2 grad_antes', model.fc2.bias.grad) + + optimizer.step() + print('\n') + + print('f1 depois', model.fc1.bias) + print('f2 depois', model.fc2.bias) + print('\n\n') + + print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}') + loss_list.append(loss[0]) \ No newline at end of file