diff --git a/README.md b/README.md index 6854be3..5266f7a 100644 --- a/README.md +++ b/README.md @@ -62,12 +62,88 @@ class MyModel(nn.Module): return out ``` +### 3.3 - Example training +```python +import norch +from norch.utils.data.dataloader import Dataloader +from norch.norchvision import transforms +import norch +import norch.nn as nn +import norch.optim as optim +import random +random.seed(1) + +BATCH_SIZE = 32 +device = "cpu" +epochs = 10 + +transform = transforms.Sequential( + [ + transforms.ToTensor(), + transforms.Reshape([-1, 784, 1]) + ] +) + +target_transform = transforms.Sequential( + [ + transforms.ToTensor() + ] +) + +train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform) +train_loader = Dataloader(train_data, batch_size = BATCH_SIZE) + +class MyModel(nn.Module): + def __init__(self): + super(MyModel, self).__init__() + self.fc1 = nn.Linear(784, 30) + self.sigmoid1 = nn.Sigmoid() + self.fc2 = nn.Linear(30, 10) + self.sigmoid2 = nn.Sigmoid() + + def forward(self, x): + out = self.fc1(x) + out = self.sigmoid1(out) + out = self.fc2(out) + out = self.sigmoid2(out) + + return out + +model = MyModel().to(device) +criterion = nn.CrossEntropyLoss() +optimizer = optim.SGD(model.parameters(), lr=0.01) +loss_list = [] + +for epoch in range(epochs): + for idx, batch in enumerate(train_loader): + + inputs, target = batch + + inputs = inputs.to(device) + target = target.to(device) + + outputs = model(inputs) + + loss = criterion(outputs, target) + + optimizer.zero_grad() + + loss.backward() + + optimizer.step() + + print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}') + loss_list.append(loss[0]) + +``` + + # 4 - Progress | Development | Status | Feature | | ---------------------------- | ----------- | ---------------------------------------------------------------------- | -| Operations | in progress | | -| Loss | in progress | | -| Data | in progress | | +| Operations | in progress | | +| Loss | in progress | | +| Data | in progress | | | Convolutional Neural Network | in progress | | | Distributed | in progress | diff --git a/build/cpu.o b/build/cpu.o index 6429509..9e6ddf7 100644 Binary files a/build/cpu.o and b/build/cpu.o differ diff --git a/build/cuda.cu.o b/build/cuda.cu.o index 60b85e1..8c288f1 100644 Binary files a/build/cuda.cu.o and b/build/cuda.cu.o differ diff --git a/build/tensor.o b/build/tensor.o index 10402c8..eb3c24e 100644 Binary files a/build/tensor.o and b/build/tensor.o differ diff --git a/examples/train.ipynb b/examples/train.ipynb index b29a29d..f1e51a5 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -26,20088 +26,139 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 9, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Download train-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to .data/mnist/train-images-idx3-ubyte.gz\n", - "Downloading... 100% | [==================================================] | Done !\n", - ".data/mnist/mnist-data-py\n", - "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", - "Download train-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to .data/mnist/train-labels-idx1-ubyte.gz\n", - "Downloading... 100% | [==================================================] | Done !\n", - ".data/mnist/mnist-data-py\n", - "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", - "Download t10k-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to .data/mnist/t10k-labels-idx1-ubyte.gz\n", - "Downloading... 100% | [==================================================] | Done !\n", - ".data/mnist/mnist-data-py\n", - "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", - "Download t10k-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to .data/mnist/t10k-images-idx3-ubyte.gz\n", - "Downloading... 100% | [==================================================] | Done !\n", - ".data/mnist/mnist-data-py\n", - "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n" - ] - }, - { - "ename": "BadGzipFile", - "evalue": "Not a gzipped file (b' 6\u001b[0m train_data, test_data \u001b[38;5;241m=\u001b[39m \u001b[43mnorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdatasets\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mMNIST\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplits\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mreshape\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:75\u001b[0m, in \u001b[0;36mMNIST.splits\u001b[0;34m(cls, root, train_data, train_label, test_data, test_label, **kwargs)\u001b[0m\n\u001b[1;32m 73\u001b[0m test_data \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_data)\n\u001b[1;32m 74\u001b[0m test_label \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_label)\n\u001b[0;32m---> 75\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMNIST\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_label\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m, MNIST(test_data, test_label, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:39\u001b[0m, in \u001b[0;36mMNIST.__init__\u001b[0;34m(self, path_data, path_label, transform)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, path_data, path_label, transform\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m---> 39\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_load_mnist\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheader_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m16\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mreshape((\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m28\u001b[39m, \u001b[38;5;241m28\u001b[39m))\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m transform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 41\u001b[0m data \u001b[38;5;241m=\u001b[39m transform(data)\n", - "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:47\u001b[0m, in \u001b[0;36mMNIST._load_mnist\u001b[0;34m(self, path, header_size)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_load_mnist\u001b[39m(\u001b[38;5;28mself\u001b[39m, path, header_size):\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m gzip\u001b[38;5;241m.\u001b[39mopen(path, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m---> 47\u001b[0m \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43mheader_size\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Skip the header\u001b[39;00m\n\u001b[1;32m 48\u001b[0m data \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m 50\u001b[0m data_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(data)\n", - "File \u001b[0;32m/usr/lib/python3.8/gzip.py:384\u001b[0m, in \u001b[0;36mGzipFile.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 382\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;241m==\u001b[39m READ:\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_not_closed()\n\u001b[0;32m--> 384\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_buffer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwhence\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moffset\n", - "File \u001b[0;32m/usr/lib/python3.8/_compression.py:143\u001b[0m, in \u001b[0;36mDecompressReader.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;66;03m# Read and discard data until we reach the desired position.\u001b[39;00m\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m offset \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 143\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDEFAULT_BUFFER_SIZE\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m 145\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", - "File \u001b[0;32m/usr/lib/python3.8/gzip.py:479\u001b[0m, in \u001b[0;36m_GzipReader.read\u001b[0;34m(self, size)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_new_member:\n\u001b[1;32m 476\u001b[0m \u001b[38;5;66;03m# If the _new_member flag is set, we have to\u001b[39;00m\n\u001b[1;32m 477\u001b[0m \u001b[38;5;66;03m# jump to the next member, if there is one.\u001b[39;00m\n\u001b[1;32m 478\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_read()\n\u001b[0;32m--> 479\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_gzip_header\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 480\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pos\n\u001b[1;32m 481\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", - "File \u001b[0;32m/usr/lib/python3.8/gzip.py:427\u001b[0m, in \u001b[0;36m_GzipReader._read_gzip_header\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 424\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 426\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m magic \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\037\u001b[39;00m\u001b[38;5;130;01m\\213\u001b[39;00m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 427\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m BadGzipFile(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mNot a gzipped file (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m magic)\n\u001b[1;32m 429\u001b[0m (method, flag,\n\u001b[1;32m 430\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_last_mtime) \u001b[38;5;241m=\u001b[39m struct\u001b[38;5;241m.\u001b[39munpack(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m" ] }, - "execution_count": 9, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "loss_list" + "train_data, test_data = norch.norchvision.datasets.MNIST.splits()\n", + "input_sample, target_sample = train_data[0]\n", + "\n", + "fig = plt.figure(figsize = (20, 10))\n", + "columns = 4\n", + "rows = 2\n", + "\n", + "for i in range(1, columns * rows + 1):\n", + " # Choose a random image\n", + " image_index = random.randint(0, len(train_data))\n", + " image, label = train_data[image_index]\n", + "\n", + " fig.add_subplot(rows, columns, i)\n", + " plt.imshow(np.array(image).reshape(28, 28))\n", + " plt.title(label)\n", + " plt.axis('off')\n", + " \n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training " ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'model' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\n", - "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined" - ] - } - ], + "outputs": [], "source": [ - "model" + "BATCH_SIZE = 32\n", + "device = \"cpu\"\n", + "epochs = 10\n", + "\n", + "transform = transforms.Sequential(\n", + " [\n", + " transforms.ToTensor(),\n", + " transforms.Reshape([-1, 784, 1])\n", + " ]\n", + ")\n", + "\n", + "target_transform = transforms.Sequential(\n", + " [\n", + " transforms.ToTensor()\n", + " ]\n", + ")\n", + "\n", + "train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform)\n", + "train_loader = Dataloader(train_data, batch_size = BATCH_SIZE)\n", + "\n", + "class MyModel(nn.Module):\n", + " def __init__(self):\n", + " super(MyModel, self).__init__()\n", + " self.fc1 = nn.Linear(784, 30)\n", + " self.sigmoid1 = nn.Sigmoid()\n", + " self.fc2 = nn.Linear(30, 10)\n", + " self.sigmoid2 = nn.Sigmoid()\n", + "\n", + " def forward(self, x):\n", + " out = self.fc1(x)\n", + " out = self.sigmoid1(out)\n", + " out = self.fc2(out)\n", + " out = self.sigmoid2(out)\n", + " \n", + " return out\n", + "\n", + "model = MyModel().to(device)\n", + "criterion = nn.CrossEntropyLoss()\n", + "optimizer = optim.SGD(model.parameters(), lr=0.01)\n", + "loss_list = []\n", + "\n", + "for epoch in range(epochs): \n", + " for idx, batch in enumerate(train_loader):\n", + "\n", + " inputs, target = batch\n", + "\n", + " inputs = inputs.to(device)\n", + " target = target.to(device)\n", + "\n", + " outputs = model(inputs)\n", + " \n", + " loss = criterion(outputs, target)\n", + " \n", + " optimizer.zero_grad()\n", + " \n", + " loss.backward()\n", + "\n", + " optimizer.step()\n", + "\n", + " print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}')\n", + " loss_list.append(loss[0])" ] }, { @@ -20116,19 +167,23 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([[-1.4443700313568115,]], device=\"cpu\", requires_grad=True)\n", - "0.1516466453264173\n" - ] + "data": { + "text/plain": [ + "MyModel(\n", + " (fc1): Linear(input_dim=784, output_dim=30, bias=True)\n", + " (sigmoid1): Sigmoid()\n", + " (fc2): Linear(input_dim=30, output_dim=10, bias=True)\n", + " (sigmoid2): Sigmoid()\n", + ")" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "x = 0.4\n", - "input = norch.Tensor([[x]]).T\n", - "print(model(input))\n", - "print(math.pow(math.sin(x), 2))" + "model" ] }, { diff --git a/norch/__init__.py b/norch/__init__.py index 3eb0b7f..838b2db 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -2,7 +2,7 @@ from norch.tensor import Tensor from .nn import * from .optim import * from .utils import * -from .datasets import * +from .norchvision import * __version__ = "0.0.1" __author__ = 'Lucas de Lima Nogueira' diff --git a/norch/__pycache__/__init__.cpython-38.pyc b/norch/__pycache__/__init__.cpython-38.pyc index 8202d1d..a6e6d05 100644 Binary files a/norch/__pycache__/__init__.cpython-38.pyc and b/norch/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index bcc1fe0..718f29e 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/autograd/__pycache__/functions.cpython-38.pyc b/norch/autograd/__pycache__/functions.cpython-38.pyc index 71b5662..154463f 100644 Binary files a/norch/autograd/__pycache__/functions.cpython-38.pyc and b/norch/autograd/__pycache__/functions.cpython-38.pyc differ diff --git a/norch/autograd/functions.py b/norch/autograd/functions.py index c9a393e..917bd94 100644 --- a/norch/autograd/functions.py +++ b/norch/autograd/functions.py @@ -28,7 +28,7 @@ class AddBroadcastedBackward: for i in range(len(shape)): if shape[i] == 1: gradient = gradient.sum(axis=i, keepdim=True) - + return gradient class SubBackward: @@ -183,6 +183,7 @@ class DivisionBackward: x, y = self.input grad_x = gradient / y grad_y = -1 * gradient * (x / (y * y)) + return [grad_x, grad_y] class SinBackward: @@ -291,4 +292,13 @@ class CrossEntropyLossBackward: return [grad_logits, None] # targets do not have a gradient +class SigmoidBackward: + def __init__(self, input): + self.input = [input] + + def backward(self, gradient): + sigmoid_x = self.input[0].sigmoid() + grad_input = gradient * sigmoid_x * (1 - sigmoid_x) + + return [grad_input] diff --git a/norch/csrc/cpu.cpp b/norch/csrc/cpu.cpp index 9584c23..7c96f58 100644 --- a/norch/csrc/cpu.cpp +++ b/norch/csrc/cpu.cpp @@ -405,6 +405,22 @@ void sin_tensor_cpu(Tensor* tensor, float* result_data) { } } +void sigmoid_tensor_cpu(Tensor* tensor, float* result_data) { + for (int i = 0; i < tensor->size; i++) { + // avoid overflow + if (tensor->data[i] >= 0) { + + float z = expf(-tensor->data[i]); + result_data[i] = 1 / (1 + z); + + } else { + + float z = expf(tensor->data[i]); + result_data[i] = z / (1 + z); + } + } +} + void cos_tensor_cpu(Tensor* tensor, float* result_data) { for (int i = 0; i < tensor->size; i++) { result_data[i] = cosf(tensor->data[i]); diff --git a/norch/csrc/cpu.h b/norch/csrc/cpu.h index 83bd83f..1d9547b 100644 --- a/norch/csrc/cpu.h +++ b/norch/csrc/cpu.h @@ -32,5 +32,6 @@ void transpose_axes_cpu(Tensor* tensor, float* result_data, int axis1, int axis2 void assign_tensor_cpu(Tensor* tensor, float* result_data); void sin_tensor_cpu(Tensor* tensor, float* result_data); void cos_tensor_cpu(Tensor* tensor, float* result_data); +void sigmoid_tensor_cpu(Tensor* tensor, float* result_data); #endif /* CPU_H */ diff --git a/norch/csrc/cuda.cu b/norch/csrc/cuda.cu index 7d056ab..da241e4 100644 --- a/norch/csrc/cuda.cu +++ b/norch/csrc/cuda.cu @@ -764,6 +764,39 @@ __host__ void cos_tensor_cuda(Tensor* tensor, float* result_data) { cudaDeviceSynchronize(); } +__global__ void sigmoid_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + + if (i < size) { + // avoid overflow + if (data[i] >= 0) { + + float z = expf(-data[i]); + result_data[i] = 1 / (1 + z); + + } else { + + float z = expf(data[i]); + result_data[i] = z / (1 + z); + } + } +} + +__host__ void sigmoid_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sigmoid_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + diff --git a/norch/csrc/cuda.h b/norch/csrc/cuda.h index 73eb831..7f0f49f 100644 --- a/norch/csrc/cuda.h +++ b/norch/csrc/cuda.h @@ -73,6 +73,9 @@ __global__ void cos_tensor_cuda_kernel(float* data, float* result_data, int size); __host__ void cos_tensor_cuda(Tensor* tensor, float* result_data); + __global__ void sigmoid_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void sigmoid_tensor_cuda(Tensor* tensor, float* result_data); + diff --git a/norch/csrc/tensor.cpp b/norch/csrc/tensor.cpp index 889fff6..6ac29e2 100644 --- a/norch/csrc/tensor.cpp +++ b/norch/csrc/tensor.cpp @@ -1290,6 +1290,43 @@ extern "C" { } } + Tensor* sigmoid_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + sigmoid_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + sigmoid_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + Tensor* transpose_tensor(Tensor* tensor) { char* device = (char*)malloc(strlen(tensor->device) + 1); if (device != NULL) { diff --git a/norch/libtensor.so b/norch/libtensor.so index 81254ad..921564d 100755 Binary files a/norch/libtensor.so and b/norch/libtensor.so differ diff --git a/norch/nn/__pycache__/activation.cpython-38.pyc b/norch/nn/__pycache__/activation.cpython-38.pyc index 0a2693b..cabcf18 100644 Binary files a/norch/nn/__pycache__/activation.cpython-38.pyc and b/norch/nn/__pycache__/activation.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/loss.cpython-38.pyc b/norch/nn/__pycache__/loss.cpython-38.pyc index b8133b9..52d18ca 100644 Binary files a/norch/nn/__pycache__/loss.cpython-38.pyc and b/norch/nn/__pycache__/loss.cpython-38.pyc differ diff --git a/norch/nn/functional.py b/norch/nn/functional.py index 9493591..4c4d757 100644 --- a/norch/nn/functional.py +++ b/norch/nn/functional.py @@ -1,9 +1,11 @@ import math import norch import numpy as np +from norch.autograd.functions import * def sigmoid(x): - return 1.0 / (1.0 + (math.e) ** (-x)) + z = x.sigmoid() + return z def softmax(x, dim=None): if dim is not None and dim < 0: @@ -27,7 +29,4 @@ def one_hot_encode(x, num_classes): target_idx = int(x.tensor.contents.data[i]) one_hot[i][target_idx] = 1 - if x.numel < 2: - one_hot = one_hot[0] - return norch.Tensor(one_hot) \ No newline at end of file diff --git a/norch/nn/loss.py b/norch/nn/loss.py index 8d7b0f9..124da6f 100644 --- a/norch/nn/loss.py +++ b/norch/nn/loss.py @@ -38,13 +38,16 @@ class CrossEntropyLoss(Loss): assert isinstance(target, norch.Tensor), \ "Cross entropy argument 'target' must be Tensor, not {}".format(type(target)) - + if input.ndim > 2: + input = input.squeeze(-1) + if input.ndim == 1: if target.numel == 1: num_classes = input.shape[0] target = norch.one_hot_encode(target, num_classes) logits = norch.softmax(input, dim=0) + target = target.reshape(logits.shape) cost = -(logits.log() * target).sum() else: @@ -52,10 +55,13 @@ class CrossEntropyLoss(Loss): assert target.shape == input.shape, \ "Input and target shape does not match: {} and {}".format(input.shape, target.shape) logits = norch.softmax(input, dim=0) + target = target.reshape(logits.shape) cost = -(logits.log() * target).sum() elif input.ndim == 2: + if target.ndim > 1: + target = target.squeeze(-1) # batched if target.ndim == 1: # target -> Ground truth class indices: @@ -65,6 +71,7 @@ class CrossEntropyLoss(Loss): batch_size = input.shape[0] logits = norch.softmax(input, dim=1) + target = target.reshape(logits.shape) cost = -(logits.log() * target).sum() / batch_size else: @@ -74,6 +81,7 @@ class CrossEntropyLoss(Loss): batch_size = input.shape[0] logits = norch.softmax(input, dim=1) + target = target.reshape(logits.shape) cost = -(logits.log() * target).sum() / batch_size if input.requires_grad: diff --git a/norch/norchvision/__init__.py b/norch/norchvision/__init__.py new file mode 100644 index 0000000..a6ccfac --- /dev/null +++ b/norch/norchvision/__init__.py @@ -0,0 +1 @@ +from .datasets import * \ No newline at end of file diff --git a/norch/datasets/__init__.py b/norch/norchvision/datasets/__init__.py similarity index 100% rename from norch/datasets/__init__.py rename to norch/norchvision/datasets/__init__.py diff --git a/norch/datasets/mnist.py b/norch/norchvision/datasets/mnist.py similarity index 100% rename from norch/datasets/mnist.py rename to norch/norchvision/datasets/mnist.py diff --git a/norch/norchvision/transforms.py b/norch/norchvision/transforms.py new file mode 100644 index 0000000..03dafad --- /dev/null +++ b/norch/norchvision/transforms.py @@ -0,0 +1,22 @@ +import norch + +class ToTensor: + def __call__(self, x): + return norch.Tensor(x) + +class Reshape: + def __init__(self, shape): + self.shape = shape + + def __call__(self, x): + return x.reshape(self.shape) + +class Sequential: + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, x): + for transform in self.transforms: + x = transform(x) + return x + diff --git a/norch/tensor.py b/norch/tensor.py index 1121066..f7f205e 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -178,7 +178,8 @@ class Tensor: # Only squeeze the specified dimension if its size is 1 if self.shape[dim] != 1: - raise ValueError("Dimension {0} does not have size 1 and cannot be squeezed".format(dim)) + return self + #raise ValueError("Dimension {0} does not have size 1 and cannot be squeezed".format(dim)) # Create the new shape without the specified dimension new_shape = self.shape[:dim] + self.shape[dim+1:] @@ -947,6 +948,25 @@ class Tensor: return result_data + def sigmoid(self): + Tensor._C.sigmoid_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.sigmoid_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.sigmoid_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = SigmoidBackward(self) + + return result_data + def transpose(self, axis1, axis2): diff --git a/test.py b/test.py deleted file mode 100644 index 7ad7e2d..0000000 --- a/test.py +++ /dev/null @@ -1,75 +0,0 @@ -import norch -from norch.utils.data.dataloader import Dataloader -import norch -import norch.nn as nn -import norch.optim as optim -import random -random.seed(1) - - -to_tensor = lambda x: norch.Tensor(x) -reshape = lambda x: x.reshape([-1, 784]) -transform = lambda x: reshape(to_tensor(x)) -target_transform = lambda x: to_tensor(x) - -train_data, test_data = norch.datasets.MNIST.splits(transform=transform, target_transform=target_transform) -sample, _ = train_data[0] - -BATCH_SIZE = 100 - -train_loader = Dataloader(train_data, batch_size = BATCH_SIZE) - -class MyModel(nn.Module): - def __init__(self): - super(MyModel, self).__init__() - self.fc1 = nn.Linear(784, 5) - self.sigmoid = nn.Sigmoid() - self.fc2 = nn.Linear(5, 10) - - def forward(self, x): - out = self.fc1(x) - out = self.sigmoid(out) - out = self.fc2(out) - - return out - -device = "cpu" -epochs = 10 - -model = MyModel().to(device) -criterion = nn.CrossEntropyLoss() -optimizer = optim.SGD(model.parameters(), lr=0.001) -loss_list = [] - -for epoch in range(epochs): - for idx, batch in enumerate(train_loader): - x, target = batch - - x = x.unsqueeze(-1) - target = target - - x = x.to(device) - target = target.to(device).unsqueeze(-1) - - outputs = model(x) - print(outputs.shape, target.shape, x.shape) - loss = criterion(outputs, target) - - optimizer.zero_grad() - loss.backward() - print('loss_antes', loss) - - print('f1 antes', model.fc1.bias) - print('f1 grad_antes', model.fc1.bias.grad) - print('f2 antes', model.fc2.bias) - print('f2 grad_antes', model.fc2.bias.grad) - - optimizer.step() - print('\n') - - print('f1 depois', model.fc1.bias) - print('f2 depois', model.fc2.bias) - print('\n\n') - - print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}') - loss_list.append(loss[0]) \ No newline at end of file diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 4739e7c..2265c6d 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -601,7 +601,7 @@ class TestTensorAutograd(unittest.TestCase): torch_result.backward() torch_tensor_grad = torch_tensor.grad - + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) def test_mse_loss_autograd(self): diff --git a/tests/test_operations.py b/tests/test_operations.py index 868f982..ad11717 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -258,10 +258,6 @@ class TestTensorOperations(unittest.TestCase): torch_expected_0 = torch_tensor.squeeze(0) self.assertTrue(utils.compare_torch(torch_squeeze_0, torch_expected_0)) - # Squeeze at dim=2 (should raise an error because size is not 1) - with self.assertRaises(ValueError): - norch_tensor.squeeze(2) - # Create a tensor with a dimension of size 1 in the middle norch_tensor_middle_1 = norch.Tensor([[[1, 2]], [[3, 4]]]).to(self.device) # shape [2, 1, 2]