diff --git a/build/cpu.o b/build/cpu.o index 6228ac2..6429509 100644 Binary files a/build/cpu.o and b/build/cpu.o differ diff --git a/build/cuda.cu.o b/build/cuda.cu.o index 376fab2..60b85e1 100644 Binary files a/build/cuda.cu.o and b/build/cuda.cu.o differ diff --git a/build/tensor.o b/build/tensor.o index d330a48..10402c8 100644 Binary files a/build/tensor.o and b/build/tensor.o differ diff --git a/examples/train.ipynb b/examples/train.ipynb index 3a37c54..b29a29d 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -33,16 +33,19963 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch [1/10], Loss: 1.7035\n", - "Epoch [2/10], Loss: 0.7193\n", - "Epoch [3/10], Loss: 0.3068\n", - "Epoch [4/10], Loss: 0.1742\n", - "Epoch [5/10], Loss: 0.1342\n", - "Epoch [6/10], Loss: 0.1232\n", - "Epoch [7/10], Loss: 0.1220\n", - "Epoch [8/10], Loss: 0.1241\n", - "Epoch [9/10], Loss: 0.1270\n", - "Epoch [10/10], Loss: 0.1297\n" + "Download train-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to .data/mnist/train-images-idx3-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download train-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to .data/mnist/train-labels-idx1-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download t10k-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to .data/mnist/t10k-labels-idx1-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download t10k-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to .data/mnist/t10k-images-idx3-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n" + ] + }, + { + "ename": "BadGzipFile", + "evalue": "Not a gzipped file (b' 6\u001b[0m train_data, test_data \u001b[38;5;241m=\u001b[39m \u001b[43mnorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdatasets\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mMNIST\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplits\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mreshape\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:75\u001b[0m, in \u001b[0;36mMNIST.splits\u001b[0;34m(cls, root, train_data, train_label, test_data, test_label, **kwargs)\u001b[0m\n\u001b[1;32m 73\u001b[0m test_data \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_data)\n\u001b[1;32m 74\u001b[0m test_label \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_label)\n\u001b[0;32m---> 75\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMNIST\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_label\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m, MNIST(test_data, test_label, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:39\u001b[0m, in \u001b[0;36mMNIST.__init__\u001b[0;34m(self, path_data, path_label, transform)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, path_data, path_label, transform\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m---> 39\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_load_mnist\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheader_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m16\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mreshape((\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m28\u001b[39m, \u001b[38;5;241m28\u001b[39m))\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m transform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 41\u001b[0m data \u001b[38;5;241m=\u001b[39m transform(data)\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:47\u001b[0m, in \u001b[0;36mMNIST._load_mnist\u001b[0;34m(self, path, header_size)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_load_mnist\u001b[39m(\u001b[38;5;28mself\u001b[39m, path, header_size):\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m gzip\u001b[38;5;241m.\u001b[39mopen(path, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m---> 47\u001b[0m \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43mheader_size\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Skip the header\u001b[39;00m\n\u001b[1;32m 48\u001b[0m data \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m 50\u001b[0m data_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(data)\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:384\u001b[0m, in \u001b[0;36mGzipFile.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 382\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;241m==\u001b[39m READ:\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_not_closed()\n\u001b[0;32m--> 384\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_buffer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwhence\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moffset\n", + "File \u001b[0;32m/usr/lib/python3.8/_compression.py:143\u001b[0m, in \u001b[0;36mDecompressReader.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;66;03m# Read and discard data until we reach the desired position.\u001b[39;00m\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m offset \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 143\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDEFAULT_BUFFER_SIZE\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m 145\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:479\u001b[0m, in \u001b[0;36m_GzipReader.read\u001b[0;34m(self, size)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_new_member:\n\u001b[1;32m 476\u001b[0m \u001b[38;5;66;03m# If the _new_member flag is set, we have to\u001b[39;00m\n\u001b[1;32m 477\u001b[0m \u001b[38;5;66;03m# jump to the next member, if there is one.\u001b[39;00m\n\u001b[1;32m 478\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_read()\n\u001b[0;32m--> 479\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_gzip_header\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 480\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pos\n\u001b[1;32m 481\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:427\u001b[0m, in \u001b[0;36m_GzipReader._read_gzip_header\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 424\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 426\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m magic \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\037\u001b[39;00m\u001b[38;5;130;01m\\213\u001b[39;00m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 427\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m BadGzipFile(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mNot a gzipped file (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m magic)\n\u001b[1;32m 429\u001b[0m (method, flag,\n\u001b[1;32m 430\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_last_mtime) \u001b[38;5;241m=\u001b[39m struct\u001b[38;5;241m.\u001b[39munpack(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m 1\u001b[0m \u001b[43mmodel\u001b[49m\n", + "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined" + ] + } + ], "source": [ "model" ] }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[-1.4443700313568115,]], device=\"cpu\", requires_grad=True)\n", + "0.1516466453264173\n" + ] + } + ], + "source": [ + "x = 0.4\n", + "input = norch.Tensor([[x]]).T\n", + "print(model(input))\n", + "print(math.pow(math.sin(x), 2))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -140,12 +20140,12 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHWCAYAAABACtmGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPhElEQVR4nO3deVxVdeL/8fcFBNzAHVFJXHJPRE0js7Ioc8s119J0mr6Vlea0aKWWTZk17ZqmU2mGqGNu5UqUtjm54miZaWqSCmomICYqnN8f58dVYkfgc+/l9Xw8zoPD4Rx43zNM3jfncz7HYVmWJQAAAABArrxMBwAAAAAAV0dxAgAAAIB8UJwAAAAAIB8UJwAAAADIB8UJAAAAAPJBcQIAAACAfFCcAAAAACAfFCcAAAAAyAfFCQAAAADyQXECgDLi3nvvVWhoaJGOfe655+RwOIo3kIcJDQ1Vz549Tccw6tChQ3I4HPrXv/5lOgoAFDuKEwAY5nA4CrRs2LDBdFQj7r33XlWqVMl0DJeQWUxyW15++WXTEQHAY/mYDgAAZd38+fOzfP7RRx8pJiYm2/bmzZtf0c+ZM2eOMjIyinTss88+q/Hjx1/Rz0fxGTJkiLp3755te3h4uIE0AFA2UJwAwLC77747y+f//e9/FRMTk237X509e1YVKlQo8M8pV65ckfJJko+Pj3x8+CfDVbRt2zbf3w8AQPFiqB4AuIGbb75ZrVq10rZt23TjjTeqQoUKevrppyVJK1asUI8ePVSnTh35+fmpUaNGeuGFF5Senp7le/z1HqfL70eZPXu2GjVqJD8/P1177bXasmVLlmNzusfJ4XDo4Ycf1vLly9WqVSv5+fmpZcuWWrt2bbb8GzZsUPv27eXv769GjRrpvffeK/b7pv7zn/+oXbt2Kl++vGrUqKG7775bR44cybJPQkKCRo4cqXr16snPz0/BwcHq3bu3Dh065Nxn69at6tq1q2rUqKHy5curQYMGGjVqVIFzrF+/Xm3atJG/v79atGihpUuXOr924MABORwOvfHGG9mO++677+RwOBQdHV34F5+DzHuu8spzea677rpL1apVU4UKFXTddddp1apV2fY7d+6cnnvuOTVp0kT+/v4KDg5Wv3799Msvv2TbN7/fKQBwN/z5EADcxO+//65u3bpp8ODBuvvuuxUUFCRJmjt3ripVqqRx48apUqVK+uKLLzRp0iQlJyfr1Vdfzff7LliwQCkpKfq///s/ORwOvfLKK+rXr58OHDiQ71Wqb775RkuXLtVDDz2kypUr6+2331b//v11+PBhVa9eXZK0Y8cO3XHHHQoODtbzzz+v9PR0TZkyRTVr1rzyk/L/zZ07VyNHjtS1116rqVOnKjExUW+99Za+/fZb7dixQ1WqVJEk9e/fXz/88IMeeeQRhYaG6vjx44qJidHhw4edn99+++2qWbOmxo8frypVqujQoUM5lo2c7Nu3T4MGDdIDDzygESNG6MMPP9Rdd92ltWvX6rbbblPDhg3VqVMnRUVF6bHHHstybFRUlCpXrqzevXvn+3POnj2rkydPZttepUqVLFcG88sjSYmJibr++ut19uxZPfroo6pevbrmzZunO++8U0uWLFHfvn0lSenp6erZs6diY2M1ePBgjRkzRikpKYqJidHu3bvVqFEj58+9kt8pAHBZFgDApYwePdr663+eb7rpJkuSNWvWrGz7nz17Ntu2//u//7MqVKhgnTt3zrltxIgRVv369Z2fHzx40JJkVa9e3Tp16pRz+4oVKyxJ1qeffurcNnny5GyZJFm+vr7W/v37ndt27txpSbLeeecd57ZevXpZFSpUsI4cOeLctm/fPsvHxyfb98zJiBEjrIoVK+b69fPnz1u1atWyWrVqZf3555/O7Z999pklyZo0aZJlWZb1xx9/WJKsV199NdfvtWzZMkuStWXLlnxz/VX9+vUtSdYnn3zi3JaUlGQFBwdb4eHhzm3vvfeeJcnas2dPltdQo0YNa8SIEXn+jMz/zXJbNm3aVOg8Y8eOtSRZX3/9tXNbSkqK1aBBAys0NNRKT0+3LMuyPvjgA0uS9frrr2fLlZGRkSVfQX6nAMDdMFQPANyEn5+fRo4cmW17+fLlnespKSk6efKkOnfurLNnz+qnn37K9/sOGjRIVatWdX7euXNnSfbwrfxERkZmudLQunVrBQQEOI9NT0/X559/rj59+qhOnTrO/Ro3bqxu3brl+/0LYuvWrTp+/Lgeeugh+fv7O7f36NFDzZo1cw45K1++vHx9fbVhwwb98ccfOX6vzCtTn332mS5cuFDoLHXq1HFeoZGkgIAADR8+XDt27FBCQoIkaeDAgfL391dUVJRzv3Xr1unkyZMFvm/p/vvvV0xMTLalRYsWhc6zevVqdejQQTfccINzv0qVKun+++/XoUOH9OOPP0qSPvnkE9WoUUOPPPJItjx/HXJ5Jb9TAOCqKE4A4Cbq1q0rX1/fbNt/+OEH9e3bV4GBgQoICFDNmjWdb8CTkpLy/b5XXXVVls8z3/DmVi7yOjbz+Mxjjx8/rj///FONGzfOtl9O24ri119/lSQ1bdo029eaNWvm/Lqfn5+mTZumNWvWKCgoSDfeeKNeeeUVZ4GQpJtuukn9+/fX888/rxo1aqh379768MMPlZaWVqAsjRs3zlYimjRpIknO+6iqVKmiXr16acGCBc59oqKiVLduXd1yyy0F+jlXX321IiMjsy0BAQGFzvPrr7/meO4yZ3HMPH+//PKLmjZtWqBJQq7kdwoAXBXFCQDcxOVXljKdPn1aN910k3bu3KkpU6bo008/VUxMjKZNmyZJBZp+3NvbO8ftlmWV6LEmjB07Vj///LOmTp0qf39/TZw4Uc2bN9eOHTsk2VdOlixZok2bNunhhx/WkSNHNGrUKLVr105nzpwpthzDhw/XgQMH9N133yklJUUrV67UkCFD5OXlGf8su9vvBQAUhGf8FxoAyqgNGzbo999/19y5czVmzBj17NlTkZGRWYZJmVSrVi35+/tr//792b6W07aiqF+/viRp79692b62d+9e59czNWrUSP/4xz+0fv167d69W+fPn9drr72WZZ/rrrtOL774orZu3aqoqCj98MMPWrhwYb5Z9u/fn60c/Pzzz5KUZUbDO+64QzVr1lRUVJSWLVums2fP6p577inQ6y2MguSpX79+jucuc5hn5vlr1KiR9u7dW6QhjADgCShOAODGMv+yf/mb4/Pnz+vdd981FSkLb29vRUZGavny5Tp69Khz+/79+7VmzZpi+Rnt27dXrVq1NGvWrCxD6tasWaM9e/aoR48ekuyZ6M6dO5fl2EaNGqly5crO4/74449sRaNNmzaSVKDhekePHtWyZcucnycnJ+ujjz5SmzZtVLt2bed2Hx8fDRkyRIsXL9bcuXN1zTXXqHXr1oV74QVQkDzdu3fX5s2btWnTJud+qampmj17tkJDQ533TfXv318nT57U9OnTs/0criQBKAuYjhwA3Nj111+vqlWrasSIEXr00UflcDg0f/58l3oj+9xzz2n9+vXq1KmTHnzwQaWnp2v69Olq1aqV4uLiCvQ9Lly4oH/+85/ZtlerVk0PPfSQpk2bppEjR+qmm27SkCFDnNORh4aGOqf9/vnnn3Xrrbdq4MCBatGihXx8fLRs2TIlJiZq8ODBkqR58+bp3XffVd++fdWoUSOlpKRozpw5CggIUPfu3fPN2aRJE/3tb3/Tli1bFBQUpA8++ECJiYn68MMPs+07fPhwvf322/ryyy+dQysLavv27fr444+zbW/UqJEiIiIKlWf8+PGKjo5Wt27d9Oijj6patWqaN2+eDh48qE8++cQ5fHD48OH66KOPNG7cOG3evFmdO3dWamqqPv/8cz300EMFmkYdANwZxQkA3Fj16tX12Wef6R//+IeeffZZVa1aVXfffbduvfVWde3a1XQ8SVK7du20Zs0aPf7445o4caJCQkI0ZcoU7dmzp0Cz/kn2VbSJEydm296oUSM99NBDuvfee1WhQgW9/PLLeuqpp1SxYkX17dtX06ZNc86UFxISoiFDhig2Nlbz58+Xj4+PmjVrpsWLF6t///6S7MkhNm/erIULFyoxMVGBgYHq0KGDoqKi1KBBg3xzXn311XrnnXf0xBNPaO/evWrQoIEWLVqU4/8W7dq1U8uWLbVnzx4NGzasQOchU3R0dI4Pyh0xYkSW4lSQPEFBQfruu+/01FNP6Z133tG5c+fUunVrffrpp86rdZJ99XD16tV68cUXtWDBAn3yySeqXr26brjhBl1zzTWFyg8A7shhudKfJQEAZUafPn30ww8/aN++faajGBMeHq5q1aopNja22L93aGioWrVqpc8++6zYvzcAlEXc4wQAKHF//vlnls/37dun1atX6+abbzYTyAVs3bpVcXFxGj58uOkoAIACYKgeAKDENWzYUPfee68aNmyoX3/9VTNnzpSvr6+efPJJ09FK3e7du7Vt2za99tprCg4O1qBBg0xHAgAUAMUJAFDi7rjjDkVHRyshIUF+fn6KiIjQSy+9pKuvvtp0tFK3ZMkSTZkyRU2bNlV0dLT8/f1NRwIAFAD3OAEAAABAPrjHCQAAAADyQXECAAAAgHyUuXucMjIydPToUVWuXFkOh8N0HAAAAACGWJallJQU1alTx/nA79yUueJ09OhRhYSEmI4BAAAAwEXEx8erXr16ee5T5opT5cqVJdknJyAgwHAaAAAAAKYkJycrJCTE2RHyUuaKU+bwvICAAIoTAAAAgALdwsPkEAAAAACQD4oTAAAAAOSD4gQAAAAA+aA4AQAAAEA+KE4AAAAAkA+KEwAAAADkg+IEAAAAAPmgOAEAAABAPihOAAAAAJAPH9MByrL0dOnrr6Vjx6TgYKlzZ8nb23QqAAAAAH9FcTJk6VJpzBjpt98ubatXT3rrLalfP3O5AAAAAGTHUD0Dli6VBgzIWpok6cgRe/vSpWZyAQAAAMgZxamUpafbV5osK/vXMreNHWvvBwAAAMA1UJxK2ddfZ7/SdDnLkuLj7f0AAAAAuAaKUyk7dqx49wMAAABQ8ihOpSw4uHj3AwAAAFDyKE6lrHNne/Y8hyPnrzscUkiIvR8AAAAA10BxKmXe3vaU41L28pT5+Ztv8jwnAAAAwJVQnAzo109askSqWzfr9ho17O08xwkAAABwLRQnQ/r1kw4dkr78UrrtNnvb7bdTmgAAAABX5GM6QFnm7S3dfLPk5yfFxEjLl0upqVLFiqaTAQAAALgcV5xcwHXXSQ0b2qVpxQrTaQAAAAD8FcXJBTgc0t132+sff2w2CwAAAIDsKE4uYtgw++P69VJiotksAAAAALKiOLmIJk2kDh2k9HRp0SLTaQAAAABcjuLkQhiuBwAAALgmipMLGTTInmlvyxZp717TaQAAAABkoji5kFq1pK5d7fWoKLNZAAAAAFxCcXIxlw/XsyyzWQAAAADYKE4upndvqVIl6eBBadMm02kAAAAASBQnl1OhgtSvn73OJBEAAACAa6A4uaDM4XqLFknnz5vNAgAAAIDi5JJuuUWqXVs6dUpau9Z0GgAAAABGi9NXX32lXr16qU6dOnI4HFq+fHm+x6SlpemZZ55R/fr15efnp9DQUH3wwQclH7YUeXtLQ4fa6wzXAwAAAMwzWpxSU1MVFhamGTNmFPiYgQMHKjY2Vu+//7727t2r6OhoNW3atARTmpE5XG/lSikpyWwWAAAAoKzzMfnDu3Xrpm7duhV4/7Vr12rjxo06cOCAqlWrJkkKDQ0toXRmtWkjtWgh/fij9Mkn0qhRphMBAAAAZZdb3eO0cuVKtW/fXq+88orq1q2rJk2a6PHHH9eff/6Z6zFpaWlKTk7OsrgDhyPrM50AAAAAmONWxenAgQP65ptvtHv3bi1btkxvvvmmlixZooceeijXY6ZOnarAwEDnEhISUoqJr0zmfU4bNkjx8UajAAAAAGWaWxWnjIwMORwORUVFqUOHDurevbtef/11zZs3L9erThMmTFBSUpJziXejBlK/vnTjjZJlSdHRptMAAAAAZZdbFafg4GDVrVtXgYGBzm3NmzeXZVn67bffcjzGz89PAQEBWRZ3wnA9AAAAwDy3Kk6dOnXS0aNHdebMGee2n3/+WV5eXqpXr57BZCVnwADJ11fatUv63/9MpwEAAADKJqPF6cyZM4qLi1NcXJwk6eDBg4qLi9Phw4cl2cPshg8f7tx/6NChql69ukaOHKkff/xRX331lZ544gmNGjVK5cuXN/ESSlzVqlLPnvY6V50AAAAAM4wWp61btyo8PFzh4eGSpHHjxik8PFyTJk2SJB07dsxZoiSpUqVKiomJ0enTp9W+fXsNGzZMvXr10ttvv20kf2nJHK63YIGUnm42CwAAAFAWOSzLskyHKE3JyckKDAxUUlKS29zvlJYm1a4tnT4txcZKt9xiOhEAAADg/grTDdzqHqeyys9PGjjQXme4HgAAAFD6KE5uInO43pIlUh7P+wUAAABQAihObqJTJ/u5Tikp0qefmk4DAAAAlC0UJzfh5SUNG2avM1wPAAAAKF0UJzeSWZzWrJFOnjSbBQAAAChLKE5upEULqW1b6eJFafFi02kAAACAsoPi5GYyJ4lguB4AAABQeihObmbwYPt+p02bpF9+MZ0GAAAAKBsoTm4mOFiKjLTXo6LMZgEAAADKCoqTG7p8uJ5lmc0CAAAAlAUUJzfUt69UoYK0b5+0ZYvpNAAAAIDnozi5oUqVpD597HUmiQAAAABKHsXJTWUO11u4ULpwwWwWAAAAwNNRnNzUbbdJNWtKJ05IMTGm0wAAAACejeLkpnx8pCFD7HWG6wEAAAAli+LkxjKH6y1fLqWkGI0CAAAAeDSKkxtr315q0kT6809p2TLTaQAAAADPRXFyYw5H1mc6AQAAACgZFCc3N2yY/TE2Vjp61GwWAAAAwFNRnNxcw4bS9ddLGRn21OQAAAAAih/FyQMwXA8AAAAoWRQnDzBwoD09+Y4d0g8/mE4DAAAAeB6KkweoXl3q3t1ej4oymwUAAADwRBQnD5E5XC8qyr7fCQAAAEDxoTh5iJ49pYAA6fBh6ZtvTKcBAAAAPAvFyUOULy8NGGCvM0kEAAAAULwoTh4kc7je4sXSuXNmswAAAACehOLkQW66SapXT0pKklavNp0GAAAA8BwUJw/i5SUNHWqvM1wPAAAAKD4UJw+TOVxv1Srp1CmzWQAAAABPQXHyMNdcI7VuLZ0/Ly1ZYjoNAAAA4BkoTh4o86oTw/UAAACA4kFx8kBDhkgOh/T119KhQ6bTAAAAAO6P4uSB6tWTunSx1xcsMJsFAAAA8ARGi9NXX32lXr16qU6dOnI4HFq+fHmBj/3222/l4+OjNm3alFg+d5Y5XG/+fMmyzGYBAAAA3J3R4pSamqqwsDDNmDGjUMedPn1aw4cP16233lpCydxfv36Sv7/000/Sjh2m0wAAAADuzcfkD+/WrZu6detW6OMeeOABDR06VN7e3vlepUpLS1NaWprz8+Tk5EL/PHcUGCjdeae0eLE9SUTbtqYTAQAAAO7L7e5x+vDDD3XgwAFNnjy5QPtPnTpVgYGBziUkJKSEE7qOzOF60dHSxYtmswAAAADuzK2K0759+zR+/Hh9/PHH8vEp2MWyCRMmKCkpybnEx8eXcErX0bWrVL26lJAgffGF6TQAAACA+3Kb4pSenq6hQ4fq+eefV5MmTQp8nJ+fnwICArIsZYWvrzRokL3OM50AAACAonOb4pSSkqKtW7fq4Ycflo+Pj3x8fDRlyhTt3LlTPj4++oJLKjnKHK63dKmUmmo2CwAAAOCujE4OURgBAQHatWtXlm3vvvuuvvjiCy1ZskQNGjQwlMy1XXed1LChdOCAtGKFNHSo6UQAAACA+zFanM6cOaP9+/c7Pz948KDi4uJUrVo1XXXVVZowYYKOHDmijz76SF5eXmrVqlWW42vVqiV/f/9s23GJw2FfdZoyxR6uR3ECAAAACs/oUL2tW7cqPDxc4eHhkqRx48YpPDxckyZNkiQdO3ZMhw8fNhnRIwwbZn9cv15KTDSbBQAAAHBHDsuyLNMhSlNycrICAwOVlJRUpiaK6NhR2rxZeust6dFHTacBAAAAzCtMN3CbySFwZTIniWB2PQAAAKDwKE5lxKBBkre3tGWLtHev6TQAAACAe6E4lRG1atkPxJWkqCizWQAAAAB3Q3EqQy4frle27mwDAAAArgzFqQzp3VuqVEk6eFDatMl0GgAAAMB9UJzKkAoVpH797HUmiQAAAAAKjuJUxmQO11u0SDp/3mwWAAAAwF1QnMqYW26RateWTp2S1q41nQYAAABwDxSnMsbbWxo61F5nuB4AAABQMBSnMihzuN7KlVJSktksAAAAgDugOJVBbdpILVpIaWnSJ5+YTgMAAAC4PopTGeRwZH2mEwAAAIC8UZzKqMz7nDZskOLjjUYBAAAAXB7FqYyqX1+68UbJsqToaNNpAAAAANdGcSrDGK4HAAAAFAzFqQwbMEDy9ZV27ZL+9z/TaQAAAADXRXEqw6pWlXr2tNe56gQAAADkjuJUxmUO11uwQEpPN5sFAAAAcFUUpzKue3epShXpyBFp40bTaQAAAADXRHEq4/z8pIED7XWG6wEAAAA5ozjBOVxvyRLpzz/NZgEAAABcEcUJ6tTJfq5TSor06aem0wAAAACuh+IEeXlJw4bZ6wzXAwAAALKjOEHSpeK0Zo108qTZLAAAAICroThBktSihdS2rXTxorR4sek0AAAAgGuhOMEpc5IIhusBAAAAWVGc4DR4sH2/06ZN0i+/mE4DAAAAuA6KE5yCg6XISHs9KspsFgAAAMCVUJyQxT332B8//liyLLNZAAAAAFdBcUIWffpIFSpI+/ZJW7aYTgMAAAC4BooTsqhUSerb115nkggAAADARnFCNpmz6y1cKF24YDYLAAAA4AooTsgmMlKqVUs6cUKKiTGdBgAAADDPaHH66quv1KtXL9WpU0cOh0PLly/Pc/+lS5fqtttuU82aNRUQEKCIiAitW7eudMKWIT4+0pAh9jrD9QAAAADDxSk1NVVhYWGaMWNGgfb/6quvdNttt2n16tXatm2bunTpol69emnHjh0lnLTsyRyut3y5lJJiNAoAAABgnMOyXGPSaYfDoWXLlqlPnz6FOq5ly5YaNGiQJk2aVKD9k5OTFRgYqKSkJAUEBBQhadlgWVLz5tLevdK8edLw4aYTAQAAAMWrMN3Are9xysjIUEpKiqpVq5brPmlpaUpOTs6yIH8Ox6WrTgzXAwAAQFnn1sXpX//6l86cOaOBAwfmus/UqVMVGBjoXEJCQkoxoXsbOtT+GBsrHT1qNgsAAABgktsWpwULFuj555/X4sWLVatWrVz3mzBhgpKSkpxLfHx8KaZ0bw0bSp06SRkZ9tTkAAAAQFnllsVp4cKFuu+++7R48WJFRkbmua+fn58CAgKyLCg4husBAAAAblicoqOjNXLkSEVHR6tHjx6m43i8u+6SypWTduyQfvjBdBoAAADADKPF6cyZM4qLi1NcXJwk6eDBg4qLi9Phw4cl2cPshl82nduCBQs0fPhwvfbaa+rYsaMSEhKUkJCgpKQkE/HLhOrVpe7d7fWoKLNZAAAAAFOMFqetW7cqPDxc4eHhkqRx48YpPDzcObX4sWPHnCVKkmbPnq2LFy9q9OjRCg4Odi5jxowxkr+syByuFxVl3+8EAAAAlDUu8xyn0sJznArv3DkpKEhKTpY2bpRuvNF0IgAAAODKlZnnOKF0+Pvb9zpJTBIBAACAsonihALJHK63eLF9BQoAAAAoSyhOKJAbb5Tq1ZOSkqTVq02nAQAAAEoXxQkF4uUlDRtmrzNcDwAAAGUNxQkFljlcb9Uq6dQps1kAAACA0kRxQoG1aiWFhUnnz0tLlphOAwAAAJQeihMKJfOqE8P1AAAAUJZQnFAoQ4ZIDof09dfSoUOm0wAAAAClg+KEQqlbV7rlFnt9wQKzWQAAAIDSQnFCoWUO15s/X7Iss1kAAACA0kBxQqH16yf5+0s//STt2GE6DQAAAFDyKE4otIAAqXdve51JIgAAAFAWUJxQJJnD9aKjpYsXzWYBAAAAShrFCUXStatUvbqUkCB98YXpNAAAAEDJojihSMqVkwYPttcZrgcAAABPR3FCkWUO11u6VEpNNZsFAAAAKEkUJxRZx45So0Z2aVqxwnQaAAAAoORQnFBkDselq04M1wMAAIAnozjhigwbZn9cv15KTDSbBQAAACgpFCdckauvtofspadLixaZTgMAAACUDIoTrhjD9QAAAODpKE64YoMGSd7e0pYt0t69ptMAAAAAxY/ihCtWs6Z0xx32elSU2SwAAABASaA4oVhcPlzPssxmAQAAAIobxQnF4s47pUqVpIMHpU2bTKcBAAAAihfFCcWiQgWpf397nUkiAAAA4GkoTig2mcP1Fi2Szp83mwUAAAAoThQnFJsuXaTgYOnUKWntWtNpAAAAgOJDcUKx8faWhg611xmuBwAAAE9CcUKxyhyut3KllJRkNgsAAABQXChOKFZhYVLLllJamvTJJ6bTAAAAAMWD4oRi5XBkfaYTAAAA4AkoTih2mfc5bdggxccbjQIAAAAUC6PF6auvvlKvXr1Up04dORwOLV++PN9jNmzYoLZt28rPz0+NGzfW3LlzSzwnCueqq6SbbpIsS4qONp0GAAAAuHJGi1NqaqrCwsI0Y8aMAu1/8OBB9ejRQ126dFFcXJzGjh2r++67T+vWrSvhpCgshusBAADAkzgsy7JMh5Akh8OhZcuWqU+fPrnu89RTT2nVqlXavXu3c9vgwYN1+vRprS3gg4OSk5MVGBiopKQkBQQEXGls5OL0aSkoyH4Q7s6dUuvWphMBAAAAWRWmG7jVPU6bNm1SZGRklm1du3bVpk2bcj0mLS1NycnJWRaUvCpVpF697HWuOgEAAMDduVVxSkhIUFBQUJZtQUFBSk5O1p9//pnjMVOnTlVgYKBzCQkJKY2o0KXhegsWSOnpZrMAAAAAV8KtilNRTJgwQUlJSc4lnmneSk23blLVqtKRI9LGjabTAAAAAEXnVsWpdu3aSkxMzLItMTFRAQEBKl++fI7H+Pn5KSAgIMuC0uHnJw0caK8zXA8AAADuzK2KU0REhGJjY7Nsi4mJUUREhKFEyE/mcL0lS6RcRlMCAAAALs9ocTpz5ozi4uIUFxcnyZ5uPC4uTocPH5ZkD7MbPny4c/8HHnhABw4c0JNPPqmffvpJ7777rhYvXqzHHnvMRHwUwPXXS6GhUkqK9OmnptMAAAAARWO0OG3dulXh4eEKDw+XJI0bN07h4eGaNGmSJOnYsWPOEiVJDRo00KpVqxQTE6OwsDC99tpr+ve//62uXbsayY/8eXlJw4bZ6wzXAwAAgLtymec4lRae41T69uyRWrSQfHykY8ekGjVMJwIAAAA8+DlOcE/Nm0vt2kkXL0qLF5tOAwAAABRekYpTfHy8fvvtN+fnmzdv1tixYzV79uxiCwbPkjlJBMP1AAAA4I6KVJyGDh2qL7/8UpL9UNrbbrtNmzdv1jPPPKMpU6YUa0B4hsGD7fudNm2SfvnFdBoAAACgcIpUnHbv3q0OHTpIkhYvXqxWrVrpu+++U1RUlObOnVuc+eAhateWbrvNXo+KMpsFAAAAKKwiFacLFy7Iz89PkvT555/rzjvvlCQ1a9ZMx44dK7508CiZw/Xmz5fK1pQkAAAAcHdFKk4tW7bUrFmz9PXXXysmJkZ33HGHJOno0aOqXr16sQaE5+jTR6pQQdq/X9q82XQaAAAAoOCKVJymTZum9957TzfffLOGDBmisLAwSdLKlSudQ/iAv6pUSerb115nkggAAAC4kyI/xyk9PV3JycmqWrWqc9uhQ4dUoUIF1apVq9gCFjee42TW2rVSt272s5yOHpXKlTOdCAAAAGVViT/H6c8//1RaWpqzNP3666968803tXfvXpcuTTAvMlKqVUs6eVJav950GgAAAKBgilScevfurY8++kiSdPr0aXXs2FGvvfaa+vTpo5kzZxZrQHgWHx9pyBB7neF6AAAAcBdFKk7bt29X586dJUlLlixRUFCQfv31V3300Ud6++23izUgPE/m7HrLl0vJyUajAAAAAAVSpOJ09uxZVa5cWZK0fv169evXT15eXrruuuv066+/FmtAeJ527aSmTaVz56Rly0ynAQAAAPJXpOLUuHFjLV++XPHx8Vq3bp1uv/12SdLx48eZcAH5cjguXXViuB4AAADcQZGK06RJk/T4448rNDRUHTp0UEREhCT76lN4eHixBoRnGjrU/hgba8+uBwAAALiyIhWnAQMG6PDhw9q6davWrVvn3H7rrbfqjTfeKLZw8FwNG0qdOkmWJUVHm04DAAAA5K1IxUmSateurfDwcB09elS//fabJKlDhw5q1qxZsYWDZ2O4HgAAANxFkYpTRkaGpkyZosDAQNWvX1/169dXlSpV9MILLygjI6O4M8JD3XWX/QDcuDhp927TaQAAAIDcFak4PfPMM5o+fbpefvll7dixQzt27NBLL72kd955RxMnTizujPBQ1atL3bvb61FRZrMAAAAAeXFYlmUV9qA6depo1qxZuvPOO7NsX7FihR566CEdOXKk2AIWt+TkZAUGBiopKYkZAF3AkiX2laeQEOnQIcmryINHAQAAgMIpTDco0tvUU6dO5XgvU7NmzXTq1KmifEuUUT17SgEBUny89PXXptMAAAAAOStScQoLC9P06dOzbZ8+fbpat259xaFQdvj721ecJCaJAAAAgOsq0lC9jRs3qkePHrrqqqucz3DatGmT4uPjtXr1anXu3LnYgxYXhuq5ng0bpC5dpMBAKSHBLlMAAABASSvxoXo33XSTfv75Z/Xt21enT5/W6dOn1a9fP/3www+aP39+kUKj7LrxRqlePSkpSVq1ynQaAAAAILsiXXHKzc6dO9W2bVulp6cX17csdlxxck3jx0vTpkl9+kjLlplOAwAAgLKgxK84AcUt82G4q1ZJzC8CAAAAV0Nxgkto1UoKC5MuXJD+8x/TaQAAAICsKE5wGZlXnZhdDwAAAK7GpzA79+vXL8+vnz59+kqyoIwbMkR68knpm2+kgwelBg1MJwIAAABshbriFBgYmOdSv359DR8+vKSywsPVrSvdcou9vmCB2SwAAADA5Yp1Vj13wKx6rm3uXGnkSKlZM+nHHyWHw3QiAAAAeCpm1YPb6tfPfgDuTz9J27ebTgMAAADYKE5wKQEBUu/e9jqTRAAAAMBVUJzgcjJn14uOli5eNJsFAAAAkFykOM2YMUOhoaHy9/dXx44dtXnz5jz3f/PNN9W0aVOVL19eISEheuyxx3Tu3LlSSouS1rWrVL26lJgoxcaaTgMAAAC4QHFatGiRxo0bp8mTJ2v79u0KCwtT165ddfz48Rz3X7BggcaPH6/Jkydrz549ev/997Vo0SI9/fTTpZwcJaVcOWnwYHud4XoAAABwBcaL0+uvv66///3vGjlypFq0aKFZs2apQoUK+uCDD3Lc/7vvvlOnTp00dOhQhYaG6vbbb9eQIUPyvUoF95I5XG/pUunMGbNZAAAAAKPF6fz589q2bZsiIyOd27y8vBQZGalNmzbleMz111+vbdu2OYvSgQMHtHr1anXv3j3H/dPS0pScnJxlgevr2FFq1Eg6e1ZascJ0GgAAAJR1RovTyZMnlZ6erqCgoCzbg4KClJCQkOMxQ4cO1ZQpU3TDDTeoXLlyatSokW6++eZch+pNnTo1y0N6Q0JCiv11oPg5HJeuOjFcDwAAAKYZH6pXWBs2bNBLL72kd999V9u3b9fSpUu1atUqvfDCCznuP2HCBCUlJTmX+Pj4Uk6Moho2zP64fr09UQQAAABgio/JH16jRg15e3sr8S/vihMTE1W7du0cj5k4caLuuece3XfffZKka665Rqmpqbr//vv1zDPPyMsraxf08/OTn59fybwAlKirr7aH7H3/vbRwoTRmjOlEAAAAKKuMXnHy9fVVu3btFHvZnNMZGRmKjY1VREREjsecPXs2Wzny9vaWJFmWVXJhYQTD9QAAAOAKjA/VGzdunObMmaN58+Zpz549evDBB5WamqqRI0dKkoYPH64JEyY49+/Vq5dmzpyphQsX6uDBg4qJidHEiRPVq1cvZ4GC5xg0SPL2lrZulX76yXQaAAAAlFVGh+pJ0qBBg3TixAlNmjRJCQkJatOmjdauXeucMOLw4cNZrjA9++yzcjgcevbZZ3XkyBHVrFlTvXr10osvvmjqJaAE1awp3XGHtGqVFBUl5XIrGwAAAFCiHFYZG9+WnJyswMBAJSUlKSAgwHQcFMDChdKQIVJoqHTggD3jHgAAAHClCtMNjA/VA/Jz551SpUrSoUPSd9+ZTgMAAICyiOIEl1ehgtS/v73OJBEAAAAwgeIEt5A5u96iRdL582azAAAAoOyhOMEtdOkiBQdLf/whrVljOg0AAADKGooT3IK3tzR0qL3OcD0AAACUNooT3EbmcL1PP5VOnzYaBQAAAGUMxQluIyxMatlSSkuTXnxRio6WNmyQ0tNNJwMAAICnM/4AXKCgHA6pTRvphx+kf/3r0vZ69aS33pL69TMWDQAAAB6OK05wG0uXSgsWZN9+5Ig0YID9dQAAAKAkUJzgFtLTpTFjJMvK/rXMbWPHMmwPAAAAJYPiBLfw9dfSb7/l/nXLkuLj7f0AAACA4kZxgls4dqx49wMAAAAKg+IEtxAcXLz7AQAAAIVBcYJb6NzZnj3P4ch9n5AQez8AAACguFGc4Ba8ve0px6Xcy9PIkfZ+AAAAQHGjOMFt9OsnLVki1a2bdXuFCvbHN9+Udu8u9VgAAAAoAyhOcCv9+kmHDklffmk/0+nLL6UTJ+whesnJUs+eUkKC6ZQAAADwND6mAwCF5e0t3Xxz1m3LlkkREdK+fdKdd0obNly6EgUAAABcKa44wSNUry6tWiVVqyZt2SLdc4+UkWE6FQAAADwFxQke4+qrpeXLJV9faelSafx404kAAADgKShO8CidO0sffGCvv/qqNHu22TwAAADwDBQneJxhw6Tnn7fXH3pIWr/ebB4AAAC4P4oTPNLEifZ9Tunp0oABTFMOAACAK0NxgkdyOKQ5c6Qbb5RSUqQePZimHAAAAEVHcYLH8vOzpylv0kQ6fNiepvzsWdOpAAAA4I4oTvBo1arZ05RXr25PU3733UxTDgAAgMKjOMHjNW58aZryZcukp54ynQgAAADuhuKEMuGGG6QPP7TX//Uv6b33zOYBAACAe6E4ocwYOlSaMsVeHz1aWrfObB4AAAC4D4oTypRnn5WGD7enKb/rLmnXLtOJAAAA4A4oTihTHA5p9mzpppvsacp79pSOHTOdCgAAAK6O4oQyx89PWro06zTlqammUwEAAMCVUZxQJl0+TfnWrdI999jD9wAAAICcUJxQZjVuLK1YwTTlAAAAyJ9LFKcZM2YoNDRU/v7+6tixozZv3pzn/qdPn9bo0aMVHBwsPz8/NWnSRKtXry6ltPAknTpJc+fa66+9Js2aZTQOAAAAXJTx4rRo0SKNGzdOkydP1vbt2xUWFqauXbvq+PHjOe5//vx53XbbbTp06JCWLFmivXv3as6cOapbt24pJ4enGDJEeuEFe/3hh5mmHAAAANk5LMuyTAbo2LGjrr32Wk2fPl2SlJGRoZCQED3yyCMaP358tv1nzZqlV199VT/99JPKlSuX7/dPS0tTWlqa8/Pk5GSFhIQoKSlJAQEBxfdC4NYsSxo5Upo3T6pcWfr2W+maa0ynAgAAQElKTk5WYGBggbqB0StO58+f17Zt2xQZGenc5uXlpcjISG3atCnHY1auXKmIiAiNHj1aQUFBatWqlV566SWl53Jn/9SpUxUYGOhcQkJCSuS1wL1lTlN+8832NOU9ejBNOQAAAC4xWpxOnjyp9PR0BQUFZdkeFBSkhISEHI85cOCAlixZovT0dK1evVoTJ07Ua6+9pn/+85857j9hwgQlJSU5l/j4+GJ/HfAMvr7SJ5/Y05THx0u9ejFNOQAAAGw+pgMUVkZGhmrVqqXZs2fL29tb7dq105EjR/Tqq69q8uTJ2fb38/OTn5+fgaRwR9WqSatXS9ddJ23bJg0bZpcpb2/TyQAAAGCS0StONWrUkLe3txITE7NsT0xMVO3atXM8Jjg4WE2aNJH3Ze9kmzdvroSEBJ0/f75E86JsaNRIWr7cflDuihXSk0+aTgQAAADTjBYnX19ftWvXTrGxsc5tGRkZio2NVURERI7HdOrUSfv371dGRoZz288//6zg4GD5+vqWeGaUDZdPU/7669LMmUbjAAAAwDDj05GPGzdOc+bM0bx587Rnzx49+OCDSk1N1ciRIyVJw4cP14QJE5z7P/jggzp16pTGjBmjn3/+WatWrdJLL72k0aNHm3oJ8FCDB0uZt8498oi0dq3ZPAAAADDH+D1OgwYN0okTJzRp0iQlJCSoTZs2Wrt2rXPCiMOHD8vL61K/CwkJ0bp16/TYY4+pdevWqlu3rsaMGaOnnnrK1EuAB3v6aWnfPnua8oEDpW++kVq3Np0KAAAApc34c5xKW2Hmagck6fx5qWtXacMGKSRE+v57KTjYdCoAAABcKbd5jhPgDnx9paVLpaZNmaYcAACgrKI4AQVQtaq0apVUo8alacpzeeYyAAAAPBDFCSigRo3s6ckzpyl/4gnTiQAAAFBaKE5AIVx//aVpyt94Q3r3XaNxAAAAUEooTkAhDR4svfiivf7II9KaNWbzAAAAoORRnIAimDBBuvdeKSPDnqZ8507TiQAAAFCSKE5AETgc0nvvSV26SGfOSD17SkePmk4FAACAkkJxAorI11f65BOpWTPpt9+YphwAAMCTUZyAK5A5TXnNmtL27dLQoUxTDgAA4IkoTsAVatjw0jTlK1cyTTkAAIAnojgBxSAiQpo3z15/4w1pxgyzeQAAAFC8KE5AMRk0SHrpJXv90Uel1avN5gEAAEDxoTgBxWj8eGnUKHua8kGDmKYcAADAU1CcgGLkcEgzZ0q33MI05QAAAJ6E4gQUM19facmSS9OU9+xplygAAAC4L4oTUAIun6Z8xw6mKQcAAHB3FCeghFw+Tfmnn0qPP246EQAAAIqK4gSUoIgI6aOP7PU332SacgAAAHdFcQJK2MCBTFMOAADg7ihOQClgmnIAAAD3RnECSoHDIc2aJd16qz3DXo8e0pEjplMBAACgoChOQCkpV86eprx5c7s09erFNOUAAADuguIElKIqVZimHAAAwB1RnIBS1qCBtHKl5O9vT1P+j3+YTgQAAID8UJwAA6677tI05W+9JU2fbjYPAAAA8kZxAgy56y5p6lR7fcwYewgfAAAAXBPFCTDoqaekv/3t0jTlcXGmEwEAACAnFCfAIIdDmjnTnqY8NVXq2ZNpygEAAFwRxQkwLHOa8hYtmKYcAADAVVGcABdQpYr02WdSrVr2NOVDhjBNOQAAgCuhOAEu4vJpyj/7TBo3znQiAAAAZKI4AS6kY0dp/nx7/e23pXfeMZsHAAAANooT4GIGDJBeftleHzvWvvoEAAAAs1yiOM2YMUOhoaHy9/dXx44dtXnz5gIdt3DhQjkcDvXp06dkAwKl7Mknpfvus6cpHzzYvu8JAAAA5hgvTosWLdK4ceM0efJkbd++XWFhYeratauOHz+e53GHDh3S448/rs6dO5dSUqD0OBzSu+9KkZGXpin/7TfTqQAAAMou48Xp9ddf19///neNHDlSLVq00KxZs1ShQgV98MEHuR6Tnp6uYcOG6fnnn1fDhg1LMS1QesqVk/7zH3ua8qNHmaYcAADAJKPF6fz589q2bZsiIyOd27y8vBQZGalNmzbletyUKVNUq1Yt/e1vf8v3Z6SlpSk5OTnLAriLKlWkVavsacrj4uxhe0xTDgAAUPqMFqeTJ08qPT1dQUFBWbYHBQUpISEhx2O++eYbvf/++5ozZ06BfsbUqVMVGBjoXEJCQq44N1CaQkMvTVO+apX02GOmEwEAAJQ9xofqFUZKSoruuecezZkzRzVq1CjQMRMmTFBSUpJziY+PL+GUQPHr2FH6+GN7/Z137KnKAQAAUHp8TP7wGjVqyNvbW4mJiVm2JyYmqnbt2tn2/+WXX3To0CH16tXLuS0jI0OS5OPjo71796pRo0ZZjvHz85Ofn18JpAdKV//+0rRp0lNP2VedGjSw73sCAABAyTN6xcnX11ft2rVTbGysc1tGRoZiY2MVERGRbf9mzZpp165diouLcy533nmnunTpori4OIbhweM98YT097/b05QPGcI05QAAAKXF6BUnSRo3bpxGjBih9u3bq0OHDnrzzTeVmpqqkSNHSpKGDx+uunXraurUqfL391erVq2yHF+lShVJyrYd8EQOhzRjhnTokBQTY09T/v33Ur16ppMBAAB4NuPFadCgQTpx4oQmTZqkhIQEtWnTRmvXrnVOGHH48GF5ebnVrVhAicqcpvz666Uff7TL09dfS5Urm04GAADguRyWZVmmQ5Sm5ORkBQYGKikpSQEBAabjAEV26JA9acTx41L37tKKFZKP8T+FAAAAuI/CdAMu5QBuKjRU+vRTe5ry1auZphwAAKAkUZwAN9ahgz1NucMhTZ/ONOUAAAAlheIEuLnMacolaexY+yoUAAAAihfFCfAAjz8u3X+/ZFnS4MHS9u2mEwEAAHgWihPgATKH6t12m3T2rD3TXny86VQAAACeg+IEeIjMacpbtpSOHbPLU0qK6VQAAACegeIEeJDAQGnVKikoSPrf/+xhexcvmk4FAADg/ihOgIepX19auVIqX96epnzsWPveJwAAABQdxQnwQB06SFFR9r1PM2bY05Snp0sbNkjR0fbH9HTTKQEAANwHxQnwUH37Sq+8Yq+PHWsP3+vSRRo61P4YGiotXWoyIQAAgPugOAEe7B//sGfak6Tff8/6tSNHpAEDKE8AAAAFQXECPFhGhvTjjzl/LfO+p7FjGbYHAACQH4oT4MG+/tq+spQby7Kf9/T116WXCQAAwB1RnAAPduxY8e4HAABQVlGcAA8WHFyw/SZOlN56Szp5smTzAAAAuCuKE+DBOneW6tWzpyXPyy+/2Pc61akj9etnPwfqwoVSiQgAAOAWKE6AB/P2tq8kSdnLk8NhL3PnStOnS+3b22Vp2TKpd2+pbl3psceknTtLPTYAAIDLoTgBHq5fP2nJErsIXa5ePXv7iBHS6NHSli3Srl3S44/bz3w6cUJ6802pTRspPNwuYCdOmHgFAAAA5jksK3NS4rIhOTlZgYGBSkpKUkBAgOk4QKlJT7dnzzt2zL73qXNn+4pUTi5elNats69GrVwpnT9vb/fxkXr2lO69V+reXSpXrrTSAwAAFL/CdAOKE4A8/f67tHChXaK2br20vWZNadgwu0SFhZlKBwAAUHQUpzxQnICi271bmjdPmj9fSky8tL1NG7tADR1qFyoAAAB3QHHKA8UJuHIM5QMAAJ6A4pQHihNQvBjKBwAA3BXFKQ8UJ6DkMJQPAAC4E4pTHihOQMm7eFFav96+CrViBUP5AACAa6I45YHiBJSuU6cuDeXbsuXSdobyAQAA0yhOeaA4Aeb88MOloXwJCZe2M5QPAACYQHHKA8UJMC+voXw9elwayufrazIlAADwdBSnPFCcANeS21C+GjUuDeVr08ZQOAAA4NEoTnmgOAGuK7ehfGFhl4by1aplLB4AAPAwFKc8UJwA18dQPgAAUBooTnmgOAHuhaF8AACgpFCc8kBxAtwXQ/kAAEBxKkw38CqlTHmaMWOGQkND5e/vr44dO2rz5s257jtnzhx17txZVatWVdWqVRUZGZnn/gA8R8uW0iuvSPHx0qpV0l132cP1du6UHntMqltX6tNHWr780vA+AACA4mC8OC1atEjjxo3T5MmTtX37doWFhalr1646fvx4jvtv2LBBQ4YM0ZdffqlNmzYpJCREt99+u44cOVLKyQGY4uNj3+O0eLF07Jg0Y4Z07bX2vVErVkh9+9olauxYKS7OdFoAAOAJjA/V69ixo6699lpNnz5dkpSRkaGQkBA98sgjGj9+fL7Hp6enq2rVqpo+fbqGDx+e7/4M1QM8148/2kP5PvqIoXwAACB/bjNU7/z589q2bZsiIyOd27y8vBQZGalNmzYV6HucPXtWFy5cULVq1XL8elpampKTk7MsADxTixbStGn2UL7Vq6WBAxnKBwAAiofR4nTy5Emlp6crKCgoy/agoCAlXP7n4jw89dRTqlOnTpbydbmpU6cqMDDQuYSEhFxxbgCuzcdH6tZNWrTIHsr37rtShw4M5QMAAEVn/B6nK/Hyyy9r4cKFWrZsmfz9/XPcZ8KECUpKSnIu8fHxpZwSgEnVqkkPPih9/709K9+TT0q1a0snT0pvvSWFh9tD+d54Q8rp1sr0dGnDBik62v6Ynl7arwAAALgCo8WpRo0a8vb2VmJiYpbtiYmJql27dp7H/utf/9LLL7+s9evXq3Xr1rnu5+fnp4CAgCwLgLIpt6F8//ufNG6cfRWqd+9LQ/mWLpVCQ6UuXez7o7p0sT9futTwCwEAAKXOaHHy9fVVu3btFBsb69yWkZGh2NhYRURE5HrcK6+8ohdeeEFr165V+/btSyMqAA+S11C+lSvtoXw1akj9+0u//Zb12CNHpAEDKE8AAJQ1xofqjRs3TnPmzNG8efO0Z88ePfjgg0pNTdXIkSMlScOHD9eECROc+0+bNk0TJ07UBx98oNDQUCUkJCghIUFnzpwx9RIAuLHchvKlpOS8f+Y8pGPHMmwPAICyxMd0gEGDBunEiROaNGmSEhIS1KZNG61du9Y5YcThw4fl5XWp382cOVPnz5/XgAEDsnyfyZMn67nnnivN6AA8TOZQvttvl3KZb0aSXZ7i46Wrr5aaN5dCQrIv9epJudx6CQAA3JDx5ziVNp7jBCA/0dH2PU1XqmbNnEtV5lKnjlSu3JX/HAAAUDSF6QbGrzgBgKsJDi7Yfi+/LFWtal99+uty7px04oS9bN+e8/FeXvawwLzKVVCQ5O1dfK8NAAAUDVecAOAv0tPt2fOOHLl0T9PlHA57KN7BgzmXGsuSTp3KuVBlLr/9Jl24kH8WHx97tr+8ylWNGnYmAABQOFxxAoAr4O1tP+NpwAC7kFxenjILyptv5n4lyOGQqle3lzZtct4nI8N+blRe5eroUXumv19/tZfc+PnZRS6vclWlCuUKAIArwRUnAMjF0qXSmDFZpyQPCbFLU79+Jf/zL160p0vP7YpVfLyUkFCw71WxYt7FKiREqlSpePOnp0tff22/huBgqXNnhh0CAFxLYboBxQkA8uDqb/7Pn7eHFOZ15er33wv2vapUybtYFWamwJxKZ7169pW80iidAFDSXP3fB1flaueN4pQHihOAsubs2UtXqHJbkpML9r1ymykwc6hg3brSp5/awxz/+q9L5lDBJUsoT3lxtTcV7oLzVnScu8Ljj0NF44rnjeKUB4oTAGSXnJx1CGBOy9mzBfteXl72PVy5qVZNmj5dqlDBvoKV0+Lnl3W9rNyf5YpvKtwB563oOHeFt3QpfxwqClc9bxSnPFCcAKDwLEv644/8Zwo8f75kfv7lRaq0l9Iqbq76psLVcd6KjnNXeJmzrl5eNC+X16yrlmUvGRlml/T00v+ZFy/a9wfnNrohv9lqSxLFKQ8UJwAoGRkZ0uzZ0oMP5r9vixZSYKD9vKvcFlf61ymv4lYcpa5cOal/f3umxZw4HPYDk3futKeoz9z216Uo293ZlbyJLQmZb4rd4eOFC/aDvk+ezP31VK8uzZhhn8fifCPuzvufPZv7/08v5+ub/by50n/TXNWXX0o331y6P5PpyAEApc7LS2rWrGD7zpiR9z+OmW/s8ipWmUtaWsH2K8zy559Z3+SkpdlLUtIVnaIisyx7EpAaNUrm+xdHCSutYy7fnnn/Xm4sy74aWr++XVBLsoh44pvi33+XBg82ncI9FcfVdy+v0lm8vUv+Z/zyi/T55/m/5mPHrvy8lSSKEwCg2HTubP+FP7+HB3funPf3cTjsv9j6+komBgdYlj20pLgLWW6F78SJgv0VuyRfrye+8c905IjpBPnLLINeXsX/Madtycl5l85MzZpJtWsX/5vvkt6/pH7Gjh3SAw/kf96io6VOnYpeNDzhivDlNmwoWHEKDi7xKFeEoXoAgGKVed+ElPPDg7lvIrsNG6QuXfLfb/166aabLhWdvy5S2frajh3SY4/lf97efltq167kiklxfCxtBf2dMzF0ypVlDg/N749DJu7VcWWufN64xykPFCcAKHmmHx7sblz5TYUr47wVHeeu6PjjUNG46nkrTDfwKqVMAIAypF8/6dAh+6/VCxbYHw8e5M1Ebry97emfpexXHzI/f/NN3sD+Feet6Dh3Rdevn/0mv27drNvr1aM05cUTzhtXnAAAcBFcqSsazlvRce6KjgcHF42rnTeG6uWB4gQAcGWu9qbCXXDeio5zh7KM4pQHihMAAAAAiXucAAAAAKBYUZwAAAAAIB8UJwAAAADIB8UJAAAAAPJBcQIAAACAfFCcAAAAACAfFCcAAAAAyAfFCQAAAADyQXECAAAAgHxQnAAAAAAgHz6mA5Q2y7IkScnJyYaTAAAAADApsxNkdoS8lLnilJKSIkkKCQkxnAQAAACAK0hJSVFgYGCe+zisgtQrD5KRkaGjR4+qcuXKcjgcpuMoOTlZISEhio+PV0BAgOk4boPzVjSct6LhvBUd565oOG9Fw3krOs5d0XDeisaVzptlWUpJSVGdOnXk5ZX3XUxl7oqTl5eX6tWrZzpGNgEBAcZ/cdwR561oOG9Fw3krOs5d0XDeiobzVnScu6LhvBWNq5y3/K40ZWJyCAAAAADIB8UJAAAAAPJBcTLMz89PkydPlp+fn+koboXzVjSct6LhvBUd565oOG9Fw3krOs5d0XDeisZdz1uZmxwCAAAAAAqLK04AAAAAkA+KEwAAAADkg+IEAAAAAPmgOAEAAABAPihOhnz11Vfq1auX6tSpI4fDoeXLl5uO5BamTp2qa6+9VpUrV1atWrXUp08f7d2713Qslzdz5ky1bt3a+aC5iIgIrVmzxnQst/Pyyy/L4XBo7NixpqO4tOeee04OhyPL0qxZM9Ox3MaRI0d09913q3r16ipfvryuueYabd261XQslxYaGprtd87hcGj06NGmo7m09PR0TZw4UQ0aNFD58uXVqFEjvfDCC2LesPylpKRo7Nixql+/vsqXL6/rr79eW7ZsMR3L5eT3fteyLE2aNEnBwcEqX768IiMjtW/fPjNhC4DiZEhqaqrCwsI0Y8YM01HcysaNGzV69Gj997//VUxMjC5cuKDbb79dqamppqO5tHr16unll1/Wtm3btHXrVt1yyy3q3bu3fvjhB9PR3MaWLVv03nvvqXXr1qajuIWWLVvq2LFjzuWbb74xHckt/PHHH+rUqZPKlSunNWvW6Mcff9Rrr72mqlWrmo7m0rZs2ZLl9y0mJkaSdNdddxlO5tqmTZummTNnavr06dqzZ4+mTZumV155Re+8847paC7vvvvuU0xMjObPn69du3bp9ttvV2RkpI4cOWI6mkvJ7/3uK6+8orfffluzZs3S999/r4oVK6pr1646d+5cKSctIAvGSbKWLVtmOoZbOn78uCXJ2rhxo+kobqdq1arWv//9b9Mx3EJKSop19dVXWzExMdZNN91kjRkzxnQklzZ58mQrLCzMdAy39NRTT1k33HCD6Rhub8yYMVajRo2sjIwM01FcWo8ePaxRo0Zl2davXz9r2LBhhhK5h7Nnz1re3t7WZ599lmV727ZtrWeeecZQKtf31/e7GRkZVu3ata1XX33Vue306dOWn5+fFR0dbSBh/rjiBLeWlJQkSapWrZrhJO4jPT1dCxcuVGpqqiIiIkzHcQujR49Wjx49FBkZaTqK29i3b5/q1Kmjhg0batiwYTp8+LDpSG5h5cqVat++ve666y7VqlVL4eHhmjNnjulYbuX8+fP6+OOPNWrUKDkcDtNxXNr111+v2NhY/fzzz5KknTt36ptvvlG3bt0MJ3NtFy9eVHp6uvz9/bNsL1++PFfXC+HgwYNKSEjI8m9rYGCgOnbsqE2bNhlMljsf0wGAosrIyNDYsWPVqVMntWrVynQcl7dr1y5FRETo3LlzqlSpkpYtW6YWLVqYjuXyFi5cqO3btzN2vRA6duyouXPnqmnTpjp27Jief/55de7cWbt371blypVNx3NpBw4c0MyZMzVu3Dg9/fTT2rJlix599FH5+vpqxIgRpuO5heXLl+v06dO69957TUdxeePHj1dycrKaNWsmb29vpaen68UXX9SwYcNMR3NplStXVkREhF544QU1b95cQUFBio6O1qZNm9S4cWPT8dxGQkKCJCkoKCjL9qCgIOfXXA3FCW5r9OjR2r17N3/dKaCmTZsqLi5OSUlJWrJkiUaMGKGNGzdSnvIQHx+vMWPGKCYmJttfFpG7y/9a3bp1a3Xs2FH169fX4sWL9be//c1gMteXkZGh9u3b66WXXpIkhYeHa/fu3Zo1axbFqYDef/99devWTXXq1DEdxeUtXrxYUVFRWrBggVq2bKm4uDiNHTtWderU4fctH/Pnz9eoUaNUt25deXt7q23bthoyZIi2bdtmOhpKEEP14JYefvhhffbZZ/ryyy9Vr14903Hcgq+vrxo3bqx27dpp6tSpCgsL01tvvWU6lkvbtm2bjh8/rrZt28rHx0c+Pj7auHGj3n77bfn4+Cg9Pd10RLdQpUoVNWnSRPv37zcdxeUFBwdn+2NG8+bNGepYQL/++qs+//xz3XfffaajuIUnnnhC48eP1+DBg3XNNdfonnvu0WOPPaapU6eajubyGjVqpI0bN+rMmTOKj4/X5s2bdeHCBTVs2NB0NLdRu3ZtSVJiYmKW7YmJic6vuRqKE9yKZVl6+OGHtWzZMn3xxRdq0KCB6UhuKyMjQ2lpaaZjuLRbb71Vu3btUlxcnHNp3769hg0bpri4OHl7e5uO6BbOnDmjX375RcHBwaajuLxOnTple8TCzz//rPr16xtK5F4+/PBD1apVSz169DAdxS2cPXtWXl5Z3wp6e3srIyPDUCL3U7FiRQUHB+uPP/7QunXr1Lt3b9OR3EaDBg1Uu3ZtxcbGOrclJyfr+++/d9l7sBmqZ8iZM2ey/PX14MGDiouLU7Vq1XTVVVcZTObaRo8erQULFmjFihWqXLmycwxsYGCgypcvbzid65owYYK6deumq666SikpKVqwYIE2bNigdevWmY7m0ipXrpzt/rmKFSuqevXq3FeXh8cff1y9evVS/fr1dfToUU2ePFne3t4aMmSI6Wgu77HHHtP111+vl156SQMHDtTmzZs1e/ZszZ4923Q0l5eRkaEPP/xQI0aMkI8Pb28KolevXnrxxRd11VVXqWXLltqxY4def/11jRo1ynQ0l7du3TpZlqWmTZtq//79euKJJ9SsWTONHDnSdDSXkt/73bFjx+qf//ynrr76ajVo0EATJ05UnTp11KdPH3Oh82J6Wr+y6ssvv7QkZVtGjBhhOppLy+mcSbI+/PBD09Fc2qhRo6z69etbvr6+Vs2aNa1bb73VWr9+velYbonpyPM3aNAgKzg42PL19bXq1q1rDRo0yNq/f7/pWG7j008/tVq1amX5+flZzZo1s2bPnm06kltYt26dJcnau3ev6ShuIzk52RozZox11VVXWf7+/lbDhg2tZ555xkpLSzMdzeUtWrTIatiwoeXr62vVrl3bGj16tHX69GnTsVxOfu93MzIyrIkTJ1pBQUGWn5+fdeutt7r0/4cdlsXjoQEAAAAgL9zjBAAAAAD5oDgBAAAAQD4oTgAAAACQD4oTAAAAAOSD4gQAAAAA+aA4AQAAAEA+KE4AAAAAkA+KEwAAAADkg+IEAEAhOBwOLV++3HQMAEApozgBANzGvffeK4fDkW254447TEcDAHg4H9MBAAAojDvuuEMffvhhlm1+fn6G0gAAygquOAEA3Iqfn59q166dZalataokexjdzJkz1a1bN5UvX14NGzbUkiVLshy/a9cu3XLLLSpfvryqV6+u+++/X2fOnMmyzwcffKCWLVvKz89PwcHBevjhh7N8/eTJk+rbt68qVKigq6++WitXrizZFw0AMI7iBADwKBMnTlT//v21c+dODRs2TIMHD9aePXskSampqeratauqVq2qLVu26D//+Y8+//zzLMVo5syZGj16tO6//37t2rVLK1euVOPGjbP8jOeff14DBw7U//73P3Xv3l3Dhg3TqVOnSvV1AgBKl8OyLMt0CAAACuLee+/Vxx9/LH9//yzbn376aT399NNyOBx64IEHNHPmTOfXrrvuOrVt21bvvvuu5syZo6eeekrx8fGqWLGiJGn16tXq1auXjh49qqCgINWtW1cjR47UP//5zxwzOBwOPfvss3rhhRck2WWsUqVKWrNmDfdaAYAH4x4nAIBb6dKlS5ZiJEnVqlVzrkdERGT5WkREhOLi4iRJe/bsUVhYmLM0SVKnTp2UkZGhvXv3yuFw6OjRo7r11lvzzNC6dWvnesWKFRUQEKDjx48X9SUBANwAxQkA4FYqVqyYbehccSlfvnyB9itXrlyWzx0OhzIyMkoiEgDARXCPEwDAo/z3v//N9nnz5s0lSc2bN9fOnTuVmprq/Pq3334rLy8vNW3aVJUrV1ZoaKhiY2NLNTMAwPVxxQkA4FbS0tKUkJCQZZuPj49q1KghSfrPf/6j9u3b64YbblBUVJQ2b96s999/X5I0bNgwTZ48WSNGjNBzzz2nEydO6JFHHtE999yjoKAgSdJzzz2nBx54QLVq1VK3bt2UkpKib7/9Vo888kjpvlAAgEuhOAEA3MratWsVHBycZVvTpk31008/SbJnvFu4cKEeeughBQcHKzo6Wi1atJAkVahQQevWrdOYMWN07bXXqkKFCurfv79ef/115/caMWKEzp07pzfeeEOPP/64atSooQEDBpTeCwQAuCRm1QMAeAyHw6Fly5apT58+pqMAADwM9zgBAAAAQD4oTgAAAACQD+5xAgB4DEafAwBKClecAAAAACAfFCcAAAAAyAfFCQAAAADyQXECAAAAgHxQnAAAAAAgHxQnAAAAAMgHxQkAAAAA8kFxAgAAAIB8/D9+bdOrEm6qZgAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHWCAYAAABACtmGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRR0lEQVR4nO3deVxVZeLH8e8FZHEBdwQlccs1EbWMzFbK1Mit3PdpWrTSnGbSSk1bbB8zTcsp9zVTM3PJTMvKxtRw0kwzN1LBzARERYX7++P8uEmgF67Ac5fP+/U6Lw6Hc+HLGSb58jznOTa73W4XAAAAAOCS/EwHAAAAAAB3R3ECAAAAACcoTgAAAADgBMUJAAAAAJygOAEAAACAExQnAAAAAHCC4gQAAAAATlCcAAAAAMAJihMAAAAAOEFxAgAfMWDAAEVHR7v02meffVY2m61oA3mZ6Oho3X333aZjGHXgwAHZbDa99tprpqMAQJGjOAGAYTabrUDbhg0bTEc1YsCAASpbtqzpGG4hp5hcanvppZdMRwQArxVgOgAA+LrZs2fnen/WrFlau3ZtnuMNGza8oq8zbdo0ZWdnu/TaZ555RiNGjLiir4+i07NnT7Vv3z7P8djYWANpAMA3UJwAwLA+ffrkev/bb7/V2rVr8xz/q9OnT6t06dIF/jqlSpVyKZ8kBQQEKCCAfzLcRfPmzZ3+fAAAihZT9QDAA9xyyy1q0qSJtm7dqptuukmlS5fWU089JUn66KOP1KFDB0VGRiooKEh16tTRc889p6ysrFyf46/3OF18P8q7776rOnXqKCgoSNdee62+++67XK/N7x4nm82mRx55RMuWLVOTJk0UFBSkxo0ba/Xq1Xnyb9iwQS1btlRwcLDq1Kmjd955p8jvm/rggw/UokULhYSEqHLlyurTp48OHz6c65zk5GQNHDhQNWrUUFBQkCIiItSxY0cdOHDAcc6WLVvUtm1bVa5cWSEhIapVq5YGDRpU4ByffvqpmjVrpuDgYDVq1EhLlixxfGzfvn2y2Wz697//ned133zzjWw2m+bPn1/4bz4fOfdcXS7Pxbnuu+8+VaxYUaVLl9b111+vTz75JM95Z8+e1bPPPqurr75awcHBioiIUJcuXfTLL7/kOdfZzxQAeBr+fAgAHuL3339Xu3bt1KNHD/Xp00fh4eGSpBkzZqhs2bIaPny4ypYtq88//1yjR49WWlqaXn31Vaefd968eUpPT9eDDz4om82mV155RV26dNG+ffucjlJ99dVXWrJkiQYPHqxy5cpp4sSJ6tq1qw4dOqRKlSpJkr7//nvdddddioiI0NixY5WVlaVx48apSpUqV35R/t+MGTM0cOBAXXvttRo/frxSUlL05ptv6uuvv9b333+v8uXLS5K6du2qnTt36tFHH1V0dLSOHTumtWvX6tChQ47377zzTlWpUkUjRoxQ+fLldeDAgXzLRn5+/vlnde/eXQ899JD69++v6dOn67777tPq1at1xx13qHbt2mrdurXmzp2rxx9/PNdr586dq3Llyqljx45Ov87p06d1/PjxPMfLly+fa2TQWR5JSklJ0Q033KDTp0/rscceU6VKlTRz5kzdc889Wrx4sTp37ixJysrK0t13361169apR48eGjp0qNLT07V27Vrt2LFDderUcXzdK/mZAgC3ZQcAuJUhQ4bY//qf55tvvtkuyT516tQ8558+fTrPsQcffNBeunRp+9mzZx3H+vfvb69Zs6bj/f3799sl2StVqmQ/ceKE4/hHH31kl2T/+OOPHcfGjBmTJ5Mke2BgoH3v3r2OY9u3b7dLsr/11luOYwkJCfbSpUvbDx8+7Dj2888/2wMCAvJ8zvz079/fXqZMmUt+/Ny5c/aqVavamzRpYj9z5ozj+IoVK+yS7KNHj7bb7Xb7H3/8YZdkf/XVVy/5uZYuXWqXZP/uu++c5vqrmjVr2iXZP/zwQ8ex1NRUe0REhD02NtZx7J133rFLsu/atSvX91C5cmV7//79L/s1cv43u9S2adOmQucZNmyYXZJ948aNjmPp6en2WrVq2aOjo+1ZWVl2u91uf//99+2S7G+88UaeXNnZ2bnyFeRnCgA8DVP1AMBDBAUFaeDAgXmOh4SEOPbT09N1/PhxtWnTRqdPn9ZPP/3k9PN2795dFSpUcLzfpk0bSdb0LWfi4+NzjTQ0bdpUoaGhjtdmZWXps88+U6dOnRQZGek4r27dumrXrp3Tz18QW7Zs0bFjxzR48GAFBwc7jnfo0EENGjRwTDkLCQlRYGCgNmzYoD/++CPfz5UzMrVixQqdP3++0FkiIyMdIzSSFBoaqn79+un7779XcnKyJKlbt24KDg7W3LlzHeetWbNGx48fL/B9Sw888IDWrl2bZ2vUqFGh86xcuVLXXXedbrzxRsd5ZcuW1QMPPKADBw7oxx9/lCR9+OGHqly5sh599NE8ef465fJKfqYAwF1RnADAQ1SvXl2BgYF5ju/cuVOdO3dWWFiYQkNDVaVKFccv4KmpqU4/71VXXZXr/ZxfeC9VLi732pzX57z22LFjOnPmjOrWrZvnvPyOueLgwYOSpPr16+f5WIMGDRwfDwoK0ssvv6xVq1YpPDxcN910k1555RVHgZCkm2++WV27dtXYsWNVuXJldezYUdOnT1dmZmaBstStWzdPibj66qslyXEfVfny5ZWQkKB58+Y5zpk7d66qV6+u2267rUBfp169eoqPj8+zhYaGFjrPwYMH8712Oas45ly/X375RfXr1y/QIiFX8jMFAO6K4gQAHuLikaUcJ0+e1M0336zt27dr3Lhx+vjjj7V27Vq9/PLLklSg5cf9/f3zPW6324v1tSYMGzZMe/bs0fjx4xUcHKxRo0apYcOG+v777yVZIyeLFy/Wpk2b9Mgjj+jw4cMaNGiQWrRooVOnThVZjn79+mnfvn365ptvlJ6eruXLl6tnz57y8/OOf5Y97ecCAArCO/4LDQA+asOGDfr99981Y8YMDR06VHfffbfi4+NzTZMyqWrVqgoODtbevXvzfCy/Y66oWbOmJGn37t15PrZ7927Hx3PUqVNH//jHP/Tpp59qx44dOnfunF5//fVc51x//fV64YUXtGXLFs2dO1c7d+7UggULnGbZu3dvnnKwZ88eScq1ouFdd92lKlWqaO7cuVq6dKlOnz6tvn37Fuj7LYyC5KlZs2a+1y5nmmfO9atTp452797t0hRGAPAGFCcA8GA5f9m/+Jfjc+fO6e233zYVKRd/f3/Fx8dr2bJlOnLkiOP43r17tWrVqiL5Gi1btlTVqlU1derUXFPqVq1apV27dqlDhw6SrJXozp49m+u1derUUbly5Ryv++OPP/IUjWbNmklSgabrHTlyREuXLnW8n5aWplmzZqlZs2aqVq2a43hAQIB69uypRYsWacaMGbrmmmvUtGnTwn3jBVCQPO3bt9fmzZu1adMmx3kZGRl69913FR0d7bhvqmvXrjp+/LgmTZqU5+swkgTAF7AcOQB4sBtuuEEVKlRQ//799dhjj8lms2n27Nlu9Yvss88+q08//VStW7fWww8/rKysLE2aNElNmjRRYmJigT7H+fPn9fzzz+c5XrFiRQ0ePFgvv/yyBg4cqJtvvlk9e/Z0LEceHR3tWPZ7z549uv3229WtWzc1atRIAQEBWrp0qVJSUtSjRw9J0syZM/X222+rc+fOqlOnjtLT0zVt2jSFhoaqffv2TnNeffXV+tvf/qbvvvtO4eHhev/995WSkqLp06fnObdfv36aOHGi1q9f75haWVDbtm3TnDlz8hyvU6eO4uLiCpVnxIgRmj9/vtq1a6fHHntMFStW1MyZM7V//359+OGHjumD/fr106xZszR8+HBt3rxZbdq0UUZGhj777DMNHjy4QMuoA4AnozgBgAerVKmSVqxYoX/84x965plnVKFCBfXp00e333672rZtazqeJKlFixZatWqVnnjiCY0aNUpRUVEaN26cdu3aVaBV/yRrFG3UqFF5jtepU0eDBw/WgAEDVLp0ab300kt68sknVaZMGXXu3Fkvv/yyY6W8qKgo9ezZU+vWrdPs2bMVEBCgBg0aaNGiRerataska3GIzZs3a8GCBUpJSVFYWJiuu+46zZ07V7Vq1XKas169enrrrbf0z3/+U7t371atWrW0cOHCfP+3aNGihRo3bqxdu3apd+/eBboOOebPn5/vg3L79++fqzgVJE94eLi++eYbPfnkk3rrrbd09uxZNW3aVB9//LFjtE6yRg9XrlypF154QfPmzdOHH36oSpUq6cYbb9Q111xTqPwA4Ilsdnf6syQAwGd06tRJO3fu1M8//2w6ijGxsbGqWLGi1q1bV+SfOzo6Wk2aNNGKFSuK/HMDgC/iHicAQLE7c+ZMrvd//vlnrVy5UrfccouZQG5gy5YtSkxMVL9+/UxHAQAUAFP1AADFrnbt2howYIBq166tgwcPasqUKQoMDNS//vUv09FK3I4dO7R161a9/vrrioiIUPfu3U1HAgAUAMUJAFDs7rrrLs2fP1/JyckKCgpSXFycXnzxRdWrV890tBK3ePFijRs3TvXr19f8+fMVHBxsOhIAoAC4xwkAAAAAnOAeJwAAAABwguIEAAAAAE743D1O2dnZOnLkiMqVKyebzWY6DgAAAABD7Ha70tPTFRkZ6Xjg96X4XHE6cuSIoqKiTMcAAAAA4CaSkpJUo0aNy57jc8WpXLlykqyLExoaajgNAAAAAFPS0tIUFRXl6AiX43PFKWd6XmhoKMUJAAAAQIFu4WFxCAAAAABwguIEAAAAAE5QnAAAAADACYoTAAAAADhBcQIAAAAAJyhOAAAAAOAExQkAAAAAnKA4AQAAAIATFCcAAAAAcCLAdABflpUlbdwoHT0qRURIbdpI/v6mUwEAAAD4K4qTIUuWSEOHSr/++uexGjWkN9+UunQxlwsAAABAXkzVM2DJEunee3OXJkk6fNg6vmSJmVwAAAAA8kdxKmFZWdZIk92e92M5x4YNs84DAAAA4B4oTiVs48a8I00Xs9ulpCTrPAAAAADugeJUwo4eLdrzAAAAABQ/ilMJi4go2vMAAAAAFD+KUwlr08ZaPc9my//jNpsUFWWdBwAAAMA9UJxKmL+/teS4lLc85bw/YQLPcwIAAADcCcXJgC5dpMWLperVcx+vVMk6znOcAAAAAPdCcTKkSxfpwAFp/XqpQwfrWJs2lCYAAADAHQWYDuDL/P2lW26xRpo++URasUI6cUKqWNF0MgAAAAAXY8TJDVxzjdSsmXT+vLRwoek0AAAAAP6K4uQm+vWz3s6aZTYHAAAAgLwoTm6iZ09r6t6330p79phOAwAAAOBiFCc3Ua2a1LattT97ttksAAAAAHKjOLmRnOl6s2dL2dlmswAAAAD4E8XJjdxzjxQaKh08KH35pek0AAAAAHJQnNxISIjUrZu1zyIRAAAAgPugOLmZnOl6H3wgnT5tNgsAAAAAC8XJzbRuLdWqJZ06JS1bZjoNAAAAAIni5Hb8/KS+fa19pusBAAAA7oHi5IZyitPatdKRI2azAAAAAKA4uaW6daUbbrCWJJ83z3QaAAAAABQnN5WzSMTMmZLdbjYLAAAA4OuMFqcvv/xSCQkJioyMlM1m07ICrIaQmZmpp59+WjVr1lRQUJCio6P1/vvvF3/YEtatmxQYKO3YIW3fbjoNAAAA4NuMFqeMjAzFxMRo8uTJBX5Nt27dtG7dOr333nvavXu35s+fr/r16xdjSjMqVLAeiCuxSAQAAABgWoDJL96uXTu1a9euwOevXr1aX3zxhfbt26eKFStKkqKjo4spnXn9+kmLF0tz50qvvCIFGP1fCwAAAPBdHnWP0/Lly9WyZUu98sorql69uq6++mo98cQTOnPmzCVfk5mZqbS0tFybp7jrLqlyZenYMenTT02nAQAAAHyXRxWnffv26auvvtKOHTu0dOlSTZgwQYsXL9bgwYMv+Zrx48crLCzMsUVFRZVg4itTqpTUq5e1z3Q9AAAAwByPKk7Z2dmy2WyaO3eurrvuOrVv315vvPGGZs6ceclRp5EjRyo1NdWxJSUllXDqK5Ozut6yZdLJkyaTAAAAAL7Lo4pTRESEqlevrrCwMMexhg0bym6369dff833NUFBQQoNDc21eZLmzaVGjaTMTOt+JwAAAAAlz6OKU+vWrXXkyBGdOnXKcWzPnj3y8/NTjRo1DCYrPjbbn6NOTNcDAAAAzDBanE6dOqXExEQlJiZKkvbv36/ExEQdOnRIkjXNrl9Oa5DUq1cvVapUSQMHDtSPP/6oL7/8Uv/85z81aNAghYSEmPgWSkTv3laB2rhR2rfPdBoAAADA9xgtTlu2bFFsbKxiY2MlScOHD1dsbKxGjx4tSTp69KijRElS2bJltXbtWp08eVItW7ZU7969lZCQoIkTJxrJX1Jq1JBuv93anzPHbBYAAADAF9nsdrvddIiSlJaWprCwMKWmpnrU/U6zZ1tT9urUkX7+2RqBAgAAAOC6wnQDj7rHyZd17iyVKSP98ou0aZPpNAAAAIBvoTh5iLJlpa5drX0WiQAAAABKFsXJg+Ssk7FwoXT2rNksAAAAgC+hOHmQW2+VoqKsB+GuWGE6DQAAAOA7KE4exM9P6tPH2me6HgAAAFByKE4epm9f6+2qVdKxY2azAAAAAL6C4uRhGjaUrr1WunBBWrDAdBoAAADAN1CcPFDOIhFM1wMAAABKBsXJA/XoIQUESFu3Sjt3mk4DAAAAeD+KkweqXFnq0MHanz3bbBYAAADAF1CcPFTOdL05c6SsLLNZAAAAAG9HcfJQHTpIFSpIhw9L69ebTgMAAAB4N4qThwoKsu51klgkAgAAAChuFCcPljNd78MPpVOnzGYBAAAAvBnFyYO1aiXVqyedPi0tWWI6DQAAAOC9KE4ezGbjmU4AAABASaA4ebg+fay3n38uJSWZzQIAAAB4K4qTh4uOlm6+WbLbpblzTacBAAAAvBPFyQtcPF3PbjebBQAAAPBGFCcvcO+9UnCwtGuXtGWL6TQAAACA96E4eYHQUKlzZ2ufRSIAAACAokdx8hI50/Xmz5fOnTObBQAAAPA2FCcvER8vVasm/f67tGqV6TQAAACAd6E4eYmAAKl3b2uf6XoAAABA0aI4eZGc6XoffyydOGE2CwAAAOBNKE5epGlTKSZGOn9eWrjQdBoAAADAe1CcvMzFz3QCAAAAUDQoTl6mVy/Jz0/69ltpzx7TaQAAAADvQHHyMtWqSW3bWvuzZ5vNAgAAAHgLipMXypmuN3u2lJ1tNgsAAADgDShOXqhjRyk0VDp4UNq40XQaAAAAwPNRnLxQSIh0333WPotEAAAAAFeO4uSlcqbrffCBdPq02SwAAACApzNanL788kslJCQoMjJSNptNy5YtK/Brv/76awUEBKhZs2bFls+T3XijFB0tpadLH31kOg0AAADg2YwWp4yMDMXExGjy5MmFet3JkyfVr18/3X777cWUzPP5+Ul9+1r7TNcDAAAArozNbrfbTYeQJJvNpqVLl6pTp05Oz+3Ro4fq1asnf39/LVu2TImJiQX+OmlpaQoLC1NqaqpCQ0NdD+wBfv5Zuvpqq0T9+qsUEWE6EQAAAOA+CtMNPO4ep+nTp2vfvn0aM2ZMgc7PzMxUWlpars1X1KsnxcVZS5LPm2c6DQAAAOC5PKo4/fzzzxoxYoTmzJmjgICAAr1m/PjxCgsLc2xRUVHFnNK95CwSwXQ9AAAAwHUeU5yysrLUq1cvjR07VldffXWBXzdy5EilpqY6tqSkpGJM6X66dZMCA6X//U/avt10GgAAAMAzeUxxSk9P15YtW/TII48oICBAAQEBGjdunLZv366AgAB9/vnn+b4uKChIoaGhuTZfUrGilJBg7TPqBAAAALjGY4pTaGiofvjhByUmJjq2hx56SPXr11diYqJatWplOqLbypmuN3eudOGC2SwAAACAJyrYjULF5NSpU9q7d6/j/f379ysxMVEVK1bUVVddpZEjR+rw4cOaNWuW/Pz81KRJk1yvr1q1qoKDg/McR27t2kmVK0spKdLatdb7AAAAAArO6IjTli1bFBsbq9jYWEnS8OHDFRsbq9GjR0uSjh49qkOHDpmM6BVKlZJ69bL2ma4HAAAAFJ7bPMeppPjSc5wutnWr1LKlFBwsJSdLYWGmEwEAAABmefVznOCa5s2lRo2ks2elxYtNpwEAAAA8C8XJR9hsPNMJAAAAcBXFyYf07m0VqC+/lPbvN50GAAAA8BwUJx9So4Z0++3W/pw5ZrMAAAAAnoTi5GMunq7nW8uCAAAAAK6jOPmYzp2lMmWkvXulb781nQYAAADwDBQnH1O2rNS1q7XPIhEAAABAwVCcfFDOdL0FC6TMTLNZAAAAAE9AcfJBt9xiLRRx8qS0YoXpNAAAAID7ozj5IH9/qU8fa5/pegAAAIBzFCcf1bev9XblSum338xmAQAAANwdxclHNWoktWwpXbhg3esEAAAA4NIoTj7s4mc6AQAAALg0ipMP69FDCgiQtmyRfvzRdBoAAADAfVGcfFiVKlL79tY+o04AAADApVGcfFzOdL05c6SsLLNZAAAAAHdFcfJxd98tlS8vHT4srV9vOg0AAADgnihOPi4oyLrXSWK6HgAAAHApFCc4put9+KF06pTZLAAAAIA7ojhB118v1a0rnT4tLVliOg0AAADgfihOkM3GM50AAACAy6E4QZLUp4/19vPPpaQks1kAAAAAd0NxgiSpVi3pppsku12aO9d0GgAAAMC9UJzgcPF0PbvdbBYAAADAnVCc4HDvvVJwsLRrl7R1q+k0AAAAgPugOMEhLEzq1MnaZ5EIAAAA4E8UJ+SSM11v/nzp3DmzWQAAAAB3QXFCLnfcIYWHS8ePS6tXm04DAAAAuAeKE3IJCJB697b2ma4HAAAAWChOyCNnut7HH0snTpjNAgAAALgDihPyiImRmja17nFatMh0GgAAAMA8ihPydfEznQAAAABfR3FCvnr1kvz8pE2bpJ9/Np0GAAAAMMtocfryyy+VkJCgyMhI2Ww2LVu27LLnL1myRHfccYeqVKmi0NBQxcXFac2aNSUT1sdEREh33mntz55tNgsAAABgmtHilJGRoZiYGE2ePLlA53/55Ze64447tHLlSm3dulW33nqrEhIS9P333xdzUt/Uv7/1dvZsKTvbbBYAAADAJJvdbrebDiFJNptNS5cuVadOnQr1usaNG6t79+4aPXp0gc5PS0tTWFiYUlNTFRoa6kJS33HmjFStmpSWJn3xhXTTTaYTAQAAAEWnMN3Ao+9xys7OVnp6uipWrHjJczIzM5WWlpZrQ8GEhEj33Wfts0gEAAAAfJlHF6fXXntNp06dUrdu3S55zvjx4xUWFubYoqKiSjCh58tZXW/RImsECgAAAPBFHluc5s2bp7Fjx2rRokWqWrXqJc8bOXKkUlNTHVtSUlIJpvR8N94oRUdL6enSRx+ZTgMAAACY4ZHFacGCBbr//vu1aNEixcfHX/bcoKAghYaG5tpQcH5+Ut++1j7T9QAAAOCrPK44zZ8/XwMHDtT8+fPVoUMH03F8Qk5xWrNGSk42mwUAAAAwwWhxOnXqlBITE5WYmChJ2r9/vxITE3Xo0CFJ1jS7fjk32ciantevXz+9/vrratWqlZKTk5WcnKzU1FQT8X1GvXpSXJy1JPm8eabTAAAAACXPaHHasmWLYmNjFRsbK0kaPny4YmNjHUuLHz161FGiJOndd9/VhQsXNGTIEEVERDi2oUOHGsnvS3L6K9P1AAAA4Ivc5jlOJYXnOLnmxAkpIkI6d05KTJRiYkwnAgAAAK6MzzzHCSWnYkUpIcHanz3bbBYAAACgpFGcUGA50/XmzpUuXDCbBQAAAChJFCcU2F13SZUrWyvrffaZ6TQAAABAyaE4ocACA6WePa19FokAAACAL6E4oVBypustXSqlpZnNAgAAAJQUihMKpUULqWFD6exZafFi02kAAACAkkFxQqHYbDzTCQAAAL6H4oRC693bKlBffCHt3286DQAAAFD8KE4otKgo6bbbrP05c8xmAQAAAEoCxQkuuXi6nt1uNgsAAABQ3ChOcEmXLlLp0tLevdK335pOAwAAABQvihNcUras1LWrtc8iEQAAAPB2FCe4LGe63oIFUmam2SwAAABAcaI4wWW33ipVry6dPCmtWGE6DQAAAFB8KE5wmb+/1KePtc90PQAAAHgzihOuSN++1tuVK6XffjObBQAAACguFCdckcaNpRYtpAsXrHudAAAAAG9EccIVu/iZTgAAAIA3ojjhivXoIQUESFu2SD/+aDoNAAAAUPQoTrhiVatK7dpZ+7Nnm80CAAAAFAeKE4pEznS9OXOkrCyzWQAAAICiRnFCkbj7bql8eenXX6UNG0ynAQAAAIoWxQlFIjhY6t7d2meRCAAAAHgbihOKTM50vQ8/lE6dMpsFAAAAKEoUJxSZuDipTh0pI0NautR0GgAAAKDoUJxQZGw2nukEAAAA70RxQpHq08d6u26dtVAEAAAA4A0oTihStWtLbdpIdrs0d67pNAAAAEDRoDihyPXvb72dNcsqUAAAAICnozihyN17r7U8+Y8/Stu2mU4DAAAAXDmKE4pcWJjUqZO1zyIRAAAA8AYUJxSLnNX15s2Tzp83mwUAAAC4UhQnFIs77pDCw6Xjx6XVq02nAQAAAK6M0eL05ZdfKiEhQZGRkbLZbFq2bJnT12zYsEHNmzdXUFCQ6tatqxkzZhR7ThReQIDUu7e1z3Q9AAAAeDqjxSkjI0MxMTGaPHlygc7fv3+/OnTooFtvvVWJiYkaNmyY7r//fq1Zs6aYk8IVOdP1li+X/vjDbBYAAADgStjsdvdYMNpms2np0qXqlLOqQD6efPJJffLJJ9qxY4fjWI8ePXTy5EmtLuB8sLS0NIWFhSk1NVWhoaFXGhtOxMRI//ufNHWq9OCDptMAAAAAfypMN/Coe5w2bdqk+Pj4XMfatm2rTZs2XfI1mZmZSktLy7Wh5OSMOjFdDwAAAJ7Mo4pTcnKywsPDcx0LDw9XWlqazpw5k+9rxo8fr7CwMMcWFRVVElHx/3r1kvz8pG++kfbuNZ0GAAAAcI1HFSdXjBw5UqmpqY4tKSnJdCSfEhEh3XmntT97ttksAAAAgKs8qjhVq1ZNKSkpuY6lpKQoNDRUISEh+b4mKChIoaGhuTaUrIun62Vnm80CAAAAuMKjilNcXJzWrVuX69jatWsVFxdnKBEKomNHqVw56cAB6euvTacBAAAACs9ocTp16pQSExOVmJgoyVpuPDExUYcOHZJkTbPrlzNcIemhhx7Svn379K9//Us//fST3n77bS1atEiPP/64ifgooNKlpfvus/ZZJAIAAACeyGhx2rJli2JjYxUbGytJGj58uGJjYzV69GhJ0tGjRx0lSpJq1aqlTz75RGvXrlVMTIxef/11/ec//1Hbtm2N5EfB5fTfRYukS6zjAQAAALgtt3mOU0nhOU5mZGdLtWtLBw9KCxZI3bubTgQAAABf57XPcYLn8vOT+va19pmuBwAAAE/jUnFKSkrSr7/+6nh/8+bNGjZsmN59990iCwbvk1Oc1qyRkpPNZgEAAAAKw6Xi1KtXL61fv16S9VDaO+64Q5s3b9bTTz+tcePGFWlAeI+rr5auv17KypLmzTOdBgAAACg4l4rTjh07dN1110mSFi1apCZNmuibb77R3LlzNWPGjKLMBy9z8TOdAAAAAE/hUnE6f/68goKCJEmfffaZ7rnnHklSgwYNdPTo0aJLB6/TvbtUqpS0fbu1AQAAAJ7ApeLUuHFjTZ06VRs3btTatWt11113SZKOHDmiSpUqFWlAeJeKFaWEBGt/9myzWQAAAICCcqk4vfzyy3rnnXd0yy23qGfPnoqJiZEkLV++3DGFD7iUnOl6c+dKFy6YzQIAAAAUhMvPccrKylJaWpoqVKjgOHbgwAGVLl1aVatWLbKARY3nOJl37pwUGSn9/ru0apX0/wOWAAAAQIkq9uc4nTlzRpmZmY7SdPDgQU2YMEG7d+9269IE9xAYKPXsae2zSAQAAAA8gUvFqWPHjpr1/7/xnjx5Uq1atdLrr7+uTp06acqUKUUaEN4pZ7re0qVSWprZLAAAAIAzLhWnbdu2qU2bNpKkxYsXKzw8XAcPHtSsWbM0ceLEIg0I79SypdSggXT2rLR4sek0AAAAwOW5VJxOnz6tcuXKSZI+/fRTdenSRX5+frr++ut18ODBIg0I72Sz8UwnAAAAeA6XilPdunW1bNkyJSUlac2aNbrzzjslSceOHWPBBRRY795WgfriC+nAAdNpAAAAgEtzqTiNHj1aTzzxhKKjo3XdddcpLi5OkjX6FBsbW6QB4b2uukq69VZrf84cs1kAAACAy3F5OfLk5GQdPXpUMTEx8vOz+tfmzZsVGhqqBg0aFGnIosRy5O5l5kxpwACpXj1p925rBAoAAAAoCYXpBi4Xpxy//vqrJKlGjRpX8mlKDMXJvaSnS9WqSadPS5s2SddfbzoRAAAAfEWxP8cpOztb48aNU1hYmGrWrKmaNWuqfPnyeu6555Sdne1SaPimcuWkLl2sfRaJAAAAgLtyqTg9/fTTmjRpkl566SV9//33+v777/Xiiy/qrbfe0qhRo4o6I7xczup6CxZImZlmswAAAAD5cWmqXmRkpKZOnap77rkn1/GPPvpIgwcP1uHDh4ssYFFjqp77ycqyFoo4ckT68MM/R6AAAACA4lTsU/VOnDiR7wIQDRo00IkTJ1z5lPBh/v5Snz7WPtP1AAAA4I5cKk4xMTGaNGlSnuOTJk1S06ZNrzgUfE/OdL1PPpGOHzebBQAAAPirAFde9Morr6hDhw767LPPHM9w2rRpk5KSkrRy5coiDQjf0Lix1KKFtHWrda/TI4+YTgQAAAD8yaURp5tvvll79uxR586ddfLkSZ08eVJdunTRzp07NXv27KLOCB+RM+rEdD0AAAC4myt+jtPFtm/frubNmysrK6uoPmWRY3EI93XsmFS9unThgvTjj1LDhqYTAQAAwJsV++IQQHGoWlVq187aZ+ASAAAA7oTiBLeSM11v9myJZykDAADAXVCc4FbuvlsqX1769VdpwwbTaQAAAABLoVbV6+LkyaQnT568kiyAgoOl7t2ld96xFom47TbTiQAAAIBCjjiFhYVddqtZs6b65cy1AlyU8yO0eLGUkWE2CwAAACAV8ap6noBV9dyf3S7Vqyf98ot1r1OfPqYTAQAAwBuxqh48ms3GM50AAADgXihOcEs5o0yffSYdPmw2CwAAAEBxgluqXVtq08aatjd3ruk0AAAA8HVuUZwmT56s6OhoBQcHq1WrVtq8efNlz58wYYLq16+vkJAQRUVF6fHHH9fZs2dLKC1KSs50vZkzrQIFAAAAmGK8OC1cuFDDhw/XmDFjtG3bNsXExKht27Y6duxYvufPmzdPI0aM0JgxY7Rr1y699957WrhwoZ566qkSTo7idt99UlCQ9OOP0vffm04DAAAAX2a8OL3xxhv6+9//roEDB6pRo0aaOnWqSpcurffffz/f87/55hu1bt1avXr1UnR0tO6880717NnT6SgVPE9YmNSpk7XPIhEAAAAwyWhxOnfunLZu3ar4+HjHMT8/P8XHx2vTpk35vuaGG27Q1q1bHUVp3759Wrlypdq3b5/v+ZmZmUpLS8u1wXPkTNebN086f95sFgAAAPguo8Xp+PHjysrKUnh4eK7j4eHhSk5Ozvc1vXr10rhx43TjjTeqVKlSqlOnjm655ZZLTtUbP358rof0RkVFFfn3geJz551S1arSb79Ja9aYTgMAAABfZXyqXmFt2LBBL774ot5++21t27ZNS5Ys0SeffKLnnnsu3/NHjhyp1NRUx5aUlFTCiXElAgKk3r2tfabrAQAAwJQAk1+8cuXK8vf3V0pKSq7jKSkpqlatWr6vGTVqlPr27av7779fknTNNdcoIyNDDzzwgJ5++mn5+eXugkFBQQoKCiqebwAlol8/6d//lj76SPrjD6lCBdOJAAAA4GuMjjgFBgaqRYsWWrduneNYdna21q1bp7i4uHxfc/r06TzlyN/fX5JkZ81qrxQTI11zjXTunLRokek0AAAA8EXGp+oNHz5c06ZN08yZM7Vr1y49/PDDysjI0MCBAyVJ/fr108iRIx3nJyQkaMqUKVqwYIH279+vtWvXatSoUUpISHAUKHgXm+3PRSKYrgcAAAATjE7Vk6Tu3bvrt99+0+jRo5WcnKxmzZpp9erVjgUjDh06lGuE6ZlnnpHNZtMzzzyjw4cPq0qVKkpISNALL7xg6ltACejVS3rySembb6S9e6W6dU0nAgAAgC+x2X1sfltaWprCwsKUmpqq0NBQ03FQCHfdZa2sN3q0NHas6TQAAADwdIXpBsan6gEFdfF0vexss1kAAADgW4xP1QMKqlMnqWxZ6cABadIkqUoVKSJCatNG4vY2AAAAFCdGnOAxSpeWrr3W2h861Lrv6dZbpehoackSo9EAAADg5ShO8BhLlkjr1+c9fviwdO+9lCcAAAAUH4oTPEJWljXKlJ+c5U2GDbPOAwAAAIoaxQkeYeNG6ddfL/1xu11KSrLOAwAAAIoaxQke4ejRoj0PAAAAKAyKEzxCRETRngcAAAAUBsUJHqFNG6lGDclmu/Q5UVHWeQAAAEBRozjBI/j7S2++ae1fqjw9/TTPcwIAAEDxoDjBY3TpIi1eLFWvnvt4qVLW20mTpNTUks8FAAAA70dxgkfp0kU6cMB6ntO8edbb3bulatWkHTuk++6Tzp83nRIAAADeJsB0AKCw/P2lW27JfeyTT6SbbpLWrpUefliaNu3y90MBAAAAhcGIE7xC8+bSggWSn5/03nvS+PGmEwEAAMCbUJzgNe6+W5o40dp/+mlp/nyzeQAAAOA9KE7wKkOGSMOHW/sDBkgbNxqNAwAAAC9BcYLXefVVaxGJc+ekTp2kPXtMJwIAAICnozjB6/j5SbNnS9ddJ504IbVvL/32m+lUAAAA8GQUJ3il0qWl5culWrWkX36ROnaUzpwxnQoAAACeiuIErxUeLq1cKZUvL23aJPXrJ2Vnm04FAAAAT0Rxgldr0EBatkwqVUpavFgaOdJ0IgAAAHgiihO83s03S++/b+2/8oo0darZPAAAAPA8FCf4hD59pHHjrP0hQ6RVq8zmAQAAgGehOMFnPPOM9Wyn7GypWzcpMdF0IgAAAHgKihN8hs0mvfOOdNtt0qlTUocO0q+/mk4FAAAAT0Bxgk8JDJQ+/FBq1Eg6csQqT2lpplMBAADA3VGc4HPKl7eWKQ8Pl/73P2va3vnzplMBAADAnVGc4JNq1pRWrLAelLtmjbVghN1uOhUAAADcFcUJPqtlS2n+fOvep2nTrKXKAQAAgPxQnODT7rlHmjDB2h8xQlq0yGgcAAAAuCmKE3zeY49JQ4da+/36SV9/bTYPAAAA3A/FCZD0+utSx45SZqb1du9e04kAAADgTihOgCR/f2nuXOu+p99/l9q1k44fN50KAAAA7sItitPkyZMVHR2t4OBgtWrVSps3b77s+SdPntSQIUMUERGhoKAgXX311Vq5cmUJpYW3KlNG+vhja8W9vXulTp2ks2dNpwIAAIA7MF6cFi5cqOHDh2vMmDHatm2bYmJi1LZtWx07dizf88+dO6c77rhDBw4c0OLFi7V7925NmzZN1atXL+Hk8EbVqlnPeAoLs+51GjhQys42nQoAAACm2ex2s0+vadWqla699lpNmjRJkpSdna2oqCg9+uijGjFiRJ7zp06dqldffVU//fSTSpUqVeivl5aWprCwMKWmpio0NPSK88M7ff651LatdOGCNHKk9OKLphMBAACgqBWmGxgdcTp37py2bt2q+Ph4xzE/Pz/Fx8dr06ZN+b5m+fLliouL05AhQxQeHq4mTZroxRdfVFZWVr7nZ2ZmKi0tLdcGOHPbbdJ//mPtjx9vPecJAAAAvstocTp+/LiysrIUHh6e63h4eLiSk5Pzfc2+ffu0ePFiZWVlaeXKlRo1apRef/11Pf/88/meP378eIWFhTm2qKioIv8+4J3695fGjLH2H35Y+vRTs3kAAABgjvF7nAorOztbVatW1bvvvqsWLVqoe/fuevrppzV16tR8zx85cqRSU1MdW1JSUgknhicbM0bq21fKypLuvVf63/9MJwIAAIAJASa/eOXKleXv76+UlJRcx1NSUlStWrV8XxMREaFSpUrJ39/fcaxhw4ZKTk7WuXPnFBgYmOv8oKAgBQUFFX14+ASbzZqyl5Qkbdggdegg/fe/UmSk6WQAAAAoSUZHnAIDA9WiRQutW7fOcSw7O1vr1q1TXFxcvq9p3bq19u7dq+yLljrbs2ePIiIi8pQmoCgEBkpLlkgNGki//mqVp/R006kAAABQkoxP1Rs+fLimTZummTNnateuXXr44YeVkZGhgQMHSpL69eunkSNHOs5/+OGHdeLECQ0dOlR79uzRJ598ohdffFFDhgwx9S3AB1SoYC1TXrWqlJgo9ehhrbgHAAAA32B0qp4kde/eXb/99ptGjx6t5ORkNWvWTKtXr3YsGHHo0CH5+f3Z76KiorRmzRo9/vjjatq0qapXr66hQ4fqySefNPUtwEfUqmU9IPeWW6wS9dhj0uTJ1nQ+AAAAeDfjz3EqaTzHCVdq6VKpa1fJbpdee036xz9MJwIAAIArPOY5ToAn6txZev11a/+JJ6TFi83mAQAAQPGjOAEuGDZMeuQRa79vX+nbb43GAQAAQDGjOAEusNmkCROkhATp7FnpnnukX34xnQoAAADFheIEuMjfX5o3T2reXPrtN6l9e+nECdOpAAAAUBwoTsAVKFtWWrFCioqS9uyROnWSMjNNpwIAAEBRozgBVygiwlqePDRU2rhRGjTIWnEPAAAA3oPiBBSBJk2kDz+UAgKs6XujR5tOBAAAgKJEcQKKSHy89M471v7zz0vvv282DwAAAIoOxQkoQoMGSc88Y+0/+KD02Wdm8wAAAKBoUJyAIjZunNSrl3ThgtS1q7Rjh+lEAAAAuFIUJ6CI2WzWNL02baS0NKlDB+noUdOpAAAAcCUoTkAxCAqSli2T6teXDh2S7r5bOnXKdCoAAAC4iuIEFJOKFaVPPpEqV5a2bbOm72VlmU4FAAAAV1CcgGJUp460fLkUHCx9/LE0bBjPeAIAAPBEFCegmMXFSXPmWPc+TZokvfmm6UQAAAAoLIoTUAK6dpVeecXaHz5cWrrUbB4AAAAUDsUJKCH/+If08MPWVL3evaXNm00nAgAAQEFRnIASYrNJEydK7dtLZ85ICQnS/v2mUwEAAKAgKE5ACQoIkBYulJo1k44ds0rUH3+YTgUAAABnKE5ACStbVlqxQqpRQ/rpJ6lLF+ncOdOpAAAAcDkUJ8CA6tWtZzyVKydt2CDdfz/LlAMAALgzihNgSNOm0uLFkr+/NHu2NHas6UQAAAC4FIoTYNCdd0pTplj7Y8dKM2eazQMAAID8UZwAw/7+d2nkSGv//vulzz83mwcAAAB5UZwAN/D881L37tKFC9ZiET/+aDoRAAAALkZxAtyAn580Y4bUurWUmmotU56cbDoVAAAAclCcADcRHCwtWybVqycdPGg9IDcjw3QqAAAASBQnwK1UriytXClVqiRt2SL17i1lZZlOBQAAAIoT4Gbq1pU++kgKCrLePvGE6UQAAACgOAFuqHVradYsa3/CBGniRKNxAAAAfB7FCXBT3bpJL71k7Q8bJi1fbjQOAACAT6M4AW7sX/+SHnhAstulnj2t+54AAABQ8ihOgBuz2aTJk6W77pJOn5buvttacQ8AAAAlyy2K0+TJkxUdHa3g4GC1atVKmzdvLtDrFixYIJvNpk6dOhVvQMCggABp4UKpaVMpJUXq0EE6edJ0KgAAAN9ivDgtXLhQw4cP15gxY7Rt2zbFxMSobdu2Onbs2GVfd+DAAT3xxBNq06ZNCSUFzAkNlT75RIqMlHbulLp2lc6dM50KAADAdxgvTm+88Yb+/ve/a+DAgWrUqJGmTp2q0qVL6/3337/ka7KystS7d2+NHTtWtWvXLsG0gDk1aljlqWxZ6fPPpQcftO59AgAAQPEzWpzOnTunrVu3Kj4+3nHMz89P8fHx2rRp0yVfN27cOFWtWlV/+9vfnH6NzMxMpaWl5doAT9WsmbRokeTvL82YIb3wgulEAAAAvsFocTp+/LiysrIUHh6e63h4eLiSk5Pzfc1XX32l9957T9OmTSvQ1xg/frzCwsIcW1RU1BXnBkxq185aMEKSRo2S5swxmwcAAMAXGJ+qVxjp6enq27evpk2bpsqVKxfoNSNHjlRqaqpjS0pKKuaUQPF78EFrqXJJGjRI+uILs3kAAAC8XYDJL165cmX5+/srJSUl1/GUlBRVq1Ytz/m//PKLDhw4oISEBMex7OxsSVJAQIB2796tOnXq5HpNUFCQgoKCiiE9YNb48dL+/dIHH0idO0vffCM1aGA6FQAAgHcyOuIUGBioFi1aaN26dY5j2dnZWrduneLi4vKc36BBA/3www9KTEx0bPfcc49uvfVWJSYmMg0PPsXPT5o5U4qLk/74Q2rfXnKyGCUAAABcZHTESZKGDx+u/v37q2XLlrruuus0YcIEZWRkaODAgZKkfv36qXr16ho/fryCg4PVpEmTXK8vX768JOU5DviCkBDpo4+s8vTLL9I990jr11vHAQAAUHSMF6fu3bvrt99+0+jRo5WcnKxmzZpp9erVjgUjDh06JD8/j7oVCyhRVapIK1da5em//5X69LGm7/F/GwAAgKJjs9t960kwaWlpCgsLU2pqqkJDQ03HAYrMxo1SfLz1YNx//EN67TXTiQAAANxbYboBf5MGvESbNtaznSTp9delt982GgcAAMCrUJwAL9Kz558PxX30UemTT8zmAQAA8BYUJ8DLjBwp/e1vUna21L27tG2b6UQAAACej+IEeBmbTZoyRbrjDikjQ7r7bonnPgMAAFwZihPghUqVslbWa9JEOnrUesbTiRPShg3S/PnW26ws0ykBAAA8h/HlyAEUj7Aw6x6n66+XduyQIiOlzMw/P16jhvTmm1KXLuYyAgAAeApGnAAvdtVV0vDh1v7FpUmSDh+W7r1XWrKk5HMBAAB4GooT4MWysqxRpfzkPMFt2DCm7QEAADhDcQK82MaN0q+/Xvrjdru1cMTGjSWXCQAAwBNRnAAvdvRowc7bsOHPESgAAADkRXECvFhERMHOGzvWWoHvzTelP/4o3kwAAACeiOIEeLE2bazV82y2S59TpowUEiL9+KN1v1NkpNS/v/T114xCAQAA5KA4AV7M3//PxSH+Wp5sNmubNcua0jd5stS0qXT2rHXsxhula66RJk5kFAoAAIDiBHi5Ll2kxYul6tVzH69RwzrepYv1zKfBg6XEROnbb6VBg6TSpaWdO6WhQ/8chfrmG0ahAACAb7LZ7b71a1BaWprCwsKUmpqq0NBQ03GAEpOVZa2ed/Sode9TmzbWiNSlpKZKc+dK77wj/e9/fx5v0kR64AGpTx+pQoXizw0AAFBcCtMNKE4ALstulzZvtgrUggXSmTPW8eBgqVs36cEHpbi4y99HBQAA4I4K0w2Yqgfgsmw2qVUr6f33rdGqSZOse59y7oVq3dq6N+qtt7gXCgAAeC+KE4ACCwuThgyRtm+XNm2SBg60VuTbsUN67DHrXqgBA7gXCgAAeB+KE4BCs9mk66+3RqGOHMk9CjVzZu5RqJMnTacFAAC4chQnAFekfPnco1ADBuQdhRo40PoYo1AAAMBTUZwAFImcUajp061RqLfeslbgO3NGmjFDuuEGKSbGGp1iFAoAAHgaihOAIle+vPTII9Yy5t988+co1A8/SI8+yigUAADwPBQnAMXGZrOWKmcUCgAAeDqKE4ASUdBRqG+/ZRQKAAC4H4oTgBLlbBQqLk5q1kyaPJlRKAAA4D4oTgCM+esoVP/+UnCw9f4jj1ijUIMGMQoFAADMozgBMC5nFGrGDGsUauJEqXFjaxRq+vTco1CpqabTAgAAX0RxAuBWKlSw7nn64Qfp66/zjkJFRFijUP/9L6NQAACg5FCcALglm81ade9So1DXX2+NQr39NqNQAACg+FGcALi9v45C9ev35yjUkCHWvVB/+xujUAAAoPhQnAB4jJxRqJkzrVGoN9+UGjWSTp+W3n/fGoWKjWUUCgAAFD2KEwCPVKGC9Nhj0o4d0ldf/TkKtX177lGozZsZhQIAAFeO4gTAo9lsUuvW1ijU4cN5R6FatbJGoaZMYRQKAAC4zi2K0+TJkxUdHa3g4GC1atVKmzdvvuS506ZNU5s2bVShQgVVqFBB8fHxlz0fgO+oWDH3KFTfvlJQkDUKNXiwNQp1//2MQgEAgMIzXpwWLlyo4cOHa8yYMdq2bZtiYmLUtm1bHTt2LN/zN2zYoJ49e2r9+vXatGmToqKidOedd+rw4cMlnByAu8oZhZo1y7oXasIEqWFDaxTqvfesUajmza1RqLQ002kBAIAnsNntZv/u2qpVK1177bWaNGmSJCk7O1tRUVF69NFHNWLECKevz8rKUoUKFTRp0iT169fP6flpaWkKCwtTamqqQkNDrzg/AM9gt1sr8r37rrRokZSZaR0vXVrq2VN68EGpZUurdF0sK0vauFE6etR6hlSbNpK/f8nnBwAARa8w3cDoiNO5c+e0detWxcfHO475+fkpPj5emzZtKtDnOH36tM6fP6+KFSvm+/HMzEylpaXl2gD4HptNuvHGS49CXXedNQo1deqfo1BLlkjR0dKtt0q9ellvo6Ot4wAAwLcYLU7Hjx9XVlaWwsPDcx0PDw9XcnJygT7Hk08+qcjIyFzl62Ljx49XWFiYY4uKirri3AA8W8WK0tCh0s6d1mhSnz7WvVCJidLDD1v3QsXHS127Sr/+mvu1hw9L995LeQIAwNcYv8fpSrz00ktasGCBli5dquDg4HzPGTlypFJTUx1bUlJSCacE4K5yRqFmz7ZGof79b2sUKiNDWrcu/9fkTG4eNsyaxgcAAHyD0eJUuXJl+fv7KyUlJdfxlJQUVatW7bKvfe211/TSSy/p008/VdOmTS95XlBQkEJDQ3NtAPBXFStaZWjnTmtJ88ux26WkJOnVV6Vdu6QzZ0okIgAAMMhocQoMDFSLFi207qI/7WZnZ2vdunWKi4u75OteeeUVPffcc1q9erVatmxZElEB+AibTapSpWDnjhxpPTOqdGlr4YjWra1pf6NGSdOnSxs2SAcPMjIFAIA3CDAdYPjw4erfv79atmyp6667ThMmTFBGRoYGDhwoSerXr5+qV6+u8ePHS5JefvlljR49WvPmzVN0dLTjXqiyZcuqbNmyxr4PAN4jIqJg59WpI/32m7WYRHKytX3zTd7zAgKkmjWlWrWsrXbt3PuVKuVdzQ8AALgX48Wpe/fu+u233zR69GglJyerWbNmWr16tWPBiEOHDsnP78+BsSlTpujcuXO69957c32eMWPG6Nlnny3J6AC8VJs2Uo0a1kIQ+T2wwWazPr57t+TnJ/3xh7R/v7Rvn/X24v2DB6Vz56RffrG2/JQtm3+pytnKlCne7xcAADhn/DlOJY3nOAEoiCVLrNXzpNzlKWdkaPFiqUsX558nK8taeCKnUP21YBXk2d1Vq+ZfqmrXlqKirBEtAABQeIXpBhQnALiEJUusZcsvXpI8Ksp6BlRBSlNBnD1rjUrlV6r27ZNOnrz86/39rUz5lapatazSxTRAAADyR3G6DIoTgMLIyrKe9XT0qHXvU5s2VlkpKSdP5i1UOe8fOCBlZl7+9aVL5536d3HJKleu+LKbvnYAADhDcboMihMAb5GdbS1Ikd+9Vfv3WyNlzv4LX7nypUvVVVdJgYGuZctvtK5GDWup96IarQMA4EpRnC6D4gTAV2RmSocO5V+q9u+Xfv/98q/387PKzqVWA6xWLf9pgDn3h/31X5fC3h8GAEBxozhdBsUJACxpaZe+t+rAAecP9g0OlqKj845SPfqo9JfnmjvkrEi4fz/T9gAA5lGcLoPiBADO2e1W+bnUMutJSdZUQVf16SM1bmwttV6QLSTEdxa54N4wACg5FKfLoDgBwJU7fz73NMCcUrVly6WfV3UlbDZroYucInXxflFs7lJMuDfMdRRO13HtXMN1c427XTeK02VQnACg+GzYIN16q/PzOnWSypeXMjIuv509W8yB/19QUOHLVkHLW2BgwUbLuDfMdRRO13HtXMN1c407XjeK02VQnACg+GRlWfc9HT6c/4p+hb3HKStLOn3a2pyVLFe2kvgX0N+/YFMRFyyQTp269OepVEmaPNkqYgEB1uf19/9z/69vXf2Yn1/xX5OiROF0HdfONVw317jrdaM4XQbFCQCKV84/jlLufyBN/+P4V3a7NaKVX6EqiqJ2/rzp79B1hSlcxVHeCvoxPz9p9OjLPyi6UiVp0iTrfJvNfTbJ7NfPzpaaNZOOHMn/utlsUmSktHOndb0vPn7x24LsF+bcS73OXeT8cejiEZOLsQBO/tz5ulGcLoPiBADFL7/pGFFR0oQJ7lGaSsL58wUvWd9+axVKZ+rXt4pAVpZ04cLl3zr7GOCpSqKoXWo/K6tgU4jLlJFKlXL++U0fK6mvlZFhrdbqzPr10i23OD+vKBWmGwRc9qMAALigSxepY0f3ugG4pJUqZd3HVb6883M3bChYcZo6tWh+qbDbrRGHKylernysqD/3/v3S5s3Ov98GDaQqVazv+0q2nGtnenMlhze5+Ptx1+8tI8N0As909KjpBJdHcQIAFAt//5L/y6GnatPGmqbi7N6wNm2K5uvZbH9OewsMLJrPaUJBFyOZMoWfRSl3kdqwQYqPd/6aVaukm266dFlxtu9tr/v2W6l3bzk1a5bUqtXlP7/pYyX5tRITpeHD5VREhPNzTGKqHgAAbsBT7g1zJ0W9GIkv4dq5huvmGne+boXpBh62dg4AAN6pSxerHFWvnvt4jRqUpkvx97eWMZbyLiSQ8/6ECfwCmx+unWu4bq7xlutGcQIAwE106WLdQL1+vTRvnvV2/35K0+VQOF3HtXMN18013nDdmKoHAAA8XlaWby9GciW4dq7hurnG3a4by5FfBsUJAAAAgMQ9TgAAAABQpChOAAAAAOAExQkAAAAAnKA4AQAAAIATFCcAAAAAcILiBAAAAABOUJwAAAAAwAmKEwAAAAA4QXECAAAAACcoTgAAAADgRIDpACXNbrdLktLS0gwnAQAAAGBSTifI6QiX43PFKT09XZIUFRVlOAkAAAAAd5Cenq6wsLDLnmOzF6ReeZHs7GwdOXJE5cqVk81mMx1HaWlpioqKUlJSkkJDQ03H8RhcN9dw3VzDdXMd1841XDfXcN1cx7VzDdfNNe503ex2u9LT0xUZGSk/v8vfxeRzI05+fn6qUaOG6Rh5hIaGGv/B8URcN9dw3VzDdXMd1841XDfXcN1cx7VzDdfNNe5y3ZyNNOVgcQgAAAAAcILiBAAAAABOUJwMCwoK0pgxYxQUFGQ6ikfhurmG6+YarpvruHau4bq5huvmOq6da7hurvHU6+Zzi0MAAAAAQGEx4gQAAAAATlCcAAAAAMAJihMAAAAAOEFxAgAAAAAnKE6GfPnll0pISFBkZKRsNpuWLVtmOpJHGD9+vK699lqVK1dOVatWVadOnbR7927TsdzelClT1LRpU8eD5uLi4rRq1SrTsTzOSy+9JJvNpmHDhpmO4taeffZZ2Wy2XFuDBg1Mx/IYhw8fVp8+fVSpUiWFhITommuu0ZYtW0zHcmvR0dF5fuZsNpuGDBliOppby8rK0qhRo1SrVi2FhISoTp06eu6558S6Yc6lp6dr2LBhqlmzpkJCQnTDDTfou+++Mx3L7Tj7fddut2v06NGKiIhQSEiI4uPj9fPPP5sJWwAUJ0MyMjIUExOjyZMnm47iUb744gsNGTJE3377rdauXavz58/rzjvvVEZGhulobq1GjRp66aWXtHXrVm3ZskW33XabOnbsqJ07d5qO5jG+++47vfPOO2ratKnpKB6hcePGOnr0qGP76quvTEfyCH/88Ydat26tUqVKadWqVfrxxx/1+uuvq0KFCqajubXvvvsu18/b2rVrJUn33Xef4WTu7eWXX9aUKVM0adIk7dq1Sy+//LJeeeUVvfXWW6ajub37779fa9eu1ezZs/XDDz/ozjvvVHx8vA4fPmw6mltx9vvuK6+8ookTJ2rq1Kn673//qzJlyqht27Y6e/ZsCSctIDuMk2RfunSp6Rge6dixY3ZJ9i+++MJ0FI9ToUIF+3/+8x/TMTxCenq6vV69eva1a9fab775ZvvQoUNNR3JrY8aMscfExJiO4ZGefPJJ+4033mg6hscbOnSovU6dOvbs7GzTUdxahw4d7IMGDcp1rEuXLvbevXsbSuQZTp8+bff397evWLEi1/HmzZvbn376aUOp3N9ff9/Nzs62V6tWzf7qq686jp08edIeFBRknz9/voGEzjHiBI+WmpoqSapYsaLhJJ4jKytLCxYsUEZGhuLi4kzH8QhDhgxRhw4dFB8fbzqKx/j5558VGRmp2rVrq3fv3jp06JDpSB5h+fLlatmype677z5VrVpVsbGxmjZtmulYHuXcuXOaM2eOBg0aJJvNZjqOW7vhhhu0bt067dmzR5K0fft2ffXVV2rXrp3hZO7twoULysrKUnBwcK7jISEhjK4Xwv79+5WcnJzr39awsDC1atVKmzZtMpjs0gJMBwBclZ2drWHDhql169Zq0qSJ6Thu74cfflBcXJzOnj2rsmXLaunSpWrUqJHpWG5vwYIF2rZtG3PXC6FVq1aaMWOG6tevr6NHj2rs2LFq06aNduzYoXLlypmO59b27dunKVOmaPjw4Xrqqaf03Xff6bHHHlNgYKD69+9vOp5HWLZsmU6ePKkBAwaYjuL2RowYobS0NDVo0ED+/v7KysrSCy+8oN69e5uO5tbKlSunuLg4Pffcc2rYsKHCw8M1f/58bdq0SXXr1jUdz2MkJydLksLDw3MdDw8Pd3zM3VCc4LGGDBmiHTt28NedAqpfv74SExOVmpqqxYsXq3///vriiy8oT5eRlJSkoUOHau3atXn+sohLu/iv1U2bNlWrVq1Us2ZNLVq0SH/7298MJnN/2dnZatmypV588UVJUmxsrHbs2KGpU6dSnArovffeU7t27RQZGWk6ittbtGiR5s6dq3nz5qlx48ZKTEzUsGHDFBkZyc+bE7Nnz9agQYNUvXp1+fv7q3nz5urZs6e2bt1qOhqKEVP14JEeeeQRrVixQuvXr1eNGjVMx/EIgYGBqlu3rlq0aKHx48crJiZGb775pulYbm3r1q06duyYmjdvroCAAAUEBOiLL77QxIkTFRAQoKysLNMRPUL58uV19dVXa+/evaajuL2IiIg8f8xo2LAhUx0L6ODBg/rss890//33m47iEf75z39qxIgR6tGjh6655hr17dtXjz/+uMaPH286mturU6eOvvjiC506dUpJSUnavHmzzp8/r9q1a5uO5jGqVasmSUpJScl1PCUlxfExd0Nxgkex2+165JFHtHTpUn3++eeqVauW6UgeKzs7W5mZmaZjuLXbb79dP/zwgxITEx1by5Yt1bt3byUmJsrf3990RI9w6tQp/fLLL4qIiDAdxe21bt06zyMW9uzZo5o1axpK5FmmT5+uqlWrqkOHDqajeITTp0/Lzy/3r4L+/v7Kzs42lMjzlClTRhEREfrjjz+0Zs0adezY0XQkj1GrVi1Vq1ZN69atcxxLS0vTf//7X7e9B5upeoacOnUq119f9+/fr8TERFWsWFFXXXWVwWTubciQIZo3b54++ugjlStXzjEHNiwsTCEhIYbTua+RI0eqXbt2uuqqq5Senq558+Zpw4YNWrNmjelobq1cuXJ57p8rU6aMKlWqxH11l/HEE08oISFBNWvW1JEjRzRmzBj5+/urZ8+epqO5vccff1w33HCDXnzxRXXr1k2bN2/Wu+++q3fffdd0NLeXnZ2t6dOnq3///goI4NebgkhISNALL7ygq666So0bN9b333+vN954Q4MGDTIdze2tWbNGdrtd9evX1969e/XPf/5TDRo00MCBA01HcyvOft8dNmyYnn/+edWrV0+1atXSqFGjFBkZqU6dOpkLfTmml/XzVevXr7dLyrP179/fdDS3lt81k2SfPn266WhubdCgQfaaNWvaAwMD7VWqVLHffvvt9k8//dR0LI/EcuTOde/e3R4REWEPDAy0V69e3d69e3f73r17TcfyGB9//LG9SZMm9qCgIHuDBg3s7777rulIHmHNmjV2Sfbdu3ebjuIx0tLS7EOHDrVfddVV9uDgYHvt2rXtTz/9tD0zM9N0NLe3cOFCe+3ate2BgYH2atWq2YcMGWI/efKk6Vhux9nvu9nZ2fZRo0bZw8PD7UFBQfbbb7/drf8/bLPbeTw0AAAAAFwO9zgBAAAAgBMUJwAAAABwguIEAAAAAE5QnAAAAADACYoTAAAAADhBcQIAAAAAJyhOAAAAAOAExQkAAAAAnKA4AQBQCDabTcuWLTMdAwBQwihOAACPMWDAANlstjzbXXfdZToaAMDLBZgOAABAYdx1112aPn16rmNBQUGG0gAAfAUjTgAAjxIUFKRq1arl2ipUqCDJmkY3ZcoUtWvXTiEhIapdu7YWL16c6/U//PCDbrvtNoWEhKhSpUp64IEHdOrUqVznvP/++2rcuLGCgoIUERGhRx55JNfHjx8/rs6dO6t06dKqV6+eli9fXrzfNADAOIoTAMCrjBo1Sl27dtX27dvVu3dv9ejRQ7t27ZIkZWRkqG3btqpQoYK+++47ffDBB/rss89yFaMpU6ZoyJAheuCBB/TDDz9o+fLlqlu3bq6vMXbsWHXr1k3/+9//1L59e/Xu3VsnTpwo0e8TAFCybHa73W46BAAABTFgwADNmTNHwcHBuY4/9dRTeuqpp2Sz2fTQQw9pypQpjo9df/31at68ud5++21NmzZNTz75pJKSklSmTBlJ0sqVK5WQkKAjR44oPDxc1atX18CBA/X888/nm8Fms+mZZ57Rc889J8kqY2XLltWqVau41woAvBj3OAEAPMqtt96aqxhJUsWKFR37cXFxuT4WFxenxMRESdKuXbsUExPjKE2S1Lp1a2VnZ2v37t2y2Ww6cuSIbr/99stmaNq0qWO/TJkyCg0N1bFjx1z9lgAAHoDiBADwKGXKlMkzda6ohISEFOi8UqVK5XrfZrMpOzu7OCIBANwE9zgBALzKt99+m+f9hg0bSpIaNmyo7du3KyMjw/Hxr7/+Wn5+fqpfv77KlSun6OhorVu3rkQzAwDcHyNOAACPkpmZqeTk5FzHAgICVLlyZUnSBx98oJYtW+rGG2/U3LlztXnzZr333nuSpN69e2vMmDHq37+/nn32Wf3222969NFH1bdvX4WHh0uSnn32WT300EOqWrWq2rVrp/T0dH399dd69NFHS/YbBQC4FYoTAMCjrF69WhEREbmO1a9fXz/99JMka8W7BQsWaPDgwYqIiND8+fPVqFEjSVLp0qW1Zs0aDR06VNdee61Kly6trl276o033nB8rv79++vs2bP697//rSeeeEKVK1fWvffeW3LfIADALbGqHgDAa9hsNi1dulSdOnUyHQUA4GW4xwkAAAAAnKA4AQAAAIAT3OMEAPAazD4HABQXRpwAAAAAwAmKEwAAAAA4QXECAAAAACcoTgAAAADgBMUJAAAAAJygOAEAAACAExQnAAAAAHCC4gQAAAAATvwfNybdWQxhs64AAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -168,6 +20168,13 @@ "plt.xticks(epochs_list)\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/norch/__init__.py b/norch/__init__.py index ca7be0b..3eb0b7f 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -2,6 +2,7 @@ from norch.tensor import Tensor from .nn import * from .optim import * from .utils import * +from .datasets import * __version__ = "0.0.1" __author__ = 'Lucas de Lima Nogueira' diff --git a/norch/__pycache__/__init__.cpython-38.pyc b/norch/__pycache__/__init__.cpython-38.pyc index d172276..8202d1d 100644 Binary files a/norch/__pycache__/__init__.cpython-38.pyc and b/norch/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 5a33b4a..bcc1fe0 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/autograd/__pycache__/functions.cpython-38.pyc b/norch/autograd/__pycache__/functions.cpython-38.pyc index d48733e..71b5662 100644 Binary files a/norch/autograd/__pycache__/functions.cpython-38.pyc and b/norch/autograd/__pycache__/functions.cpython-38.pyc differ diff --git a/norch/autograd/functions.py b/norch/autograd/functions.py index 8ce8625..c9a393e 100644 --- a/norch/autograd/functions.py +++ b/norch/autograd/functions.py @@ -1,4 +1,5 @@ import math +import norch class AddBackward: def __init__(self, x, y): @@ -15,6 +16,7 @@ class AddBroadcastedBackward: x, y = self.input grad_x = self._reshape_gradient(gradient, x.shape) grad_y = self._reshape_gradient(gradient, y.shape) + return [grad_x, grad_y] def _reshape_gradient(self, gradient, shape): @@ -25,11 +27,17 @@ class AddBroadcastedBackward: # Sum along axes where the target shape dimension is 1 for i in range(len(shape)): if shape[i] == 1: - gradient = gradient.sum(axis=i) + gradient = gradient.sum(axis=i, keepdim=True) return gradient +class SubBackward: + def __init__(self, x, y): + self.input = [x, y] + def backward(self, gradient): + return [gradient, -gradient] + class SubBroadcastedBackward: def __init__(self, x, y): self.input = [x, y] @@ -49,15 +57,7 @@ class SubBroadcastedBackward: for i in range(len(shape)): if shape[i] == 1: gradient = gradient.sum(axis=i) - return gradient - -class SubBackward: - def __init__(self, x, y): - self.input = [x, y] - - def backward(self, gradient): - return [gradient, -gradient] class ScalarMulBackward: def __init__(self, x, scalar): @@ -82,7 +82,13 @@ class MatmulBackward: def backward(self, gradient): x, y = self.input - return [gradient @ y.transpose(-1,-2), x.transpose(-1,-2) @ gradient] + + if x.ndim != y.ndim: # broadcasted case + aux = (gradient @ y.transpose(-1,-2)) + aux_sum = aux.sum(axis=0) + return [aux_sum, x.transpose(-1,-2) @ gradient] + else: + return [gradient @ y.transpose(-1,-2), x.transpose(-1,-2) @ gradient] """class PowBackward: def __init__(self, x, power): @@ -121,12 +127,29 @@ class LogBackward: return [grad_input] class SumBackward: - def __init__(self, x): + def __init__(self, x, axis=None, keepdim=False): self.input = [x] + self.axis = axis + self.keepdim = keepdim def backward(self, gradient): - # Since sum reduces a tensor to a scalar, gradient is broadcasted to match the original shape. - return [float(gradient.tensor.contents.data[0]) * self.input[0].ones_like()] + input_shape = self.input[0].shape.copy() + if self.axis == -1: + # If axis is None, sum reduces the tensor to a scalar. + grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + else: + + if self.keepdim: + input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:] + else: + input_shape = input_shape[:self.axis] + input_shape[self.axis+1:] + + # Broadcast the gradient to the input shape along the specified axis. + grad_output_shape = list(input_shape) + grad_output = gradient.reshape(grad_output_shape) + grad_output = grad_output + self.input[0].zeros_like() + + return [grad_output] class ReshapeBackward: def __init__(self, x): @@ -177,5 +200,95 @@ class CosBackward: def backward(self, gradient): x = self.input[0] return [-gradient * x.sin()] + +class MaxBackward: + def __init__(self, x, axis=None, keepdim=False): + self.input = [x] + self.axis = axis + self.keepdim = keepdim + + def backward(self, gradient): + input_shape = self.input[0].shape.copy() + if self.axis == -1: + max_value = self.input[0].max() + mask = self.input[0].equal(max_value) + + grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + + grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0] + + else: + + if self.keepdim: + input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:] + else: + input_shape = input_shape[:self.axis] + input_shape[self.axis+1:] + + # Broadcast the gradient to the input shape along the specified axis. + grad_output_shape = list(input_shape) + + grad_output = gradient.reshape(grad_output_shape) + grad_output = grad_output + self.input[0].zeros_like() + max_values = self.input[0].max(axis=self.axis, keepdim=True) + mask = self.input[0].equal(max_values) + + grad_output = (grad_output * mask) + + return [grad_output] + + +class MinBackward: + def __init__(self, x, axis=None, keepdim=False): + self.input = [x] + self.axis = axis + self.keepdim = keepdim + + def backward(self, gradient): + input_shape = self.input[0].shape.copy() + if self.axis == -1: + min_value = self.input[0].min() + mask = self.input[0].equal(min_value) + + grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + + grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0] + + else: + if self.keepdim: + input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:] + else: + input_shape = input_shape[:self.axis] + input_shape[self.axis+1:] + + # Broadcast the gradient to the input shape along the specified axis. + grad_output_shape = list(input_shape) + grad_output = gradient.reshape(grad_output_shape) + grad_output = grad_output + self.input[0].zeros_like() + max_values = self.input[0].min(axis=self.axis, keepdim=True) + mask = self.input[0].equal(max_values) + + grad_output = (grad_output * mask) + + return [grad_output] + + +class CrossEntropyLossBackward: + def __init__(self, logits, targets): + self.input = [logits, targets] + + def backward(self, gradient): + logits, targets = self.input + + if logits.ndim == 1: + softmax = norch.softmax(logits, dim=0) + grad_logits = (softmax - targets) + + elif logits.ndim == 2: + # batched + batch_size = logits.shape[0] + softmax = norch.softmax(logits, dim=1) + + grad_logits = (softmax - targets) / batch_size + + return [grad_logits, None] # targets do not have a gradient diff --git a/norch/csrc/cpu.cpp b/norch/csrc/cpu.cpp index bde7b0c..9584c23 100644 --- a/norch/csrc/cpu.cpp +++ b/norch/csrc/cpu.cpp @@ -11,7 +11,7 @@ void add_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { } } -void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape) { +void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; // Calculate strides for broadcasting @@ -28,12 +28,12 @@ void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_ int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; - stride1 *= broadcasted_shape[i]; - stride2 *= broadcasted_shape[i]; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; } // Perform element-wise addition with broadcasting - for (int i = 0; i < tensor1->size; i++) { + for (int i = 0; i < broadcasted_size; i++) { int index1 = 0, index2 = 0; int linear_index = i; for (int j = max_ndim - 1; j >= 0; j--) { @@ -58,7 +58,7 @@ void sub_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { } } -void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape) { +void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; // Calculate strides for broadcasting @@ -75,12 +75,12 @@ void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_ int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; - stride1 *= broadcasted_shape[i]; - stride2 *= broadcasted_shape[i]; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; } // Perform element-wise addition with broadcasting - for (int i = 0; i < tensor1->size; i++) { + for (int i = 0; i < broadcasted_size; i++) { int index1 = 0, index2 = 0; int linear_index = i; for (int j = max_ndim - 1; j >= 0; j--) { @@ -205,7 +205,7 @@ void log_tensor_cpu(Tensor* tensor, float* result_data) { } } -void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis) { +void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis) { if (axis == -1) { // Sum over all elements float sum = 0.0; @@ -219,26 +219,14 @@ void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis) { return; } - int result_shape[tensor->ndim - 1]; - int result_size = 1; int axis_stride = tensor->strides[axis]; - int idx = 0; - for (int i = 0; i < tensor->ndim; i++) { - if (i != axis) { - result_shape[idx++] = tensor->shape[i]; - result_size *= tensor->shape[i]; - } - } - - memset(result_data, 0, result_size * sizeof(float)); - for (int i = 0; i < tensor->shape[axis]; i++) { - for (int j = 0; j < result_size; j++) { + for (int j = 0; j < size; j++) { int index = 0; int remainder = j; for (int k = tensor->ndim - 2; k >= 0; k--) { - index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; + index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; remainder /= result_shape[k]; } result_data[j] += tensor->data[index + i * axis_stride]; @@ -247,8 +235,115 @@ void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis) { } } +void max_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis) { + if (axis == -1) { + float max_value = -INFINITY; + for (int i = 0; i < tensor->size; i++) { + max_value = fmax(max_value, tensor->data[i]); + } + *result_data = max_value; + } else { + for (int i = 0; i < size; i++) { + result_data[i] = -INFINITY; + } + if (axis < 0 || axis >= tensor->ndim) { + printf("Invalid axis"); + return; + } + + int axis_stride = tensor->strides[axis]; + for (int i = 0; i < tensor->shape[axis]; i++) { + for (int j = 0; j < size; j++) { + int index = 0; + int remainder = j; + for (int k = tensor->ndim - 2; k >= 0; k--) { + index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; + remainder /= result_shape[k]; + } + result_data[j] = fmax(result_data[j], tensor->data[index + i * axis_stride]); + } + } + } +} +void min_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis) { + if (axis == -1) { + float min_value = INFINITY; + for (int i = 0; i < tensor->size; i++) { + min_value = fmin(min_value, tensor->data[i]); + } + *result_data = min_value; + } else { + for (int i = 0; i < size; i++) { + result_data[i] = INFINITY; + } + if (axis < 0 || axis >= tensor->ndim) { + printf("Invalid axis"); + return; + } + + int axis_stride = tensor->strides[axis]; + + for (int i = 0; i < tensor->shape[axis]; i++) { + for (int j = 0; j < size; j++) { + int index = 0; + int remainder = j; + for (int k = tensor->ndim - 2; k >= 0; k--) { + index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; + remainder /= result_shape[k]; + } + result_data[j] = fmin(result_data[j], tensor->data[index + i * axis_stride]); + } + } + } +} + +void equal_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + for (int i = 0; i < tensor1->size; i++) { + result_data[i] = (tensor1->data[i] == tensor2->data[i]) ? 1.0f : 0.0f; + } +} + +void equal_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Calculate strides for broadcasting + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + // Perform element-wise equal with broadcasting + for (int i = 0; i < broadcasted_size; i++) { + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = (tensor1->data[index1] == tensor2->data[index2]) ? 1.0f : 0.0f; + } + + // Free strides + free(strides1); + free(strides2); +} void ones_like_tensor_cpu(Tensor* tensor, float* result_data) { diff --git a/norch/csrc/cpu.h b/norch/csrc/cpu.h index fad467c..83bd83f 100644 --- a/norch/csrc/cpu.h +++ b/norch/csrc/cpu.h @@ -4,10 +4,12 @@ #include "tensor.h" void add_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); -void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape); -void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis); +void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); +void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* shape, int axis); +void max_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis); +void min_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis); void sub_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); -void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape); +void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); void elementwise_mul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); void scalar_div_tensor_cpu(float scalar, Tensor* tensor, float* result_data); void tensor_div_scalar_cpu(Tensor* tensor, float scalar, float* result_data); @@ -19,6 +21,8 @@ void scalar_pow_tensor_cpu(float base, Tensor* tensor, float* result_data); void tensor_pow_scalar_cpu(Tensor* tensor, float exponent, float* result_data); void log_tensor_cpu(Tensor* tensor, float* result_data); void scalar_mul_tensor_cpu(Tensor* tensor, float scalar, float* result_data); +void equal_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void equal_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); void ones_like_tensor_cpu(Tensor* tensor, float* result_data); void zeros_like_tensor_cpu(Tensor* tensor, float* result_data); void transpose_1D_tensor_cpu(Tensor* tensor, float* result_data); diff --git a/norch/csrc/cuda.cu b/norch/csrc/cuda.cu index 9ebe6d3..7d056ab 100644 --- a/norch/csrc/cuda.cu +++ b/norch/csrc/cuda.cu @@ -71,7 +71,7 @@ __global__ void add_broadcasted_tensor_cuda_kernel(float* data1, float* data2, f result_data[i] = data1[index1] + data2[index2]; } -__host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape) { +__host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; int* strides1 = (int*)malloc(max_ndim * sizeof(int)); @@ -87,8 +87,8 @@ __host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, floa int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; - stride1 *= broadcasted_shape[i]; - stride2 *= broadcasted_shape[i]; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; } int* d_broadcasted_shape; @@ -104,8 +104,8 @@ __host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, floa cudaMalloc((void**)&d_strides2, max_ndim * sizeof(int)); cudaMemcpy(d_strides2, strides2, max_ndim * sizeof(int), cudaMemcpyHostToDevice); - int number_of_blocks = (tensor1->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; - add_broadcasted_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, d_broadcasted_shape, d_strides1, d_strides2, max_ndim, tensor1->size); + int number_of_blocks = (broadcasted_size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + add_broadcasted_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, d_broadcasted_shape, d_strides1, d_strides2, max_ndim, broadcasted_size); cudaError_t error = cudaGetLastError(); if (error != cudaSuccess) { @@ -187,6 +187,67 @@ __host__ void sub_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_da cudaDeviceSynchronize(); } +__global__ void sub_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= size) return; + + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = data1[index1] - data2[index2]; +} + +__host__ void sub_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + int* d_broadcasted_shape; + int* d_strides1; + int* d_strides2; + + cudaMalloc((void**)&d_broadcasted_shape, max_ndim * sizeof(int)); + cudaMemcpy(d_broadcasted_shape, broadcasted_shape, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides1, max_ndim * sizeof(int)); + cudaMemcpy(d_strides1, strides1, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides2, max_ndim * sizeof(int)); + cudaMemcpy(d_strides2, strides2, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + int number_of_blocks = (broadcasted_size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sub_broadcasted_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, d_broadcasted_shape, d_strides1, d_strides2, max_ndim, broadcasted_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + cudaFree(d_broadcasted_shape); +} + __global__ void elementwise_mul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -481,6 +542,89 @@ __host__ void log_tensor_cuda(Tensor* tensor, float* result_data) { cudaDeviceSynchronize(); } +__global__ void equal_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = (data1[i] == data2[i]) ? 1.0f : 0.0f; + } +} + +__host__ void equal_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int number_of_blocks = (tensor1->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + equal_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, tensor1->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void equal_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= size) return; + + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = (data1[index1] == data2[index2]) ? 1.0f : 0.0f; +} + +__host__ void equal_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + int* d_broadcasted_shape; + int* d_strides1; + int* d_strides2; + + cudaMalloc((void**)&d_broadcasted_shape, max_ndim * sizeof(int)); + cudaMemcpy(d_broadcasted_shape, broadcasted_shape, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides1, max_ndim * sizeof(int)); + cudaMemcpy(d_strides1, strides1, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides2, max_ndim * sizeof(int)); + cudaMemcpy(d_strides2, strides2, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + int number_of_blocks = (broadcasted_size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + equal_broadcasted_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, d_broadcasted_shape, d_strides1, d_strides2, max_ndim, broadcasted_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + cudaFree(d_broadcasted_shape); +} + __global__ void ones_like_tensor_cuda_kernel(float* data, float* result_data, int size) { diff --git a/norch/csrc/cuda.h b/norch/csrc/cuda.h index 5d5e353..73eb831 100644 --- a/norch/csrc/cuda.h +++ b/norch/csrc/cuda.h @@ -8,7 +8,10 @@ __host__ void add_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); __global__ void add_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size); - __host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape); + __host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); + + __global__ void sub_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size); + __host__ void sub_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); __global__ void sum_tensor_cuda_kernel(float* data, float* result_data); __host__ void sum_tensor_cuda(Tensor* tensor, float* result_data); @@ -46,6 +49,12 @@ __global__ void log_tensor_cuda_kernel(float* data, float* result_data, int size); __host__ void log_tensor_cuda(Tensor* tensor, float* result_data); + __global__ void equal_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size); + __host__ void equal_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void equal_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size); + __host__ void equal_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); + __global__ void ones_like_tensor_cuda_kernel(float* data, float* result_data, int size); __host__ void ones_like_tensor_cuda(Tensor* tensor, float* result_data); diff --git a/norch/csrc/tensor.cpp b/norch/csrc/tensor.cpp index 104994e..889fff6 100644 --- a/norch/csrc/tensor.cpp +++ b/norch/csrc/tensor.cpp @@ -149,26 +149,30 @@ extern "C" { broadcasted_shape[max_ndim - 1 - i] = dim1 > dim2 ? dim1 : dim2; } + int broadcasted_size = 1; + for (int i = 0; i < max_ndim; i++) { + broadcasted_size *= broadcasted_shape[i]; + } + if (strcmp(tensor1->device, "cuda") == 0) { float* result_data; - cudaMalloc((void **)&result_data, tensor1->size * sizeof(float)); - add_broadcasted_tensor_cuda(tensor1, tensor2, result_data, broadcasted_shape); + cudaMalloc((void **)&result_data, broadcasted_size * sizeof(float)); + add_broadcasted_tensor_cuda(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); } else { - float* result_data = (float*)malloc(tensor1->size * sizeof(float)); + float* result_data = (float*)malloc(broadcasted_size * sizeof(float)); if (result_data == NULL) { fprintf(stderr, "Memory allocation failed\n"); exit(1); } - add_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape); + add_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); } } - Tensor* sum_tensor(Tensor* tensor, int axis) { - + Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdim) { char* device = (char*)malloc(strlen(tensor->device) + 1); if (device != NULL) { strcpy(device, tensor->device); @@ -179,11 +183,8 @@ extern "C" { int ndim; int* shape; if (axis == -1) { + shape = (int*) malloc(sizeof(int)); - if (shape == NULL) { - fprintf(stderr, "Memory allocation failed\n"); - exit(1); - } shape[0] = 1; ndim = 1; } else { @@ -195,21 +196,179 @@ extern "C" { } ndim = tensor->ndim - 1; } + + int size = 1; + for (int i = 0; i < ndim; i++) { + size *= shape[i]; + } if (strcmp(tensor->device, "cuda") == 0) { float* result_data; - cudaMalloc((void**)&result_data, tensor->size * sizeof(float)); + cudaMalloc((void**)&result_data, size * sizeof(float)); + cudaMemset(result_data, 0, size * sizeof(float)); sum_tensor_cuda(tensor, result_data); return create_tensor(result_data, shape, ndim, device); } else { - float* result_data = (float*)malloc(1 * sizeof(float)); + float* result_data = (float*)calloc(size, sizeof(float)); if (result_data == NULL) { fprintf(stderr, "Memory allocation failed\n"); exit(1); } - sum_tensor_cpu(tensor, result_data, axis); + + sum_tensor_cpu(tensor, result_data, size, shape, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + + } + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* max_tensor(Tensor* tensor, int axis, bool keepdim) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim; + int* shape; + if (axis == -1) { + shape = (int*) malloc(sizeof(int)); + shape[0] = 1; + ndim = 1; + } else { + shape = (int*) malloc((tensor->ndim - 1) * sizeof(int)); + for (int i = 0, j = 0; i < tensor->ndim; ++i) { + if (i != axis) { + shape[j++] = tensor->shape[i]; + } + } + ndim = tensor->ndim - 1; + } + + int size = 1; + for (int i = 0; i < ndim; i++) { + size *= shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void**)&result_data, size * sizeof(float)); + //cudaMemset(result_data, -INFINITY, size * sizeof(float)); + //max_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + max_tensor_cpu(tensor, result_data, size, shape, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + } + + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* min_tensor(Tensor* tensor, int axis, bool keepdim) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim; + int* shape; + if (axis == -1) { + + shape = (int*) malloc(sizeof(int)); + shape[0] = 1; + ndim = 1; + } else { + shape = (int*) malloc((tensor->ndim - 1) * sizeof(int)); + for (int i = 0, j = 0; i < tensor->ndim; ++i) { + if (i != axis) { + shape[j++] = tensor->shape[i]; + } + } + ndim = tensor->ndim - 1; + } + + int size = 1; + for (int i = 0; i < ndim; i++) { + size *= shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void**)&result_data, size * sizeof(float)); + //cudaMemset(result_data, INFINITY, size * sizeof(float)); + //min_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + min_tensor_cpu(tensor, result_data, size, shape, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + } + return create_tensor(result_data, shape, ndim, device); } } @@ -290,16 +449,27 @@ extern "C" { broadcasted_shape[max_ndim - 1 - i] = dim1 > dim2 ? dim1 : dim2; } - // Allocate memory for result tensor - float* result_data = (float*)malloc(tensor1->size * sizeof(float)); - if (result_data == NULL) { - fprintf(stderr, "Memory allocation failed\n"); - exit(1); + int broadcasted_size = 1; + for (int i = 0; i < max_ndim; i++) { + broadcasted_size *= broadcasted_shape[i]; } - sub_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape); + if (strcmp(tensor1->device, "cuda") == 0) { + float* result_data; + cudaMalloc((void **)&result_data, broadcasted_size * sizeof(float)); + sub_broadcasted_tensor_cuda(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + else { + float* result_data = (float*)malloc(broadcasted_size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } - return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + sub_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } } Tensor* elementwise_mul_tensor(Tensor* tensor1, Tensor* tensor2) { @@ -469,7 +639,7 @@ extern "C" { Tensor* tensor_div_tensor(Tensor* tensor1, Tensor* tensor2) { if (tensor1->ndim != tensor2->ndim) { - fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for element-wise multiplication\n", tensor1->ndim, tensor2->ndim); + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for element-wise division\n", tensor1->ndim, tensor2->ndim); exit(1); } @@ -494,7 +664,7 @@ extern "C" { for (int i = 0; i < ndim; i++) { if (tensor1->shape[i] != tensor2->shape[i]) { - fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for subtraction\n", tensor1->shape[i], tensor2->shape[i], i); + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for division\n", tensor1->shape[i], tensor2->shape[i], i); exit(1); } shape[i] = tensor1->shape[i]; @@ -872,6 +1042,106 @@ extern "C" { } } + Tensor* equal_tensor(Tensor* tensor1, Tensor* tensor2) { + if (tensor1->ndim != tensor2->ndim) { + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for equal\n", tensor1->ndim, tensor2->ndim); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor1->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + if (tensor1->shape[i] != tensor2->shape[i]) { + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for equal\n", tensor1->shape[i], tensor2->shape[i], i); + exit(1); + } + shape[i] = tensor1->shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor1->size * sizeof(float)); + equal_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor1->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + equal_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* equal_broadcasted_tensor(Tensor* tensor1, Tensor* tensor2) { + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Determine the broadcasted shape + int* broadcasted_shape = (int*)malloc(max_ndim * sizeof(int)); + if (broadcasted_shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + for (int i = 0; i < max_ndim; i++) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - 1 - i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - 1 - i] : 1; + if (dim1 != dim2 && dim1 != 1 && dim2 != 1) { + fprintf(stderr, "Shapes are not compatible for broadcasting\n"); + exit(1); + } + broadcasted_shape[max_ndim - 1 - i] = dim1 > dim2 ? dim1 : dim2; + } + + int broadcasted_size = 1; + for (int i = 0; i < max_ndim; i++) { + broadcasted_size *= broadcasted_shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + float* result_data; + cudaMalloc((void **)&result_data, broadcasted_size * sizeof(float)); + equal_broadcasted_tensor_cuda(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + else { + float* result_data = (float*)malloc(broadcasted_size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + equal_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + } + + Tensor* ones_like_tensor(Tensor* tensor) { char* device = (char*)malloc(strlen(tensor->device) + 1); if (device != NULL) { diff --git a/norch/csrc/tensor.h b/norch/csrc/tensor.h index 5ff62d0..5669d4f 100644 --- a/norch/csrc/tensor.h +++ b/norch/csrc/tensor.h @@ -16,7 +16,9 @@ extern "C" { Tensor* create_tensor(float* data, int* shape, int ndim, char* device); float get_item(Tensor* tensor, int* indices); Tensor* add_tensor(Tensor* tensor1, Tensor* tensor2); - Tensor* sum_tensor(Tensor* tensor, int axis); + Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdims); + Tensor* max_tensor(Tensor* tensor, int axis, bool keepdim); + Tensor* min_tensor(Tensor* tensor, int axis, bool keepdim); Tensor* sub_tensor(Tensor* tensor1, Tensor* tensor2); Tensor* elementwise_mul_tensor(Tensor* tensor1, Tensor* tensor2); Tensor* scalar_mul_tensor(Tensor* tensor, float scalar); @@ -28,6 +30,8 @@ extern "C" { Tensor* tensor_pow_scalar(Tensor* tensor, float exponent); Tensor* scalar_pow_tensor(float base, Tensor* tensor); Tensor* log_tensor(Tensor* tensor); + Tensor* equal_tensor(Tensor* tensor1, Tensor* tensor2); + Tensor* equal_broadcasted_tensor(Tensor* tensor1, Tensor* tensor2); void to_device(Tensor* tensor, char* device); Tensor* ones_like_tensor(Tensor* tensor); Tensor* zeros_like_tensor(Tensor* tensor); diff --git a/norch/datasets/__init__.py b/norch/datasets/__init__.py new file mode 100644 index 0000000..fe34b11 --- /dev/null +++ b/norch/datasets/__init__.py @@ -0,0 +1 @@ +from .mnist import * \ No newline at end of file diff --git a/norch/datasets/mnist.py b/norch/datasets/mnist.py new file mode 100644 index 0000000..ff11edf --- /dev/null +++ b/norch/datasets/mnist.py @@ -0,0 +1,90 @@ +import gzip +import os +import norch +from norch.utils.data import Dataset +import numpy as np + +class MNIST(Dataset): + """ + Loads training, validation, and test partitions of the mnist dataset + (http://yann.lecun.com/exdb/mnist/). If the data is not already contained in data_dir, it will + try to download it. + + This dataset contains 60000 training examples, and 10000 test examples of handwritten digits + in {0, ..., 9} and corresponding labels. Each handwritten image has an "original" dimension of + 28x28x1, and is stored row-wise as a string of 784x1 bytes. Pixel values are in range 0 to 255 + (inclusive). + + Args: + data_dir: String. Relative or absolute path of the dataset. + devel_size: Integer. Size of the development (validation) dataset partition. + + Returns: + X_train: float64 numpy array with shape [784, 60000-devel_size] with values in [0, 1]. + Y_train: uint8 numpy array with shape [60000-devel_size]. Labels. + X_devel: float64 numpy array with shape [784, devel_size] with values in [0, 1]. + Y_devel: uint8 numpy array with shape [devel_size]. Labels. + X_test: float64 numpy array with shape [784, 10000] with values in [0, 1]. + Y_test: uint8 numpy array with shape [10000]. Labels. + """ + + urls = ['https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz',] + name = 'mnist-data-py' + dirname = 'mnist' + + def __init__(self, path_data, path_label, transform=None, target_transform=None): + self.data = self._load_mnist(path_data, header_size=16).reshape((-1, 28, 28)) + self.labels = self._load_mnist(path_label, header_size=8) + self.transform = transform + self.target_transform = target_transform + + def _load_mnist(self, path, header_size): + with gzip.open(path, 'rb') as f: + data = np.frombuffer(f.read(), np.uint8, offset=header_size) + return np.asarray(data, dtype=np.uint8) + + + @classmethod + def splits(cls, root='.data', train_data='train-images-idx3-ubyte.gz', train_label='train-labels-idx1-ubyte.gz', + test_data='t10k-images-idx3-ubyte.gz', test_label='t10k-labels-idx1-ubyte.gz', **kwargs): + r""" + Loads training and test partitions of the [mnist dataset](https://www.cs.toronto.edu/~kriz/cifar.html). If + the data is not already contained in the ``root`` folder, it will download it. + + Args: + root (str): relative or absolute path of the dataset. + + Returns: + tuple(Dataset): training and testing datasets + """ + path = os.path.join(root, cls.dirname, cls.name) + if not os.path.isdir(path): + path = cls.download(root) + train_data = os.path.join(path, train_data) + train_label = os.path.join(path, train_label) + test_data = os.path.join(path, test_data) + test_label = os.path.join(path, test_label) + return MNIST(train_data, train_label, **kwargs), MNIST(test_data, test_label, **kwargs) + + def __getitem__(self, item): + data = self.data[item].tolist() + label = self.labels[item].tolist() + + if self.transform is not None: + data = self.transform(data) + + if self.target_transform is not None: + label = self.target_transform(label) + + + + return data, label + + def __setitem__(self, key, value): + self.data[key], self.labels[key] = value + + def __len__(self): + return len(self.data) diff --git a/norch/libtensor.so b/norch/libtensor.so index 06f1149..81254ad 100755 Binary files a/norch/libtensor.so and b/norch/libtensor.so differ diff --git a/norch/nn/__init__.py b/norch/nn/__init__.py index dadface..8f7997b 100644 --- a/norch/nn/__init__.py +++ b/norch/nn/__init__.py @@ -1,3 +1,4 @@ from .modules import * from .activation import * -from .loss import * \ No newline at end of file +from .loss import * +from .functional import * \ No newline at end of file diff --git a/norch/nn/__pycache__/__init__.cpython-38.pyc b/norch/nn/__pycache__/__init__.cpython-38.pyc index 1f19ab6..5b7fd1c 100644 Binary files a/norch/nn/__pycache__/__init__.cpython-38.pyc and b/norch/nn/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/activation.cpython-38.pyc b/norch/nn/__pycache__/activation.cpython-38.pyc index f6d4dfe..0a2693b 100644 Binary files a/norch/nn/__pycache__/activation.cpython-38.pyc and b/norch/nn/__pycache__/activation.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/loss.cpython-38.pyc b/norch/nn/__pycache__/loss.cpython-38.pyc index 170a29b..b8133b9 100644 Binary files a/norch/nn/__pycache__/loss.cpython-38.pyc and b/norch/nn/__pycache__/loss.cpython-38.pyc differ diff --git a/norch/nn/activation.py b/norch/nn/activation.py index 21cf891..a1361f6 100644 --- a/norch/nn/activation.py +++ b/norch/nn/activation.py @@ -1,4 +1,5 @@ from .module import Module +from . import functional as F import math class Activation(Module): @@ -17,4 +18,13 @@ class Sigmoid(Activation): super().__init__() def forward(self, x): - return 1.0 / (1.0 + (math.e) ** (-x)) \ No newline at end of file + return F.sigmoid(x) + +class Softmax(Activation): + def __init__(self, dim): + super(Softmax, self).__init__() + + self.dim = dim + + def forward(self, x): + return F.softmax(x, self.dim) \ No newline at end of file diff --git a/norch/nn/functional.py b/norch/nn/functional.py new file mode 100644 index 0000000..9493591 --- /dev/null +++ b/norch/nn/functional.py @@ -0,0 +1,33 @@ +import math +import norch +import numpy as np + +def sigmoid(x): + return 1.0 / (1.0 + (math.e) ** (-x)) + +def softmax(x, dim=None): + if dim is not None and dim < 0: + dim = x.ndim + dim + + x_max = x.max(axis=dim, keepdim=True) + exp_x = math.e ** (x - x_max) + + if dim is not None: + sum_exp_x = exp_x.sum(axis=dim, keepdim=True) + exp_x.zeros_like() + return exp_x / sum_exp_x + else: + sum_exp_x = exp_x.sum() + return exp_x / sum_exp_x + +def one_hot_encode(x, num_classes): + one_hot = [[0] * num_classes for _ in range(x.numel)] + + # Set the appropriate elements to 1 + for i in range(x.numel): + target_idx = int(x.tensor.contents.data[i]) + one_hot[i][target_idx] = 1 + + if x.numel < 2: + one_hot = one_hot[0] + + return norch.Tensor(one_hot) \ No newline at end of file diff --git a/norch/nn/loss.py b/norch/nn/loss.py index 789fab6..8d7b0f9 100644 --- a/norch/nn/loss.py +++ b/norch/nn/loss.py @@ -1,4 +1,6 @@ from .module import Module +from norch.autograd.functions import * +import norch from abc import ABC class Loss(Module, ABC): @@ -18,8 +20,66 @@ class MSELoss(Loss): def __init__(self): super().__init__() - def forward(self, predictions, labels): - assert labels.shape == predictions.shape, \ - "Labels and predictions shape does not match: {} and {}".format(labels.shape, predictions.shape) + def forward(self, predictions, target): + assert target.shape == predictions.shape, \ + "Labels and predictions shape does not match: {} and {}".format(target.shape, predictions.shape) - return ((predictions - labels) ** 2).sum() / predictions.numel \ No newline at end of file + cost = ((predictions - target) ** 2).sum() / predictions.numel + return cost + + +class CrossEntropyLoss(Loss): + def __init__(self): + super().__init__() + + def forward(self, input, target): + assert isinstance(input, norch.Tensor), \ + "Cross entropy argument 'input' must be Tensor, not {}".format(type(input)) + + assert isinstance(target, norch.Tensor), \ + "Cross entropy argument 'target' must be Tensor, not {}".format(type(target)) + + if input.ndim == 1: + if target.numel == 1: + num_classes = input.shape[0] + target = norch.one_hot_encode(target, num_classes) + + logits = norch.softmax(input, dim=0) + cost = -(logits.log() * target).sum() + + else: + # target -> class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + logits = norch.softmax(input, dim=0) + cost = -(logits.log() * target).sum() + + + elif input.ndim == 2: + # batched + if target.ndim == 1: + # target -> Ground truth class indices: + num_classes = input.shape[1] + + target = norch.one_hot_encode(target, num_classes) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + cost = -(logits.log() * target).sum() / batch_size + + else: + # target -> class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + cost = -(logits.log() * target).sum() / batch_size + + if input.requires_grad: + cost.grad_fn = CrossEntropyLossBackward(input, target) + + return cost + + + diff --git a/norch/nn/modules/__pycache__/linear.cpython-38.pyc b/norch/nn/modules/__pycache__/linear.cpython-38.pyc index d4a4c2c..583d8b6 100644 Binary files a/norch/nn/modules/__pycache__/linear.cpython-38.pyc and b/norch/nn/modules/__pycache__/linear.cpython-38.pyc differ diff --git a/norch/nn/modules/linear.py b/norch/nn/modules/linear.py index 2d61023..b90d845 100644 --- a/norch/nn/modules/linear.py +++ b/norch/nn/modules/linear.py @@ -2,15 +2,23 @@ from ..module import Module from ..parameter import Parameter class Linear(Module): - def __init__(self, input_dim, output_dim): + def __init__(self, input_dim, output_dim, bias=True): super().__init__() self.input_dim = input_dim self.output_dim = output_dim self.weight = Parameter(shape=[self.output_dim, self.input_dim]) - self.bias = Parameter(shape=[self.output_dim, 1]) + + if bias: + self.bias = Parameter(shape=[self.output_dim, 1]) + else: + self.bias = None def forward(self, x): - z = self.weight @ x + self.bias + if self.bias: + z = self.weight @ x + self.bias + else: + z = self.weight @ x + return z def inner_repr(self): diff --git a/norch/tensor.py b/norch/tensor.py index 03b37ee..1121066 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -19,13 +19,18 @@ class Tensor: def __init__(self, data=None, device="cpu", requires_grad=False): if data != None: + if isinstance(data, (float, int)): + data = [data] + data, shape = self.flatten(data) - self.data_ctype = (ctypes.c_float * len(data))(*data) - self.shape_ctype = (ctypes.c_int * len(shape))(*shape) + + self.shape = shape.copy() + + self.data_ctype = (ctypes.c_float * len(data))(*data.copy()) + self.shape_ctype = (ctypes.c_int * len(shape))(*shape.copy()) self.ndim_ctype = ctypes.c_int(len(shape)) self.device_ctype = device.encode('utf-8') - self.shape = shape self.ndim = len(shape) self.device = device @@ -109,6 +114,25 @@ class Tensor: return result_data def reshape(self, new_shape): + # Calculate the total number of elements in the tensor + total_elements = self.numel + + # Check for the presence of -1 in new_shape + if new_shape.count(-1) > 1: + raise ValueError("Only one dimension can be inferred (set to -1).") + + inferred_dim = None + known_dims_product = 1 + for dim in new_shape: + if dim == -1: + inferred_dim = dim + else: + known_dims_product *= dim + + # Calculate the inferred dimension if -1 is present + if inferred_dim == -1: + inferred_dim_size = total_elements // known_dims_product + new_shape = [inferred_dim_size if dim == -1 else dim for dim in new_shape] new_shape_ctype = (ctypes.c_int * len(new_shape))(*new_shape) new_ndim_ctype = ctypes.c_int(len(new_shape)) @@ -129,7 +153,41 @@ class Tensor: result_data.grad_fn = ReshapeBackward(self) return result_data - + + def unsqueeze(self, dim): + if dim < 0: + dim = self.ndim + dim + 1 + + # Ensure the dimension is valid + if dim > self.ndim: + raise ValueError("Dimension out of range (expected to be in range of [0, {0}], but got {1})".format(self.ndim, dim)) + + # Create the new shape with an extra dimension of size 1 + new_shape = self.shape[:dim] + [1] + self.shape[dim:] + + return self.reshape(new_shape) + + def squeeze(self, dim=None): + if dim is not None: + if dim < 0: + dim = self.ndim + dim + + # Ensure the dimension is valid + if dim >= self.ndim or dim < 0: + raise ValueError("Dimension out of range (expected to be in range of [0, {0}), but got {1})".format(self.ndim, dim)) + + # Only squeeze the specified dimension if its size is 1 + if self.shape[dim] != 1: + raise ValueError("Dimension {0} does not have size 1 and cannot be squeezed".format(dim)) + + # Create the new shape without the specified dimension + new_shape = self.shape[:dim] + self.shape[dim+1:] + else: + # Create the new shape by removing all dimensions of size 1 + new_shape = [s for s in self.shape if s != 1] + + return self.reshape(new_shape) + def to(self, device): self.device = device self.device_ctype = self.device.encode('utf-8') @@ -224,6 +282,8 @@ class Tensor: if isinstance(other, (int, float)): other = other * self.ones_like() + broadcasted_shape_add = [] + # Function to determine if broadcasting is needed and get the broadcasted shape def broadcast_shape(shape1, shape2): if shape1 == shape2: @@ -233,17 +293,25 @@ class Tensor: shape1 = [1] * (max_len - len(shape1)) + shape1 shape2 = [1] * (max_len - len(shape2)) + shape2 - broadcasted_shape = [] + for dim1, dim2 in zip(shape1, shape2): if dim1 != dim2 and dim1 != 1 and dim2 != 1: raise ValueError("Shapes are not compatible for broadcasting") - broadcasted_shape.append(max(dim1, dim2)) - return broadcasted_shape, True + broadcasted_shape_add.append(max(dim1, dim2)) + return broadcasted_shape_add, True - broadcasted_shape, needs_broadcasting = broadcast_shape(self.shape, other.shape) + broadcasted_shape_add, needs_broadcasting = broadcast_shape(self.shape, other.shape) if needs_broadcasting: # Call add_broadcasted_tensor if broadcasting is needed + if other.ndim == self.ndim - 1: + other = other.reshape([1] + other.shape) + + elif self.ndim == other.ndim - 1: + self = self.reshape([1] + self.shape) + + + Tensor._C.add_broadcasted_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] Tensor._C.add_broadcasted_tensor.restype = ctypes.POINTER(CTensor) @@ -251,11 +319,13 @@ class Tensor: result_data = Tensor() result_data.tensor = result_tensor_ptr - result_data.shape = broadcasted_shape - result_data.ndim = len(broadcasted_shape) + result_data.shape = broadcasted_shape_add.copy() + result_data.ndim = len(broadcasted_shape_add) result_data.device = self.device - result_data.numel = self.numel # Update this to calculate the correct number of elements if broadcasting + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s result_data.requires_grad = self.requires_grad or other.requires_grad if result_data.requires_grad: @@ -312,6 +382,8 @@ class Tensor: if isinstance(other, (int, float)): other = other * self.ones_like() + broadcasted_shape_sub = [] + # Function to determine if broadcasting is needed and get the broadcasted shape def broadcast_shape(shape1, shape2): if shape1 == shape2: @@ -321,16 +393,22 @@ class Tensor: shape1 = [1] * (max_len - len(shape1)) + shape1 shape2 = [1] * (max_len - len(shape2)) + shape2 - broadcasted_shape = [] + for dim1, dim2 in zip(shape1, shape2): if dim1 != dim2 and dim1 != 1 and dim2 != 1: raise ValueError("Shapes are not compatible for broadcasting") - broadcasted_shape.append(max(dim1, dim2)) - return broadcasted_shape, True + broadcasted_shape_sub.append(max(dim1, dim2)) + return broadcasted_shape_sub, True - broadcasted_shape, needs_broadcasting = broadcast_shape(self.shape, other.shape) + broadcasted_shape_sub, needs_broadcasting = broadcast_shape(self.shape, other.shape) if needs_broadcasting: + if other.ndim == self.ndim - 1: + other = other.reshape([1] + other.shape) + + elif self.ndim == other.ndim - 1: + self = self.reshape([1] + self.shape) + # Call add_broadcasted_tensor if broadcasting is needed Tensor._C.sub_broadcasted_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] Tensor._C.sub_broadcasted_tensor.restype = ctypes.POINTER(CTensor) @@ -339,8 +417,8 @@ class Tensor: result_data = Tensor() result_data.tensor = result_tensor_ptr - result_data.shape = broadcasted_shape - result_data.ndim = len(broadcasted_shape) + result_data.shape = broadcasted_shape_sub.copy() + result_data.ndim = len(broadcasted_shape_sub) result_data.device = self.device result_data.numel = self.numel # Update this to calculate the correct number of elements if broadcasting @@ -569,6 +647,9 @@ class Tensor: result_data.grad_fn = DivisionBackward(self, other) elif isinstance(self, Tensor) and isinstance(other, Tensor): + if other.numel == 1: + return self.__truediv__(other.tensor.contents.data[0]) + Tensor._C.tensor_div_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] Tensor._C.tensor_div_tensor.restype = ctypes.POINTER(CTensor) @@ -611,6 +692,80 @@ class Tensor: return result_data + def equal(self, other): + + if isinstance(other, Tensor) and other.numel == 1: + # other is a single value tensor + other = self.zeros_like() + other + return self.equal(other) + + if not isinstance(other, Tensor): + # other is a single value + if isinstance(other, (int, float)): + other = self.zeros_like() + other + + return self.equal(other) + else: + return False + + broadcasted_shape_add = [] + + # Function to determine if broadcasting is needed and get the broadcasted shape + def broadcast_shape(shape1, shape2): + if shape1 == shape2: + return shape1, False + + max_len = max(len(shape1), len(shape2)) + shape1 = [1] * (max_len - len(shape1)) + shape1 + shape2 = [1] * (max_len - len(shape2)) + shape2 + + + for dim1, dim2 in zip(shape1, shape2): + if dim1 != dim2 and dim1 != 1 and dim2 != 1: + raise ValueError("Shapes are not compatible for broadcasting") + broadcasted_shape_add.append(max(dim1, dim2)) + return broadcasted_shape_add, True + + broadcasted_shape_add, needs_broadcasting = broadcast_shape(self.shape, other.shape) + + if needs_broadcasting: + if other.ndim == self.ndim - 1: + other = other.reshape([1] + other.shape) + + elif self.ndim == other.ndim - 1: + self = self.reshape([1] + self.shape) + + # Call equal_broadcasted_tensor if broadcasting is needed + Tensor._C.equal_broadcasted_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.equal_broadcasted_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.equal_broadcasted_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = broadcasted_shape_add.copy() + result_data.ndim = len(broadcasted_shape_add) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + else: + Tensor._C.equal_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.equal_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.equal_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + return result_data + def log(self): Tensor._C.log_tensor.argtypes = [ctypes.POINTER(CTensor)] Tensor._C.log_tensor.restype = ctypes.POINTER(CTensor) @@ -630,20 +785,34 @@ class Tensor: return result_data - def sum(self, axis=-1): - Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int] + def sum(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + + if axis == None: + axis = -1 + + Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.sum_tensor.restype = ctypes.POINTER(CTensor) - result_tensor_ptr = Tensor._C.sum_tensor(self.tensor, axis) + result_tensor_ptr = Tensor._C.sum_tensor(self.tensor, axis, keepdim) result_data = Tensor() result_data.tensor = result_tensor_ptr if axis == -1: - result_data.shape = [1] - result_data.ndim = 1 + if keepdim: + result_data.ndim = self.ndim + result_data.shape = [1] * self.ndim + + else: + result_data.shape = [1] + result_data.ndim = 1 else: - result_data.shape = self.shape[:axis] + self.shape[axis+1:] + if keepdim: + result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:] + else: + result_data.shape = self.shape[:axis] + self.shape[axis+1:] result_data.ndim = len(result_data.shape) result_data.device = self.device @@ -653,7 +822,89 @@ class Tensor: result_data.requires_grad = self.requires_grad if result_data.requires_grad: - result_data.grad_fn = SumBackward(self) + result_data.grad_fn = SumBackward(self, axis, keepdim=keepdim) + + return result_data + + def max(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + + if axis == None: + axis = -1 + + Tensor._C.max_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] + Tensor._C.max_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.max_tensor(self.tensor, axis, keepdim) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + + if axis == -1: + if keepdim: + result_data.ndim = self.ndim + result_data.shape = [1] * self.ndim + + else: + result_data.shape = [1] + result_data.ndim = 1 + else: + if keepdim: + result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:] + else: + result_data.shape = self.shape[:axis] + self.shape[axis+1:] + result_data.ndim = len(result_data.shape) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = MaxBackward(self, axis, keepdim=keepdim) + + return result_data + + def min(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + + if axis == None: + axis = -1 + + Tensor._C.min_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] + Tensor._C.min_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.min_tensor(self.tensor, axis, keepdim) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + + if axis == -1: + if keepdim: + result_data.ndim = self.ndim + result_data.shape = [1] * self.ndim + + else: + result_data.shape = [1] + result_data.ndim = 1 + else: + if keepdim: + result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:] + else: + result_data.shape = self.shape[:axis] + self.shape[axis+1:] + result_data.ndim = len(result_data.shape) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = MinBackward(self, axis, keepdim=keepdim) return result_data diff --git a/norch/utils/__init__.py b/norch/utils/__init__.py index 90f60fd..c6adfd1 100644 --- a/norch/utils/__init__.py +++ b/norch/utils/__init__.py @@ -1 +1,2 @@ -from .utils import * \ No newline at end of file +from .utils import * +from .data import * \ No newline at end of file diff --git a/norch/utils/__pycache__/utils.cpython-38.pyc b/norch/utils/__pycache__/utils.cpython-38.pyc index 95d3b45..d78c80a 100644 Binary files a/norch/utils/__pycache__/utils.cpython-38.pyc and b/norch/utils/__pycache__/utils.cpython-38.pyc differ diff --git a/norch/utils/data/__init__.py b/norch/utils/data/__init__.py new file mode 100644 index 0000000..2397554 --- /dev/null +++ b/norch/utils/data/__init__.py @@ -0,0 +1,4 @@ +from .dataset import * +from .example import * +from .dataloader import * +from .batch import * \ No newline at end of file diff --git a/norch/utils/data/batch.py b/norch/utils/data/batch.py new file mode 100644 index 0000000..3c9b92c --- /dev/null +++ b/norch/utils/data/batch.py @@ -0,0 +1,13 @@ +class Batch(object): + """ + A ``Batch``depends on a ``Dataset`` and is made of ``batch_size`` ``Example``. + """ + def __init__(self, example, batch_size): + self.example = example + self.batch_size = batch_size + + def __getitem__(self, item): + return self.example[item] + + def __len__(self): + return self.batch_size \ No newline at end of file diff --git a/norch/utils/data/dataloader.py b/norch/utils/data/dataloader.py new file mode 100644 index 0000000..01bd41e --- /dev/null +++ b/norch/utils/data/dataloader.py @@ -0,0 +1,20 @@ +import numpy as np +from .batch import Batch + + +class Dataloader(object): + + def __init__(self, dataset, batch_size=32): + self.dataset = dataset + self.batch_size = batch_size + + def __iter__(self): + starts = np.arange(0, len(self.dataset), self.batch_size) + + for start in starts: + end = start + self.batch_size + batch_size = min(end, len(self.dataset)) - start + yield Batch(self.dataset[start:end], batch_size) + + def __len__(self): + return len(self.dataset) // self.batch_size diff --git a/norch/utils/data/dataset.py b/norch/utils/data/dataset.py new file mode 100644 index 0000000..d38c1ea --- /dev/null +++ b/norch/utils/data/dataset.py @@ -0,0 +1,74 @@ +from abc import ABC, abstractmethod +import os +from norch.utils import extract_to_dir, download_from_url +from .example import Example +import norch + + +class Dataset(ABC): + r""" + Abstract Dataset class. All dataset for machine learning purposes can inherits from this architecture, + for convenience. + """ + urls = [] + name = '' + dirname = '' + + def __init__(self, examples, fields): + self.examples = examples + self.fields = fields + + @classmethod + def splits(cls, train=None, test=None, valid=None, root='.'): + raise NotImplementedError + + @classmethod + def download(cls, root): + r"""Download and unzip a web archive (.zip, .gz, or .tgz). + + Args: + root (str): Folder to download data to. + + Returns: + string: Path to extracted dataset. + """ + path_dirname = os.path.join(root, cls.dirname) + path_name = os.path.join(path_dirname, cls.name) + if not os.path.isdir(path_dirname): + for url in cls.urls: + filename = os.path.basename(url) + zpath = os.path.join(path_dirname, filename) + if not os.path.isfile(zpath): + if not os.path.exists(os.path.dirname(zpath)): + os.makedirs(os.path.dirname(zpath)) + print(f'Download {filename} from {url} to {zpath}') + download_from_url(url, zpath) + extract_to_dir(zpath, path_name) + + return path_name + + def __repr__(self): + name = self.__class__.__name__ + string = f"Dataset {name}(" + tab = " " + for (key, value) in self.__dict__.items(): + if key[0] != "_": + if isinstance(value, Example): + fields = self.fields + for (name, field) in fields: + string += f"\n{tab}({name}): {field.__class__.__name__}" \ + f"(transform={True if field.transform is not None else None}, dtype={field.dtype})" + elif isinstance(value, norch.Tensor): + string += f"\n{tab}({key}): {value.__class__.__name__}(shape={value.shape}, dtype={value.dtype})" + else: + string += f"\n{tab}({key}): {value.__class__.__name__}" + return f'{string}\n)' + + def __getitem__(self, item): + return self.examples[item] + + def __setitem__(self, key, value): + self.examples[key] = value + + def __len__(self): + return len(self.examples) diff --git a/norch/utils/data/example.py b/norch/utils/data/example.py new file mode 100644 index 0000000..a4f4ebf --- /dev/null +++ b/norch/utils/data/example.py @@ -0,0 +1,65 @@ +# File: example.py +# Creation: Wednesday August 19th 2020 +# Author: Arthur Dujardin +# Contact: arthur.dujardin@ensg.eu +# arthurd@ifi.uio.no +# -------- +# Copyright (c) 2020 Arthur Dujardin + + +class Field(object): + r""" + A ``Field`` defines the data to process from a raw dataset. It will convert the data into a tensor. The data can + be a string, integers, float etc. The data is meant to be preprocessed and the attribute ``transform`` handles + the way the user want to process the raw data. + """ + + def __init__(self, transform=None, dtype=None): + self.transform = transform + self.dtype = dtype + + def process(self, value): + """Applies the transformation and changes the data's type if necessary. + + Args: + value (Tensor): + + Returns: + + """ + if self.transform is not None: + value = self.transform(value) + if self.dtype is not None: + value = value.astype(self.dtype) + return value + + +class Example(object): + r""" + Store a single training / testing example, and store it as an attribute. + Highly inspired from PyTorch `Example `__. + + """ + @classmethod + def fromlist(cls, values, fields): + """ + Add an example from a list of data with respect to the fields. + + Args: + values: raw data + fields (tuple(string, Field)): fields to preprocess the data on + + Returns: + None + """ + example = cls() + for (value, field) in zip(values, fields): + assert len(field) == 2, f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + assert isinstance(field[1], Field), f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + name = field[0] + value = field[1].process(value) + setattr(example, name, value) + + return example diff --git a/norch/utils/utils.py b/norch/utils/utils.py index 1652e16..f8b9b30 100644 --- a/norch/utils/utils.py +++ b/norch/utils/utils.py @@ -1,4 +1,9 @@ import random +import requests +import tarfile +import zipfile +import shutil +import os def generate_random_list(shape): """ @@ -11,4 +16,105 @@ def generate_random_list(shape): if len(inner_shape) == 0: return [random.uniform(-1, 1) for _ in range(shape[0])] else: - return [generate_random_list(inner_shape) for _ in range(shape[0])] \ No newline at end of file + return [generate_random_list(inner_shape) for _ in range(shape[0])] + + +def download_from_url(url, save_path, chunk_size=128): + """Download a file from an URL. + + Original answer from https://stackoverflow.com/questions/9419162/download-returned-zip-file-from-url + + Args: + url (str): path to the URL. + save_path (str): path to the saving directory. + chunk_size (int): download chunk. + + Returns: + None + """ + response = requests.get(url, stream=True) + total = response.headers.get('content-length') + with open(save_path, 'wb') as f: + if total is None: + f.write(response.content) + else: + downloaded = 0 + total = int(total) + for data in response.iter_content(chunk_size=max(int(total / 1000), 1024 * 1024)): + downloaded += len(data) + f.write(data) + progress_bar(downloaded, total, "Downloading...") + + +def extract_to_dir(filename, dirpath='.'): + # Does not create folder twice with the same name + name, ext = os.path.splitext(filename) + # if os.path.basename(name) == os.path.basename(dirpath): + # dirpath = '.' + # Extract + print(dirpath) + print("Extracting...", end="") + if tarfile.is_tarfile(filename): + tarfile.open(filename, 'r').extractall(dirpath) + elif zipfile.is_zipfile(filename): + zipfile.ZipFile(filename, 'r').extractall(dirpath) + elif ext == '.gz': + if not os.path.exists(dirpath): + os.mkdir(dirpath) + shutil.move(filename, os.path.join(dirpath, os.path.basename(filename))) + print(f" | NOTE: gzip files are not extracted, and moved to {dirpath}", end="") + # Return the path where the file was extracted + print(" | Done !") + return os.path.abspath(dirpath) + +def progress_bar(current_index, max_index, prefix=None, suffix=None, start_time=None): + """Display a progress bar and duration. + + Args: + current_index (int): current state index (or epoch number). + max_index (int): maximal numbers of state. + prefix (str, optional): prefix of the progress bar. The default is None. + suffix (str, optional): suffix of the progress bar. The default is None. + start_time (float, optional): starting time of the progress bar. If not None, it will display the time + spent from the beginning to the current state. The default is None. + + Returns: + None. Display the progress bar in the console. + """ + # Add a prefix to the progress bar + prefix = "" if prefix is None else str(prefix) + " " + + # Get the percentage + percentage = current_index * 100 // max_index + loading = "[" + "=" * (percentage // 2) + " " * (50 - percentage // 2) + "]" + progress_display = "\r{0}{1:3d}% | {2}".format(prefix, percentage, loading) + + # Add a suffix to the progress bar + progress_display += "" if suffix is None else " | " + str(suffix) + + # Add a timer + if start_time is not None: + time_min, time_sec = get_time(start_time, time.time()) + time_display = " | Time: {0}m {1}s".format(time_min, time_sec) + progress_display += time_display + + # Print the progress bar + # TODO: return a string instead + print(progress_display, end="{}".format("" if current_index < max_index else " | Done !\n")) + +def get_time(start_time, end_time): + """Get ellapsed time in minutes and seconds. + + Args: + start_time (float): strarting time + end_time (float): ending time + + Returns: + elapsed_mins (float): elapsed time in minutes + elapsed_secs (float): elapsed time in seconds. + """ + elapsed_time = end_time - start_time + elapsed_mins = int(elapsed_time / 60) + elapsed_secs = int(elapsed_time - (elapsed_mins * 60)) + + return elapsed_mins, elapsed_secs diff --git a/test.py b/test.py new file mode 100644 index 0000000..7ad7e2d --- /dev/null +++ b/test.py @@ -0,0 +1,75 @@ +import norch +from norch.utils.data.dataloader import Dataloader +import norch +import norch.nn as nn +import norch.optim as optim +import random +random.seed(1) + + +to_tensor = lambda x: norch.Tensor(x) +reshape = lambda x: x.reshape([-1, 784]) +transform = lambda x: reshape(to_tensor(x)) +target_transform = lambda x: to_tensor(x) + +train_data, test_data = norch.datasets.MNIST.splits(transform=transform, target_transform=target_transform) +sample, _ = train_data[0] + +BATCH_SIZE = 100 + +train_loader = Dataloader(train_data, batch_size = BATCH_SIZE) + +class MyModel(nn.Module): + def __init__(self): + super(MyModel, self).__init__() + self.fc1 = nn.Linear(784, 5) + self.sigmoid = nn.Sigmoid() + self.fc2 = nn.Linear(5, 10) + + def forward(self, x): + out = self.fc1(x) + out = self.sigmoid(out) + out = self.fc2(out) + + return out + +device = "cpu" +epochs = 10 + +model = MyModel().to(device) +criterion = nn.CrossEntropyLoss() +optimizer = optim.SGD(model.parameters(), lr=0.001) +loss_list = [] + +for epoch in range(epochs): + for idx, batch in enumerate(train_loader): + x, target = batch + + x = x.unsqueeze(-1) + target = target + + x = x.to(device) + target = target.to(device).unsqueeze(-1) + + outputs = model(x) + print(outputs.shape, target.shape, x.shape) + loss = criterion(outputs, target) + + optimizer.zero_grad() + loss.backward() + print('loss_antes', loss) + + print('f1 antes', model.fc1.bias) + print('f1 grad_antes', model.fc1.bias.grad) + print('f2 antes', model.fc2.bias) + print('f2 grad_antes', model.fc2.bias.grad) + + optimizer.step() + print('\n') + + print('f1 depois', model.fc1.bias) + print('f2 depois', model.fc2.bias) + print('\n\n') + + print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}') + loss_list.append(loss[0]) \ No newline at end of file diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 42431a8..4739e7c 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -34,7 +34,204 @@ class TestTensorAutograd(unittest.TestCase): self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) - def test_broadcasting_addition_autograd(self): + def test_sum_axis(self): + """ + Test autograd from sum specifying axis + """ + norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + norch_result = (norch_tensor1 + norch_tensor2).sum(axis=0).sum(axis=0).sum() + + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + + torch_result = (torch_tensor1 + torch_tensor2).sum(axis=0).sum(axis=0).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + norch_result = (norch_tensor1 + norch_tensor2).sum(axis=1).sum() + + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + + torch_result = (torch_tensor1 + torch_tensor2).sum(axis=1).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + + def test_max(self): + """ + Test autograd from max + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result = norch_tensor.max() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_result = torch_tensor.max() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + def test_max_axis(self): + """ + Test autograd from max specifying axis + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_max_axis = norch_tensor.max(axis=1) + norch_result = norch_max_axis.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_max_axis, _ = torch_tensor.max(axis=1) + torch_result = torch_max_axis.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + norch_tensor = norch.Tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + norch_max_axis = norch_tensor.max(axis=2) + norch_result = norch_max_axis.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + + torch_max_axis, _ = torch_tensor.max(axis=2) + torch_result = torch_max_axis.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + ## evaluate case with some repeated values and axis 2 + + #def test_max_axis(self): + # """ + # Test autograd from max specifying axis + # """ + # + # norch_tensor = norch.Tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # norch_max_axis = norch_tensor.max(axis=2) + # norch_result = norch_max_axis.sum() + # + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + # + # torch_tensor = torch.tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # + # torch_max_axis, _ = torch_tensor.max(axis=2) + # torch_result = torch_max_axis.sum() + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + #def test_min_axis(self): + # """ + # Test autograd from max specifying axis + # """ + # + # norch_tensor = norch.Tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # norch_min_axis = norch_tensor.min(axis=2) + # norch_result = norch_min_axis.sum() + # + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + # + # torch_tensor = torch.tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # + # torch_min_axis, _ = torch_tensor.min(axis=2) + # torch_result = torch_min_axis.sum() + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + + def test_min(self): + """ + Test autograd from min + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result = norch_tensor.min() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_result = torch_tensor.min() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + def test_min_axis(self): + """ + Test autograd from min specifying axis + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_min = norch_tensor.min(axis=1) + norch_result = norch_min.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + + torch_min, _ = torch_tensor.min(axis=1) + torch_result = torch_min.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + norch_tensor = norch.Tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + norch_min = norch_tensor.min(axis=2) + norch_result = norch_min.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + + torch_min, _ = torch_tensor.min(axis=2) + torch_result = torch_min.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + def test_broadcasted_addition_autograd(self): """ Test autograd for broadcasting addition: tensor1 + tensor2 """ @@ -54,6 +251,26 @@ class TestTensorAutograd(unittest.TestCase): self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + ## reversed order broadcasting + norch_tensor1 = norch.Tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + + norch_result = (norch_tensor2 + norch_tensor1).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + + torch_result = (torch_tensor2 + torch_tensor1).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) def test_subtraction(self): """ @@ -76,7 +293,7 @@ class TestTensorAutograd(unittest.TestCase): self.assertTrue(utils.compare_torch(norch_tensor1_grad_sub, torch_tensor1_grad_sub)) self.assertTrue(utils.compare_torch(norch_tensor2_grad_sub, torch_tensor2_grad_sub)) - def test_broadcasting_subtraction_autograd(self): + def test_broadcasted_subtraction_autograd(self): """ Test autograd for broadcasting subtraction: tensor1 - tensor2 """ @@ -96,6 +313,26 @@ class TestTensorAutograd(unittest.TestCase): self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + # reversed order broadcasting + norch_tensor1 = norch.Tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + + norch_result = (norch_tensor2 - norch_tensor1).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + + torch_result = (torch_tensor2 - torch_tensor1).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) def test_division(self): @@ -211,7 +448,71 @@ class TestTensorAutograd(unittest.TestCase): self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul)) self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul)) - + def test_batched_matmul(self): + """ + Test autograd from batched matrix multiplication: BxMxP = BxNxM @ BxMxP + """ + B = 3 # Batch size + + norch_tensor1_matmul = norch.Tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)], requires_grad=True).to(self.device) + norch_tensor2_matmul = norch.Tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True).to(self.device) + + norch_result_matmul = norch_tensor1_matmul @ norch_tensor2_matmul + norch_result_matmul_sum = norch_result_matmul.sum() # Sum over all elements + norch_result_matmul_sum.backward() + + # Convert gradients to torch tensors + norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad).to(self.device) + norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) + + # Repeat the same process with torch tensors + torch_tensor1_matmul = torch.tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)], requires_grad=True).to(self.device) + torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True).to(self.device) + torch_result_matmul = torch.matmul(torch_tensor1_matmul, torch_tensor2_matmul) + torch_result_matmul_sum = torch_result_matmul.sum() + torch_result_matmul_sum.backward() + + # Extract gradients from torch tensors + torch_tensor1_grad_matmul = torch_tensor1_matmul.grad + torch_tensor2_grad_matmul = torch_tensor2_matmul.grad + + # Assertions to compare the gradients + self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul)) + + def test_broadcasted_batched_matmul(self): + """ + Test autograd from broadcasted batched matrix multiplication: BxMxP = NxM @ BxMxP + """ + B = 3 # Batch size + + norch_tensor1_matmul = norch.Tensor([[1., 2], [3, -4], [5, 6], [7, 8]], requires_grad=True).to(self.device) + norch_tensor2_matmul = norch.Tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True).to(self.device) + + norch_result_matmul = norch_tensor1_matmul @ norch_tensor2_matmul + norch_result_matmul_sum = norch_result_matmul.sum() # Sum over all elements + norch_result_matmul_sum.backward() + + # Convert gradients to torch tensors + norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad).to(self.device) + norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) + + # Repeat the same process with torch tensors + torch_tensor1_matmul = torch.tensor([[1., 2], [3, -4], [5, 6], [7, 8]], requires_grad=True).to(self.device) + torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True).to(self.device) + torch_result_matmul = torch.matmul(torch_tensor1_matmul, torch_tensor2_matmul) + torch_result_matmul_sum = torch_result_matmul.sum() + torch_result_matmul_sum.backward() + + # Extract gradients from torch tensors + torch_tensor1_grad_matmul = torch_tensor1_matmul.grad + torch_tensor2_grad_matmul = torch_tensor2_matmul.grad + + # Assertions to compare the gradients + self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul)) + + def test_elementwise_mul_scalar(self): """ Test autograd from elementwise multiplication with scalar: scalar * tensor @@ -282,7 +583,169 @@ class TestTensorAutograd(unittest.TestCase): torch_expected_cos_tensor_grad = torch_cos_tensor.grad self.assertTrue(utils.compare_torch(torch_result_cos_tensor_grad, torch_expected_cos_tensor_grad)) - + + def test_sigmoid(self): + """ + Test autograd from sigmoid + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_sigmoid = norch.sigmoid(norch_tensor) + norch_result = norch_sigmoid.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + torch_sigmoid = torch.sigmoid(torch_tensor) + torch_result = torch_sigmoid.sum() + + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + def test_mse_loss_autograd(self): + """ + Test the MSELoss with autograd functionality + """ + loss_fn_norch = norch.nn.MSELoss() + loss_fn_torch = torch.nn.MSELoss() + + predictions_norch = norch.Tensor([1.1, 2, 3, 4], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([4, 3, 2.1, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([1.1, 2, 3, 4], requires_grad=True).to(self.device) + labels_torch = torch.tensor([4, 3, 2.1, 1]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + def test_cross_entropy_loss_autograd(self): + """ + Test the CrossEntropyLoss with autograd functionality + """ + loss_fn_norch = norch.nn.CrossEntropyLoss() + loss_fn_torch = torch.nn.CrossEntropyLoss() + + # Test case 1: Single class, single sample + predictions_norch = norch.Tensor([2.0, 1.0, 0.1], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([2.0, 1.0, 0.1], requires_grad=True).to(self.device) + labels_torch = torch.tensor(0).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + # Test case 2: Multiple classes, multiple samples + predictions_norch = norch.Tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([2, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], requires_grad=True).to(self.device) + labels_torch = torch.tensor([2, 1]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + # Test case 3: Edge case - all predictions are zero + predictions_norch = norch.Tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([1, 2]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], requires_grad=True).to(self.device) + labels_torch = torch.tensor([1, 2]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + # Test case 4: Batched class probabilities instead of class index + predictions_norch = norch.Tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], requires_grad=True).to(self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + + # implement grad pure softmax --> 0 + # def test_softmax(self): + # """ + # Test autograd from softmax + # """ + # norch_tensor = norch.Tensor([[[-5, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + # norch_softmax = norch.softmax(norch_tensor, dim=1) + # norch_result = norch_softmax.sum() + + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + # torch_tensor = torch.tensor([[[-5, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + # torch_softmax = torch.softmax(torch_tensor, dim=1) + # torch_result = torch_softmax.sum() + + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + # norch_tensor = norch.Tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # norch_softmax = norch.softmax(norch_tensor, dim=2) + # norch_result = norch_softmax.sum() + + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + # torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + + # torch_softmax = torch.softmax(torch_tensor, dim=2) + # torch_result = torch_softmax.sum() + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + def test_reshape(self): """ @@ -361,6 +824,115 @@ class TestTensorAutograd(unittest.TestCase): self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul1, torch_tensor_grad_reshape_matmul1)) self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul2, torch_tensor_grad_reshape_matmul2)) + def test_unsqueeze(self): + """ + Test autograd from unsqueezing a tensor: tensor.unsqueeze(dim) + """ + + # Unsqueeze at dim=0 + norch_tensor_unsqueeze = norch.Tensor([[1., 2], [3, 4]], requires_grad=True).to(self.device) + norch_result_unsqueeze_0 = norch_tensor_unsqueeze.unsqueeze(0).sum() + norch_result_unsqueeze_0.backward() + norch_tensor_grad_unsqueeze_0 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) + + torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True).to(self.device) + torch_result_unsqueeze_0 = torch_tensor_unsqueeze.unsqueeze(0).sum() + torch_result_unsqueeze_0.backward() + torch_tensor_grad_unsqueeze_0 = torch_tensor_unsqueeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_0, torch_tensor_grad_unsqueeze_0)) + + # Unsqueeze at dim=1 + norch_tensor_unsqueeze = norch.Tensor([[1., 2.], [3, 4]], requires_grad=True).to(self.device) + norch_result_unsqueeze_1 = norch_tensor_unsqueeze.unsqueeze(1).sum() + norch_result_unsqueeze_1.backward() + norch_tensor_grad_unsqueeze_1 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) + + torch_tensor_unsqueeze = torch.tensor([[1., 2.], [3, 4]], requires_grad=True).to(self.device) + torch_result_unsqueeze_1 = torch_tensor_unsqueeze.unsqueeze(1).sum() + torch_result_unsqueeze_1.backward() + torch_tensor_grad_unsqueeze_1 = torch_tensor_unsqueeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_1, torch_tensor_grad_unsqueeze_1)) + + # Unsqueeze at dim=2 + norch_tensor_unsqueeze = norch.Tensor([[1., 2], [3, 4]], requires_grad=True).to(self.device) + norch_result_unsqueeze_2 = norch_tensor_unsqueeze.unsqueeze(2).sum() + norch_result_unsqueeze_2.backward() + norch_tensor_grad_unsqueeze_2 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) + + torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True).to(self.device) + torch_result_unsqueeze_2 = torch_tensor_unsqueeze.unsqueeze(2).sum() + torch_result_unsqueeze_2.backward() + torch_tensor_grad_unsqueeze_2 = torch_tensor_unsqueeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_2, torch_tensor_grad_unsqueeze_2)) + + def test_unsqueeze_then_matmul(self): + """ + Test autograd from unsqueezing a tensor then performing matrix multiplication: matmul(tensor1.unsqueeze(dim), tensor2) + """ + norch_tensor1 = norch.Tensor([[1, 2], [3, 4]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[1, 2], [3, 4]], requires_grad=True).to(self.device) + + # Unsqueeze at dim=0 then matmul + norch_result_unsqueeze_matmul = (norch_tensor1 @ norch_tensor2.unsqueeze(0)).sum() + norch_result_unsqueeze_matmul.backward() + norch_tensor_grad_unsqueeze_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor_grad_unsqueeze_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True).to(self.device) + torch_tensor2 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True).to(self.device) + + torch_result_unsqueeze_matmul = (torch_tensor1 @ torch_tensor2.unsqueeze(0)).sum() + torch_result_unsqueeze_matmul.backward() + torch_tensor_grad_unsqueeze_matmul1 = torch_tensor1.grad + torch_tensor_grad_unsqueeze_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_matmul1, torch_tensor_grad_unsqueeze_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_matmul2, torch_tensor_grad_unsqueeze_matmul2)) + + def test_squeeze(self): + """ + Test autograd from squeezing a tensor: tensor.squeeze(dim) + """ + # Squeeze at dim=0 + norch_tensor_squeeze = norch.Tensor([[[1., 2], [3, 4]]], requires_grad=True).to(self.device) + norch_result_squeeze_0 = norch_tensor_squeeze.squeeze(0).sum() + norch_result_squeeze_0.backward() + norch_tensor_grad_squeeze_0 = utils.to_torch(norch_tensor_squeeze.grad).to(self.device) + + torch_tensor_squeeze = torch.tensor([[[1., 2], [3, 4]]], requires_grad=True).to(self.device) + torch_result_squeeze_0 = torch_tensor_squeeze.squeeze(0).sum() + torch_result_squeeze_0.backward() + torch_tensor_grad_squeeze_0 = torch_tensor_squeeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_squeeze_0, torch_tensor_grad_squeeze_0)) + + def test_squeeze_then_matmul(self): + """ + Test autograd from squeezing a tensor then performing matrix multiplication: matmul(tensor1.squeeze(dim), tensor2) + """ + norch_tensor1 = norch.Tensor([[[1., 2], [3, 4]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1., 2], [3, 4]]], requires_grad=True).to(self.device) + + # Squeeze at dim=0 then matmul + norch_result_squeeze_matmul = (norch_tensor1.squeeze(0) @ norch_tensor2).sum() + norch_result_squeeze_matmul.backward() + norch_tensor_grad_squeeze_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor_grad_squeeze_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True).to(self.device) + torch_tensor2 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True).to(self.device) + + torch_result_squeeze_matmul = (torch_tensor1.squeeze(0) @ torch_tensor2).sum() + torch_result_squeeze_matmul.backward() + torch_tensor_grad_squeeze_matmul1 = torch_tensor1.grad + torch_tensor_grad_squeeze_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_squeeze_matmul1, torch_tensor_grad_squeeze_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_squeeze_matmul2, torch_tensor_grad_squeeze_matmul2)) + def test_T_then_matmul(self): """ diff --git a/tests/test_nn.py b/tests/test_nn.py index e790fa9..cd623f3 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -21,13 +21,13 @@ class TestNNModuleLoss(unittest.TestCase): loss_fn_torch = torch.nn.MSELoss() # Test case 1: Predictions and labels are equal - predictions_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device) - labels_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device) + predictions_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device) + labels_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device) loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device) - labels_torch = torch.tensor([1.1, 2, 3, 4]).to(self.device) + predictions_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device) + labels_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) @@ -44,6 +44,75 @@ class TestNNModuleLoss(unittest.TestCase): self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + def test_cross_entropy_loss(self): + """ + Test the CrossEntropyLoss + """ + loss_fn_norch = norch.nn.CrossEntropyLoss() + loss_fn_torch = torch.nn.CrossEntropyLoss() + + # Test case 1: Single class, single sample + predictions_norch = norch.Tensor([2.0, 1.0, 0.1]).to(self.device) + labels_norch = norch.Tensor([0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([2.0, 1.0, 0.1]).to(self.device) + labels_torch = torch.tensor(0).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 2: Multiple classes, multiple samples + predictions_norch = norch.Tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device) + labels_norch = norch.Tensor([2, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device) + labels_torch = torch.tensor([2, 1]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 3: Edge case - all predictions are zero + predictions_norch = norch.Tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device) + labels_norch = norch.Tensor([1, 2]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device) + labels_torch = torch.tensor([1, 2]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Class probabilities instead of class index + predictions_norch = norch.Tensor([0.5, 0.2, 0.1]).to(self.device) + labels_norch = norch.Tensor([1., 0, 0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([0.5, 0.2, 0.1]).to(self.device) + labels_torch = torch.tensor([1., 0, 0]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Batched class probabilities instead of class index + predictions_norch = norch.Tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device) + labels_norch = norch.Tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + class TestNNModuleActivationFn(unittest.TestCase): @@ -60,11 +129,11 @@ class TestNNModuleActivationFn(unittest.TestCase): sigmoid_fn_torch = torch.nn.Sigmoid() # Test case 1: Positive input - x = norch.Tensor([1, 2, 3]).to(self.device) + x = norch.Tensor([[1, 2, 3]]).to(self.device) sigmoid_norch = sigmoid_fn_norch.forward(x) sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) - x = torch.tensor([1, 2, 3]).to(self.device) + x = torch.tensor([[1, 2, 3]]).to(self.device) sigmoid_torch_expected = sigmoid_fn_torch.forward(x) self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) @@ -87,4 +156,39 @@ class TestNNModuleActivationFn(unittest.TestCase): x = torch.tensor([0, 0, 0]).to(self.device) sigmoid_torch_expected = sigmoid_fn_torch.forward(x) - self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) \ No newline at end of file + self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) + + def test_softmax_activation(self): + """ + Test Softmax activation function + """ + + # Test different axes + axes = [0, 1, 2, -1] + + # Define the input tensors for different test cases + test_cases = [ + (norch.Tensor([[[1., 2, 3], [4, 5, 6]]]), torch.tensor([[[1., 2, 3], [4, 5, 6]]])), + (norch.Tensor([[[1., -1, 0], [2, -2, 0]]]), torch.tensor([[[1., -1, 0], [2, -2, 0]]])), + (norch.Tensor([[[0., 0, 0], [0, 0, 0]]]), torch.tensor([[[0., 0, 0], [0, 0, 0]]])) + ] + + for dim in axes: + softmax_fn_norch = norch.nn.Softmax(dim=dim) + softmax_fn_torch = torch.nn.Softmax(dim=dim) + + for norch_input, torch_input in test_cases: + # Move tensors to the correct device + norch_input = norch_input.to(self.device) + torch_input = torch_input.to(self.device) + + # Forward pass using norch + softmax_norch = softmax_fn_norch.forward(norch_input) + softmax_torch_result = utils.to_torch(softmax_norch).to(self.device) + + # Forward pass using torch + softmax_torch_expected = softmax_fn_torch.forward(torch_input) + + # Compare the results + self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected)) + diff --git a/tests/test_operations.py b/tests/test_operations.py index 449c531..868f982 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -37,7 +37,7 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) - def test_broadcasting_addition(self): + def test_addition_broadcasted(self): """ Test addition of two tensors with broadcasting: tensor1 + tensor2 """ @@ -52,6 +52,29 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + norch_tensor1 = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([[10, 10], [5, 6]]).to(self.device) # Shape (3) + norch_result = norch_tensor1 + norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([[[10, 10], [5, 6]]]).to(self.device) # Shape (3) + torch_expected = torch_tensor1 + torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # reversed order broadcasting + norch_tensor1 = norch.Tensor([[0, 2]]).to(self.device) + norch_tensor2 = norch.Tensor([[3, 4], [5, -1]]).to(self.device) + norch_result = norch_tensor1 + norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[0, 2]]).to(self.device) + torch_tensor2 = torch.tensor([[3, 4], [5, -1]]).to(self.device) + torch_expected = torch_tensor1 + torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + norch_result = norch_tensor2 + norch_tensor1 torch_expected = torch_tensor2 + torch_tensor1 @@ -89,6 +112,14 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + # reversed order broadcasting + norch_result = norch_tensor2 - norch_tensor1 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_expected = torch_tensor2 - torch_tensor1 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_division_by_scalar(self): """ @@ -176,6 +207,94 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_unsqueeze(self): + """ + Test unsqueeze operation on a tensor + """ + norch_tensor = norch.Tensor([[1, 2], [3, 4]]).to(self.device) + + # Unsqueeze at dim=0 + norch_unsqueeze_0 = norch_tensor.unsqueeze(0) + torch_unsqueeze_0 = utils.to_torch(norch_unsqueeze_0).to(self.device) + torch_tensor = torch.tensor([[1, 2], [3, 4]]).to(self.device) + torch_expected_0 = torch_tensor.unsqueeze(0) + self.assertTrue(utils.compare_torch(torch_unsqueeze_0, torch_expected_0)) + + # Unsqueeze at dim=1 + norch_unsqueeze_1 = norch_tensor.unsqueeze(1) + torch_unsqueeze_1 = utils.to_torch(norch_unsqueeze_1).to(self.device) + torch_expected_1 = torch_tensor.unsqueeze(1) + self.assertTrue(utils.compare_torch(torch_unsqueeze_1, torch_expected_1)) + + # Unsqueeze at dim=2 + norch_unsqueeze_2 = norch_tensor.unsqueeze(2) + torch_unsqueeze_2 = utils.to_torch(norch_unsqueeze_2).to(self.device) + torch_expected_2 = torch_tensor.unsqueeze(2) + self.assertTrue(utils.compare_torch(torch_unsqueeze_2, torch_expected_2)) + + # Unsqueeze at dim=-1 + norch_unsqueeze_neg_1 = norch_tensor.unsqueeze(-1) + torch_unsqueeze_neg_1 = utils.to_torch(norch_unsqueeze_neg_1).to(self.device) + torch_expected_neg_1 = torch_tensor.unsqueeze(-1) + self.assertTrue(utils.compare_torch(torch_unsqueeze_neg_1, torch_expected_neg_1)) + + # Unsqueeze at dim=-2 + norch_unsqueeze_neg_2 = norch_tensor.unsqueeze(-2) + torch_unsqueeze_neg_2 = utils.to_torch(norch_unsqueeze_neg_2).to(self.device) + torch_expected_neg_2 = torch_tensor.unsqueeze(-2) + self.assertTrue(utils.compare_torch(torch_unsqueeze_neg_2, torch_expected_neg_2)) + + def test_squeeze(self): + """ + Test squeeze operation on a tensor + """ + # Create a tensor with some dimensions of size 1 + norch_tensor = norch.Tensor([[[1, 2], [3, 4]]]).to(self.device) # shape [1, 2, 2] + + # Squeeze at dim=0 + norch_squeeze_0 = norch_tensor.squeeze(0) + torch_squeeze_0 = utils.to_torch(norch_squeeze_0).to(self.device) + torch_tensor = torch.tensor([[[1, 2], [3, 4]]]).to(self.device) + torch_expected_0 = torch_tensor.squeeze(0) + self.assertTrue(utils.compare_torch(torch_squeeze_0, torch_expected_0)) + + # Squeeze at dim=2 (should raise an error because size is not 1) + with self.assertRaises(ValueError): + norch_tensor.squeeze(2) + + # Create a tensor with a dimension of size 1 in the middle + norch_tensor_middle_1 = norch.Tensor([[[1, 2]], [[3, 4]]]).to(self.device) # shape [2, 1, 2] + + # Squeeze at dim=1 + norch_squeeze_1 = norch_tensor_middle_1.squeeze(1) + torch_squeeze_1 = utils.to_torch(norch_squeeze_1).to(self.device) + torch_tensor_middle_1 = torch.tensor([[[1, 2]], [[3, 4]]]).to(self.device) + torch_expected_1 = torch_tensor_middle_1.squeeze(1) + self.assertTrue(utils.compare_torch(torch_squeeze_1, torch_expected_1)) + + # Squeeze at dim=-2 (same as dim=1 in this case) + norch_squeeze_neg_2 = norch_tensor_middle_1.squeeze(-2) + torch_squeeze_neg_2 = utils.to_torch(norch_squeeze_neg_2).to(self.device) + torch_expected_neg_2 = torch_tensor_middle_1.squeeze(-2) + self.assertTrue(utils.compare_torch(torch_squeeze_neg_2, torch_expected_neg_2)) + + # Squeeze all dimensions of size 1 (None) + norch_tensor_all_1 = norch.Tensor([[[[1, 2], [3, 4]]]]).to(self.device) # shape [1, 1, 2, 2] + norch_squeeze_all = norch_tensor_all_1.squeeze() + torch_squeeze_all = utils.to_torch(norch_squeeze_all).to(self.device) + torch_tensor_all_1 = torch.tensor([[[[1, 2], [3, 4]]]]).to(self.device) + torch_expected_all = torch_tensor_all_1.squeeze() + self.assertTrue(utils.compare_torch(torch_squeeze_all, torch_expected_all)) + + # Squeeze no dimensions (no dimensions of size 1) + norch_tensor_no_1 = norch.Tensor([[1, 2], [3, 4]]).to(self.device) # shape [2, 2] + norch_squeeze_none = norch_tensor_no_1.squeeze() + torch_squeeze_none = utils.to_torch(norch_squeeze_none).to(self.device) + torch_tensor_no_1 = torch.tensor([[1, 2], [3, 4]]).to(self.device) + torch_expected_none = torch_tensor_no_1.squeeze() + self.assertTrue(utils.compare_torch(torch_squeeze_none, torch_expected_none)) + + def test_transpose(self): """ Test transposition of a tensor: tensor.transpose(dim1, dim2) @@ -229,6 +348,134 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum(axis=-2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor, dim=-2) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_sum_axis_keepdim(self): + """ + Test summation of a tensor along a specific axis with keepdim=True + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum(axis=1, keepdim=True) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor, dim=1, keepdim=True) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_max(self): + """ + Test max of a tensor: tensor.max() + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.max(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_max_axis(self): + """ + Test max of a tensor along a specific axis without keeping the dimensions + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max(axis=1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.max(torch_tensor, dim=1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max(axis=-1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.max(torch_tensor, dim=-1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_max_axis_keepdim(self): + """ + Test max of a tensor along a specific axis with keepdim=True + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max(axis=1, keepdim=True) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.max(torch_tensor, dim=1, keepdim=True) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_min(self): + """ + Test min of a tensor: tensor.min() + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.min(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_min_axis(self): + """ + Test min of a tensor along a specific axis without keeping the dimensions + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min(axis=1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.min(torch_tensor, dim=1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min(axis=-1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.min(torch_tensor, dim=-1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_min_axis_keepdim(self): + """ + Test min of a tensor along a specific axis with keepdim=True + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min(axis=1, keepdim=True) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.min(torch_tensor, dim=1, keepdim=True) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_transpose_T(self): """ Test transposition of a tensor: tensor.T @@ -242,6 +489,26 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_matmul(self): + """ + Test matrix multiplication: MxP = NxM @ MxP + """ + # Creating batched tensors for Norch + norch_tensor1 = norch.Tensor([[1, 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + norch_tensor2 = norch.Tensor([[2, 3, 1, 0, 4], [5, -1, 2, 3, 0]]).to(self.device) + + norch_result = norch_tensor1 @ norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + # Converting to PyTorch tensors for comparison + torch_tensor1 = torch.tensor([[1, 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + torch_tensor2 = torch.tensor([[2, 3, 1, 0, 4], [5, -1, 2, 3, 0]]).to(self.device) + + torch_expected = torch.matmul(torch_tensor1, torch_tensor2) + + # Comparing results + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_reshape_then_matmul(self): """ Test reshaping a tensor followed by matrix multiplication: (tensor.reshape(shape) @ other_tensor) @@ -258,6 +525,52 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_batched_matmul(self): + """ + Test batched matrix multiplication: BxMxP = BxNxM @ BxMxP + """ + B = 3 # Batch size + + # Creating batched tensors for Norch + norch_tensor1 = norch.Tensor([[[1, 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)]).to(self.device) + norch_tensor2 = norch.Tensor([[[2, 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + norch_result = norch_tensor1 @ norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + # Converting to PyTorch tensors for comparison + torch_tensor1 = torch.tensor([[[1, 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)]).to(self.device) + torch_tensor2 = torch.tensor([[[2, 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + torch_expected = torch.matmul(torch_tensor1, torch_tensor2) + + # Comparing results + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_broadcasted_batched_matmul(self): + """ + Test broadcasted batched matrix multiplication: BxMxP = NxM @ BxMxP + """ + B = 3 # Batch size + + # Creating batched tensors for Norch + norch_tensor1 = norch.Tensor([[1, 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + norch_tensor2 = norch.Tensor([[[2, 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + norch_result = norch_tensor1 @ norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + # Converting to PyTorch tensors for comparison + torch_tensor1 = torch.tensor([[1, 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + torch_tensor2 = torch.tensor([[[2, 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + torch_expected = torch.matmul(torch_tensor1, torch_tensor2) + + # Comparing results + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_transpose_then_matmul(self): """ @@ -346,6 +659,47 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_equal(self): + """ + Test equal two tensors: tensor1.equal(tensor2) + """ + norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 1], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + norch_result = norch_tensor1.equal(norch_tensor2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 1], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + torch_expected = (torch_tensor1 == torch_tensor2).float() + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_broadcasted_equal(self): + """ + Test broadcasted equal two tensors: tensor1.equal(tensor2) + """ + norch_tensor1 = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[10, 10]], [[5, 6]]]).to(self.device) + norch_result = norch_tensor1.equal(norch_tensor2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[10, 10]], [[5, 6]]]).to(self.device) + torch_expected = (torch_tensor1 == torch_tensor2).float() + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + norch_tensor1 = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[10.0,], [-4.0,]],[[6.0,], [8.0,]]]).to(self.device) + norch_result = norch_tensor1.equal(norch_tensor2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[10.0,], [-4.0,]],[[6.0,], [8.0,]]]).to(self.device) + torch_expected = (torch_tensor1 == torch_tensor2).float() + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_zeros_like(self): """