From c808a49a3010a8eae1905e8ac6e03951ff02f922 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 18 May 2024 17:59:14 -0300 Subject: [PATCH] dataset, dataloader and mnist --- examples/train.ipynb | 19589 +++++++++++++++- norch/__init__.py | 1 + norch/__pycache__/__init__.cpython-38.pyc | Bin 342 -> 362 bytes norch/__pycache__/tensor.cpython-38.pyc | Bin 15627 -> 16110 bytes .../__pycache__/functions.cpython-38.pyc | Bin 8173 -> 8173 bytes norch/datasets/__init__.py | 1 + norch/datasets/mnist.py | 90 + norch/tensor.py | 30 +- norch/utils/__init__.py | 3 +- norch/utils/__pycache__/utils.cpython-38.pyc | Bin 759 -> 3901 bytes norch/utils/data/__init__.py | 4 + norch/utils/data/batch.py | 13 + norch/utils/data/dataloader.py | 20 + norch/utils/data/dataset.py | 74 + norch/utils/data/example.py | 65 + norch/utils/utils.py | 108 +- test.py | 75 + 17 files changed, 19257 insertions(+), 816 deletions(-) create mode 100644 norch/datasets/__init__.py create mode 100644 norch/datasets/mnist.py create mode 100644 norch/utils/data/__init__.py create mode 100644 norch/utils/data/batch.py create mode 100644 norch/utils/data/dataloader.py create mode 100644 norch/utils/data/dataset.py create mode 100644 norch/utils/data/example.py create mode 100644 test.py diff --git a/examples/train.ipynb b/examples/train.ipynb index 67d7c2d..b29a29d 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -26,7 +26,64 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Download train-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to .data/mnist/train-images-idx3-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download train-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to .data/mnist/train-labels-idx1-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download t10k-labels-idx1-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to .data/mnist/t10k-labels-idx1-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n", + "Download t10k-images-idx3-ubyte.gz from http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to .data/mnist/t10k-images-idx3-ubyte.gz\n", + "Downloading... 100% | [==================================================] | Done !\n", + ".data/mnist/mnist-data-py\n", + "Extracting... | NOTE: gzip files are not extracted, and moved to .data/mnist/mnist-data-py | Done !\n" + ] + }, + { + "ename": "BadGzipFile", + "evalue": "Not a gzipped file (b' 6\u001b[0m train_data, test_data \u001b[38;5;241m=\u001b[39m \u001b[43mnorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdatasets\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mMNIST\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplits\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mreshape\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:75\u001b[0m, in \u001b[0;36mMNIST.splits\u001b[0;34m(cls, root, train_data, train_label, test_data, test_label, **kwargs)\u001b[0m\n\u001b[1;32m 73\u001b[0m test_data \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_data)\n\u001b[1;32m 74\u001b[0m test_label \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(path, test_label)\n\u001b[0;32m---> 75\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMNIST\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_label\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m, MNIST(test_data, test_label, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:39\u001b[0m, in \u001b[0;36mMNIST.__init__\u001b[0;34m(self, path_data, path_label, transform)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, path_data, path_label, transform\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m---> 39\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_load_mnist\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheader_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m16\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mreshape((\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m28\u001b[39m, \u001b[38;5;241m28\u001b[39m))\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m transform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 41\u001b[0m data \u001b[38;5;241m=\u001b[39m transform(data)\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/datasets/mnist.py:47\u001b[0m, in \u001b[0;36mMNIST._load_mnist\u001b[0;34m(self, path, header_size)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_load_mnist\u001b[39m(\u001b[38;5;28mself\u001b[39m, path, header_size):\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m gzip\u001b[38;5;241m.\u001b[39mopen(path, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m---> 47\u001b[0m \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43mheader_size\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Skip the header\u001b[39;00m\n\u001b[1;32m 48\u001b[0m data \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m 50\u001b[0m data_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(data)\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:384\u001b[0m, in \u001b[0;36mGzipFile.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 382\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;241m==\u001b[39m READ:\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_not_closed()\n\u001b[0;32m--> 384\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_buffer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseek\u001b[49m\u001b[43m(\u001b[49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwhence\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moffset\n", + "File \u001b[0;32m/usr/lib/python3.8/_compression.py:143\u001b[0m, in \u001b[0;36mDecompressReader.seek\u001b[0;34m(self, offset, whence)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;66;03m# Read and discard data until we reach the desired position.\u001b[39;00m\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m offset \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 143\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDEFAULT_BUFFER_SIZE\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m 145\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:479\u001b[0m, in \u001b[0;36m_GzipReader.read\u001b[0;34m(self, size)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_new_member:\n\u001b[1;32m 476\u001b[0m \u001b[38;5;66;03m# If the _new_member flag is set, we have to\u001b[39;00m\n\u001b[1;32m 477\u001b[0m \u001b[38;5;66;03m# jump to the next member, if there is one.\u001b[39;00m\n\u001b[1;32m 478\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_read()\n\u001b[0;32m--> 479\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_gzip_header\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 480\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pos\n\u001b[1;32m 481\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[0;32m/usr/lib/python3.8/gzip.py:427\u001b[0m, in \u001b[0;36m_GzipReader._read_gzip_header\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 424\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 426\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m magic \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\037\u001b[39;00m\u001b[38;5;130;01m\\213\u001b[39;00m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 427\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m BadGzipFile(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mNot a gzipped file (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m magic)\n\u001b[1;32m 429\u001b[0m (method, flag,\n\u001b[1;32m 430\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_last_mtime) \u001b[38;5;241m=\u001b[39m struct\u001b[38;5;241m.\u001b[39munpack(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m" ] diff --git a/norch/__init__.py b/norch/__init__.py index ca7be0b..3eb0b7f 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -2,6 +2,7 @@ from norch.tensor import Tensor from .nn import * from .optim import * from .utils import * +from .datasets import * __version__ = "0.0.1" __author__ = 'Lucas de Lima Nogueira' diff --git a/norch/__pycache__/__init__.cpython-38.pyc b/norch/__pycache__/__init__.cpython-38.pyc index d1722766eae308a42b9c2c3e679b5faf1b625dbc..8202d1d36b6b390a36dae101fa23ac1e0b542235 100644 GIT binary patch delta 96 zcmcb{^oof$l$V!_0SJ@@JkzdEI5&Oi?!9sjM77rr_1TwKOvM>SwF~SrS delta 77 zcmaFGbd8BOl$V!_0SLHX+NJHE$ScceGErNDnn3TMI3-C#k|MfIm~M8)n_aJG zHfP3h5@vCT14$bwwC8~b(qfATWGRB8eLx5ykPuQ;p*|vYsf4&afPgj<6a=F1|7UD( zyiH0|ujZRG|2gNso$vo=eCx_ztI?+;5lw)!IVhY>R25HnDFP~U+r&6CBEh9huAEv+iZ?&?yq)TyWVTo=sv&)qXPnCk>XA-KYD zwZIjfmqBrlwb4K`JscH|OYWiJ73K}$l#r2+|4W&2$G7aFB4l}i9MAODwW3qB95-j> z&AKvPwsUS>Dq3zmFka5NZp5_ea^9|7rUBT&ekPw_NoAI-Ww(_~avz8UQ6o{JicwJ$ zlO#!Eq(!W~r4Z6XJ7Kmimy3>DSItYVX&H4jSE<0A_OkcYab-I!IKYa5PjAS;h;BvL zhJdzc8^RDmJ3=?We)ikI8wUnpl#SVz>FDL+gsFpO$EI{0u3C?<0fFo2!BG$ac}3R% zEJ!rN-U&WRPO>ebw@0|L_!X1d53h|?x{~ay7AAR6)yHmXeKTlAflGqRS%uscNNi+=B8s4V;g7su!4-szEn6D^x>X&}>_+R+rOuVx7DXV-r4M&MRQHv&r#asS zsX61E&sEI9k+Pl7m7P%-EbYj2aIV#rygdcAqttK*%|JFWVcF9bCd1Jy)HbH_F5QE6{vKX?b_WdTClPp#_Tq>qDzNCS3le7- zKoC_aN`j(HWDtAEAbZF_kT&lp$Z_Gb{X2>cw*;K)A*U3Xln>vi^8`$<(tx$kjT zM~M8bHqdcLjB!Vwz^PG$LH3)~pCj+HeaU07Ml>m$8fM>0_K*Sg<7D@trq0Uw*9dvJ zvrQKIgcm(O@L&>+Cy^H9Cy&sPvc^r>{77*?i#mZL$4}hM>#aKTJvY+uH3VffC z0a{SkfgRw%9+qD-TzkE1zZlNpnsW#SYwumJU~E~ouXpUec|F9*vC5e`*9hD+KCImD zn{M{+-fq&*l6`%{oEqFwv`k9~jWqHMS}0q}Aez=s)h7E&|1X1&@(kLz`J-mAf{QVO zw7i@!ZjjaTYj%p6rX%Bxj~`j{>l%ycNnSKOVQ~lgY>w)aQ{@NbiZd{NiZ44hRc??E zof&26{-MPBre4TQ)q*^z<6Vg4$a#u>l3 zzB>>kZ?PW?947o3HC5nKLVvr~wXQ>g0vgJ!pJ_@Qk9E0hvK{AgZvH%+|76a+Cp4_k zY-FX3#}_@u*R2RHqkMp?=%ZghY&a5a(I7-$zy9mm-Asm%6z^4oo3zjR+4aApzr5Nogi87QC@-j8q#qUJA6X7w0eFz5;jv(M}dK`g& z#yp3k34}?63kVbe8#TRza0TJ>2wyawzB$&&0?wy{inL3Ltd5$u33ki^B9ArRsu41#8})~;pAvQ{~} zN=T4#XadZH4$N@qOzD$7bUG7yfsQ(T=tCJ=hR(DveP~}anetLPZHKlzb$Xem{r|I) z9BJfUV$FW%-}9gSuirVl{?q1bm;I;P+hqa%u7B;()$7Lh`~`7qhzLT7Xu`Tk|56gR zLEa(HIX20r(;``iC6>rl5{B8=g_5A)oCI@DKIi0fE{kke=XMCJahDw19u2I3>@7I~ zW%~(`d?g29n}*n460-8bmhh#{O{pX}g=~9CdO&c1;(Vzj!UDgBE;~2|z)27T+96d* zLJ5K)+0L}n3T4CDh|zV=X>( z+~8glmW8VO*FAWy`j758dRwJbf)JkklL8>ZRQ(lv4yyD7!Pvsh|} zhxkbtegV?vaFKGqi;$s;oG~+b(^B&(z2eMeOsnG3p9eEq#ibSsFw!WSaj!}fU}27Z z)BV&v!ywTE2nP|+6%8SbBZLup0A|^M=V!+TK}rUUylyJF%(||CWz(SaZd|n=VF&?j zRGcZJaEW^G9;re50d9zNn2mYo$Z?kO-aNvs1uyH=_^;;Zy=bG+8l7Yx&WG6BvX7hv zdjsqvd0^QE>m<02!DW6Iz!I{=a_q8g(_snQ)tMc#DZ-s-Iia;U!djHFBrRG})|GWD z9!p?kJiHG5m4Rg9jtB&nt_r{{XGv1z6500c2q3lDPC!Be`eF3L*a2e)i~$(o$_D7W zRIzM2NEU)1fV18$Z>}HMOu;oq0`o!lYPnJK0eem6Ua!9HnkBMJ7Bysy(;4Z?uuOtdVp zaeohSu}A%{xq9K2_OZ!Oh<)alM|sp<%TP4^LMEj*5hL#4&Z&%jb2=>JaLpe4%L3dr&`ydQsK1xZ6&|lcuVc(eqYP0@|n6lkN+! z1uR{&sfb3C60R|AR|R?mRy4YR+T3HV5x)2dbP1yvGrgc(FWb-BKXl z4ZBtWRMdA;9QTVAnjqgVe5rR4Xb~`eieCh9Q%T+@$qGZ zQwTiSGbmxk6}O_87FCoVv#)lYbUlOfm`~WNC{!ni>kqm21r}P{~)S1_H%~Vz>o9Xq9CNQx35U{^?DHN1y zr9jKdC*_UabrP+oMO9p4Cxy~NEwd%;7F|JTm7NC_^>dCVMpHN!1GwR|8>8U_-&AjQ zJDJfhk-OO=`=`p^>ObP}Ka0xe5Y7Uy&wIL9Am(?SgPWs-jYLP-W3ji43ofVB zoJx-ub2Vz&UKf}+@?>DQEcan33Ecf&*hU-0oT9wNUL3jBejXIHgH5aLCa(VxJ21NF zTfzq^0zW(O>Aap+ln>e0M;A$bt^n^q<)iYg(TE6b43CXYHATxatTQYbSy8Rjnyx7u zs)gRFZ{h`Zek?S}GxHzknufIE((O#dpm<&*m{*$&dD+I4pO$|)7AHi-3a;=9?w(jd z+m%u<;0?ly*WA>W%|K@uSH*Y2xY!UB?=UYbnp9lujf0q4Uc1V18!wioVj?+=ca zUkK=sUPg!^*i`!GxQ9f+$#8jW;tKKIMkh>nYmaob%!NBjH?WYwHM-a8;%6NbJ?z!V-`V8>qP(8yKHQ2Z&07IY^I~kG z6hpvbgxKeac5;T@O8ogLUTZmLq-*i~KX#M8$BlGDhW6K)B!a;l`{VSO1OiW-hh}kO4~P6%;9TOTSDpLzYs0r(+-PuE`?PVY&u2{f5~jF&f^mjRA~@S( zUzz%AHCgyp-oy8Qq;S%=oM~)lEm<)54kY`Vs{Isf>N{X;(s#@-8*;1J2ZKMrVwUv> zq6FH1<-y*eJ==6!!Po~6cK2e}s|P?*!Xi!DL&AP1`MefY^Clcb`p;Q-=2Mr2>x1lG zIdYn}Dp1$oQBvMhEm9K6sZZI~?74v>*sq5WLUq8G4Lu6*HwU{l+XuAioqL9V8Picz z;r}J}yZU~VrVyqP@Y9YyjDSzmMFjr2lSFA1VGSXNuz`T5I5iQ92p148A-sT4LU<8j xO90RG3W{GwxQcKM;bnk|r$CK@POVGy6&9U8;Ej8N{MW^voIgfhWo-W0e*<)eYLEZ` diff --git a/norch/autograd/__pycache__/functions.cpython-38.pyc b/norch/autograd/__pycache__/functions.cpython-38.pyc index d56a7979b8bb61f3a906fe1b7edf92f9d2705f87..0a7fe5b867d4774623f8e809fbb694066446922d 100644 GIT binary patch delta 19 ZcmaEB|JI%>l$V!_0SMwgZsd9_4*)!51?T_( delta 19 ZcmaEB|JI%>l$V!_0SM&YZRC0^4*)vV1*HH0 diff --git a/norch/datasets/__init__.py b/norch/datasets/__init__.py new file mode 100644 index 0000000..fe34b11 --- /dev/null +++ b/norch/datasets/__init__.py @@ -0,0 +1 @@ +from .mnist import * \ No newline at end of file diff --git a/norch/datasets/mnist.py b/norch/datasets/mnist.py new file mode 100644 index 0000000..ff11edf --- /dev/null +++ b/norch/datasets/mnist.py @@ -0,0 +1,90 @@ +import gzip +import os +import norch +from norch.utils.data import Dataset +import numpy as np + +class MNIST(Dataset): + """ + Loads training, validation, and test partitions of the mnist dataset + (http://yann.lecun.com/exdb/mnist/). If the data is not already contained in data_dir, it will + try to download it. + + This dataset contains 60000 training examples, and 10000 test examples of handwritten digits + in {0, ..., 9} and corresponding labels. Each handwritten image has an "original" dimension of + 28x28x1, and is stored row-wise as a string of 784x1 bytes. Pixel values are in range 0 to 255 + (inclusive). + + Args: + data_dir: String. Relative or absolute path of the dataset. + devel_size: Integer. Size of the development (validation) dataset partition. + + Returns: + X_train: float64 numpy array with shape [784, 60000-devel_size] with values in [0, 1]. + Y_train: uint8 numpy array with shape [60000-devel_size]. Labels. + X_devel: float64 numpy array with shape [784, devel_size] with values in [0, 1]. + Y_devel: uint8 numpy array with shape [devel_size]. Labels. + X_test: float64 numpy array with shape [784, 10000] with values in [0, 1]. + Y_test: uint8 numpy array with shape [10000]. Labels. + """ + + urls = ['https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz',] + name = 'mnist-data-py' + dirname = 'mnist' + + def __init__(self, path_data, path_label, transform=None, target_transform=None): + self.data = self._load_mnist(path_data, header_size=16).reshape((-1, 28, 28)) + self.labels = self._load_mnist(path_label, header_size=8) + self.transform = transform + self.target_transform = target_transform + + def _load_mnist(self, path, header_size): + with gzip.open(path, 'rb') as f: + data = np.frombuffer(f.read(), np.uint8, offset=header_size) + return np.asarray(data, dtype=np.uint8) + + + @classmethod + def splits(cls, root='.data', train_data='train-images-idx3-ubyte.gz', train_label='train-labels-idx1-ubyte.gz', + test_data='t10k-images-idx3-ubyte.gz', test_label='t10k-labels-idx1-ubyte.gz', **kwargs): + r""" + Loads training and test partitions of the [mnist dataset](https://www.cs.toronto.edu/~kriz/cifar.html). If + the data is not already contained in the ``root`` folder, it will download it. + + Args: + root (str): relative or absolute path of the dataset. + + Returns: + tuple(Dataset): training and testing datasets + """ + path = os.path.join(root, cls.dirname, cls.name) + if not os.path.isdir(path): + path = cls.download(root) + train_data = os.path.join(path, train_data) + train_label = os.path.join(path, train_label) + test_data = os.path.join(path, test_data) + test_label = os.path.join(path, test_label) + return MNIST(train_data, train_label, **kwargs), MNIST(test_data, test_label, **kwargs) + + def __getitem__(self, item): + data = self.data[item].tolist() + label = self.labels[item].tolist() + + if self.transform is not None: + data = self.transform(data) + + if self.target_transform is not None: + label = self.target_transform(label) + + + + return data, label + + def __setitem__(self, key, value): + self.data[key], self.labels[key] = value + + def __len__(self): + return len(self.data) diff --git a/norch/tensor.py b/norch/tensor.py index 807c290..8f3ae12 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -19,13 +19,18 @@ class Tensor: def __init__(self, data=None, device="cpu", requires_grad=False): if data != None: + if isinstance(data, (float, int)): + data = [data] + data, shape = self.flatten(data) - self.data_ctype = (ctypes.c_float * len(data))(*data) - self.shape_ctype = (ctypes.c_int * len(shape))(*shape) + + self.shape = shape.copy() + + self.data_ctype = (ctypes.c_float * len(data))(*data.copy()) + self.shape_ctype = (ctypes.c_int * len(shape))(*shape.copy()) self.ndim_ctype = ctypes.c_int(len(shape)) self.device_ctype = device.encode('utf-8') - self.shape = shape self.ndim = len(shape) self.device = device @@ -109,6 +114,25 @@ class Tensor: return result_data def reshape(self, new_shape): + # Calculate the total number of elements in the tensor + total_elements = self.numel + + # Check for the presence of -1 in new_shape + if new_shape.count(-1) > 1: + raise ValueError("Only one dimension can be inferred (set to -1).") + + inferred_dim = None + known_dims_product = 1 + for dim in new_shape: + if dim == -1: + inferred_dim = dim + else: + known_dims_product *= dim + + # Calculate the inferred dimension if -1 is present + if inferred_dim == -1: + inferred_dim_size = total_elements // known_dims_product + new_shape = [inferred_dim_size if dim == -1 else dim for dim in new_shape] new_shape_ctype = (ctypes.c_int * len(new_shape))(*new_shape) new_ndim_ctype = ctypes.c_int(len(new_shape)) diff --git a/norch/utils/__init__.py b/norch/utils/__init__.py index 90f60fd..c6adfd1 100644 --- a/norch/utils/__init__.py +++ b/norch/utils/__init__.py @@ -1 +1,2 @@ -from .utils import * \ No newline at end of file +from .utils import * +from .data import * \ No newline at end of file diff --git a/norch/utils/__pycache__/utils.cpython-38.pyc b/norch/utils/__pycache__/utils.cpython-38.pyc index 95d3b4582f73d44896121e8f25e8606d2085c37f..d78c80a01f3b3ae50ea607c841e71b1083517427 100644 GIT binary patch literal 3901 zcmbVP&5s*N6|d@U+wC6DWHQWdlHFC*h}}($GIkPn7wsqnSh62T;ANAnKxQM;<0^YP z)9!Yv%R7nF2BZlB4iVzE5<(gubK%5az@O1qF3dl`As6_)>h^pwaG@hrB|NLa|t5wGSPMzbAjn4P*rduck6Fg$wqUK$0`c~I6wcWK%?Q|VeSGpBp z3HurAR>g{_i0U)mt%-BO6?L>+u_`>VgjQWN#4@Bj!SAu=`2*Ntoj#|ke5mcZ8@#x>hFv7B3fv>rL|!8-UC9z_e9R$eWv=Cw&24iYtbVdYlq zdd_<TvFya2k?W-5You!( zh56P7eJkxx@twx|CePQ;GNC9`FL=Hb#@L9SiLY6p2(fAjg;Bn66oIlM8Rr$aIh4>^ zIkse(mY8)7v|PL?HJH=jj@94|{;H*D;NaIN`UJGaL$Va)AK*>DLZR7111MoVw`O)` zKeVuU_MFcgZD(9Kd)6Nw&MFz8x8tf?S!LgWOl8hFi#wt^tLkdz>_Hl%HSJPQ)LutO zgu7>n`g4Hv!P~Q1R@L>aw#UECW>ur3MjEu2k-WEObM|QKr|ez!8N17V#zY06Sqk0* z=I$iBag+oCupNYv^al!f4PyWEyFY4q2A-d)a2Uoxgwb?YszPF0>v6i-Zl^lv?<8MJ zHHebk7U0l+G?A$elQ?a^|IUr~Z@hQ2ElLv`O6rM~}Uk%eu zW0MjJ!~WG&tLCOZ4v_Nz#O;`F!tdT++K+{lf=hp64$E0-l z4*(-%Y?VRUJF+=lSp*&KwOXy_Qtm2g;+E$2Q0lz4Edv3<$emDu`@6>E3vyobs`irr%L~GJtf~n{_$(@Hu z80YR*kjgk10gJ&_O0G1W+$EL8Al4cG9MiSQw1LkNhE_^h0v&EvpIb8z8a$HLPF54k7>6%AO#K!5EU_}@;vAvd z++uBlW$zl#AaJDXvNt)l1I6h-zq&UG&Y|DR%hf#Ru`P1?j z7&#qU?63(p&V)3H%Aj`C+sKcX+&>VgbN=&}{5uE4#|Jk}|H1V9^3&_Hr#CjgC1!7d zZl2zpO>NZEH5B*pZJR#kNB+|rvvg`bolU(%PV&?h=xA1Q8~dME>8=^*+{FoI0-ie* z8nr<(OZoB%h<59TXv?bwTEIAF^K;MaN}<9VOVvj>gQ4u!=$?W&c2^gG0)?1;=(=R7 zHwxpjmCAm%Va7^{Q#_Nz(n!6@DchPR}H~EVFhVAk-I>#I%apK@20;0pqkj^zU zGNkiGkPH!y;B+U$;e|Gl#yVz-TU+J~EV`lx72LKymLU5m3dSiBJbb1-gWD4=%YO$F zyC>2!bNnbQC6tk92>dYwAHonP&vDE{5F-nm7niO70GBM>q9~?E2FlUX&K_ARS0Up6 zV-;qL&!YS|xP{;~xgy?gR@9GC`__. + + """ + @classmethod + def fromlist(cls, values, fields): + """ + Add an example from a list of data with respect to the fields. + + Args: + values: raw data + fields (tuple(string, Field)): fields to preprocess the data on + + Returns: + None + """ + example = cls() + for (value, field) in zip(values, fields): + assert len(field) == 2, f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + assert isinstance(field[1], Field), f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + name = field[0] + value = field[1].process(value) + setattr(example, name, value) + + return example diff --git a/norch/utils/utils.py b/norch/utils/utils.py index 1652e16..f8b9b30 100644 --- a/norch/utils/utils.py +++ b/norch/utils/utils.py @@ -1,4 +1,9 @@ import random +import requests +import tarfile +import zipfile +import shutil +import os def generate_random_list(shape): """ @@ -11,4 +16,105 @@ def generate_random_list(shape): if len(inner_shape) == 0: return [random.uniform(-1, 1) for _ in range(shape[0])] else: - return [generate_random_list(inner_shape) for _ in range(shape[0])] \ No newline at end of file + return [generate_random_list(inner_shape) for _ in range(shape[0])] + + +def download_from_url(url, save_path, chunk_size=128): + """Download a file from an URL. + + Original answer from https://stackoverflow.com/questions/9419162/download-returned-zip-file-from-url + + Args: + url (str): path to the URL. + save_path (str): path to the saving directory. + chunk_size (int): download chunk. + + Returns: + None + """ + response = requests.get(url, stream=True) + total = response.headers.get('content-length') + with open(save_path, 'wb') as f: + if total is None: + f.write(response.content) + else: + downloaded = 0 + total = int(total) + for data in response.iter_content(chunk_size=max(int(total / 1000), 1024 * 1024)): + downloaded += len(data) + f.write(data) + progress_bar(downloaded, total, "Downloading...") + + +def extract_to_dir(filename, dirpath='.'): + # Does not create folder twice with the same name + name, ext = os.path.splitext(filename) + # if os.path.basename(name) == os.path.basename(dirpath): + # dirpath = '.' + # Extract + print(dirpath) + print("Extracting...", end="") + if tarfile.is_tarfile(filename): + tarfile.open(filename, 'r').extractall(dirpath) + elif zipfile.is_zipfile(filename): + zipfile.ZipFile(filename, 'r').extractall(dirpath) + elif ext == '.gz': + if not os.path.exists(dirpath): + os.mkdir(dirpath) + shutil.move(filename, os.path.join(dirpath, os.path.basename(filename))) + print(f" | NOTE: gzip files are not extracted, and moved to {dirpath}", end="") + # Return the path where the file was extracted + print(" | Done !") + return os.path.abspath(dirpath) + +def progress_bar(current_index, max_index, prefix=None, suffix=None, start_time=None): + """Display a progress bar and duration. + + Args: + current_index (int): current state index (or epoch number). + max_index (int): maximal numbers of state. + prefix (str, optional): prefix of the progress bar. The default is None. + suffix (str, optional): suffix of the progress bar. The default is None. + start_time (float, optional): starting time of the progress bar. If not None, it will display the time + spent from the beginning to the current state. The default is None. + + Returns: + None. Display the progress bar in the console. + """ + # Add a prefix to the progress bar + prefix = "" if prefix is None else str(prefix) + " " + + # Get the percentage + percentage = current_index * 100 // max_index + loading = "[" + "=" * (percentage // 2) + " " * (50 - percentage // 2) + "]" + progress_display = "\r{0}{1:3d}% | {2}".format(prefix, percentage, loading) + + # Add a suffix to the progress bar + progress_display += "" if suffix is None else " | " + str(suffix) + + # Add a timer + if start_time is not None: + time_min, time_sec = get_time(start_time, time.time()) + time_display = " | Time: {0}m {1}s".format(time_min, time_sec) + progress_display += time_display + + # Print the progress bar + # TODO: return a string instead + print(progress_display, end="{}".format("" if current_index < max_index else " | Done !\n")) + +def get_time(start_time, end_time): + """Get ellapsed time in minutes and seconds. + + Args: + start_time (float): strarting time + end_time (float): ending time + + Returns: + elapsed_mins (float): elapsed time in minutes + elapsed_secs (float): elapsed time in seconds. + """ + elapsed_time = end_time - start_time + elapsed_mins = int(elapsed_time / 60) + elapsed_secs = int(elapsed_time - (elapsed_mins * 60)) + + return elapsed_mins, elapsed_secs diff --git a/test.py b/test.py new file mode 100644 index 0000000..dfe3aae --- /dev/null +++ b/test.py @@ -0,0 +1,75 @@ +import norch +from norch.utils.data.dataloader import Dataloader +import norch +import norch.nn as nn +import norch.optim as optim +import random +random.seed(1) + +to_tensor = lambda x: norch.Tensor(x) +reshape = lambda x: x.reshape([-1, 784]) +transform = lambda x: reshape(to_tensor(x)) +target_transform = lambda x: to_tensor(x) + +train_data, test_data = norch.datasets.MNIST.splits(transform=transform, target_transform=target_transform) +sample, _ = train_data[0] + +BATCH_SIZE = 100 + +train_loader = Dataloader(train_data, batch_size = BATCH_SIZE) + +class MyModel(nn.Module): + def __init__(self): + super(MyModel, self).__init__() + self.fc1 = nn.Linear(784, 10) + self.sigmoid = nn.Sigmoid() + self.fc2 = nn.Linear(10, 1) + + def forward(self, x): + out = self.fc1(x) + out = self.sigmoid(out) + out = self.fc2(out) + + return out + +device = "cpu" +epochs = 10 + +model = MyModel().to(device) +criterion = nn.MSELoss() +optimizer = optim.SGD(model.parameters(), lr=0.001) +loss_list = [] + + +for epoch in range(epochs): + for idx, batch in enumerate(train_loader): + x, target = batch + + x = x.T + target = target.T + + x = x.to(device) + target = target.to(device) + + outputs = model(x) + + loss = criterion(outputs, target) + + optimizer.zero_grad() + loss.backward() + print('loss_antes', loss) + + print('f1 antes', model.fc1.bias) + print('f1 grad_antes', model.fc1.bias.grad) + print('f2 antes', model.fc2.bias) + print('f2 grad_antes', model.fc2.bias.grad) + + optimizer.step() + print('\n') + + print('f1 depois', model.fc1.bias) + print('f2 depois', model.fc2.bias) + print('\n\n') + + print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}') + loss_list.append(loss[0]) \ No newline at end of file