From a39bd59a3f36a1fd048a2fd4e91a31a2da52cc83 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Tue, 21 May 2024 01:22:38 -0300 Subject: [PATCH] softmax operation --- build/tensor.o | Bin 50320 -> 50480 bytes norch/__pycache__/tensor.cpython-38.pyc | Bin 18512 -> 18654 bytes norch/csrc/tensor.cpp | 4 +-- norch/libtensor.so | Bin 194792 -> 194792 bytes norch/nn/functional.py | 22 ++++++++-------- norch/tensor.py | 16 +++++++++++- test.py | 3 ++- tests/test_nn.py | 10 +++---- tests/test_operations.py | 33 ++++++++++++++++++++++++ 9 files changed, 67 insertions(+), 21 deletions(-) diff --git a/build/tensor.o b/build/tensor.o index 85b704a056485eb2ee0d1b300fba4502c79a5ca6..10402c807d458bfecf83492a19979ee8a777d22e 100644 GIT binary patch delta 238 zcmbQx$-JS9d4dMxhK-ubLnbSP*^B0+=BDPA=$2;|rz)gmmSq-a=I3!uJ`-%u1LrX? zFl;^_Vrb2j!nB!b!Z${yJxrT_O`OKa$-)ej0)pVlf>XpdZA_YTnW^$;sztr9G1;ptFG|_RV40mXJ=9< zn7r!XF(z5o$wi0Em|DXpuR3JMv@v|LVYJw!!y1e`CTkwHXS^{v^RPWrz`n_=4%;za anEVpR;YgdTb;OR-;vmF=lF1(r$^!r{{Z;k= delta 196 zcmdnc#XO;td4dL`z(&pGA)58C6gx}k_Q0z14l*x diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 1192401fdc0823e0bd6545facd8c0af2aa1078eb..141689fc6244ffff0b732112c60c98f8e693cfe5 100644 GIT binary patch delta 1547 zcmeHHPe>F|7=PcJSsmAPU3XV^w@4DjSOr@X)ZI-i{}ZMM3!;=^aW-=nSA9-p^fn3Y zuUhJ(Zo%5AE@JGd&JkTA51oWk=+ZHQphHIATg^jYr%urf@Atm{-+SMWnY9J@un5l6 z1qF73&sJ&o`A4(Qoc<785QjqPhF2Eh#LAtf3&0GN;g`(8EHWTL**i|s#i%*V64PhU z09l4UA~r+5fM2&;N?NsS%a~fVf&a6IjpJb&Fk$8^f1$eO((IO$_o59Fam@{z{1p-d7 zc{rdssUPg%rJ4RC3}7nrwr-2U6Y-}0lkFBpJ;=OhxCXEx!r`u|ZtTUuAc;J-4|hlq z+=+=cA4x&hLs_LkQ(!@wS_<9_Ocj@HNU?HEyq%u6>)F=r(w^3^N?H8wds!S0IQXr*zq!;*SF=BmKA`9nSnMFU{$>Vd}nJxwN|2? T&^-1azE$FVrw@eq+Ij9LOY46T delta 1355 zcmd6nOK1~O6o&7)Gik?UCa+0sQ$(oM+Mw37C|28QYfG()qWD<&U@VM$g*5< zcErw}v#i4$$LXMG;no4#=Vkjpy_D`d--mw{_2D^(V@TmpC} z&u<;fWPssOHXGwYx1j(oD}&2%Oy(%K)KSF^I5keKr7c$ETwnSi*ns+JkBbW)b3&eK zg={_wv`;X6((@nvXWZGGs<{)WeU5eA#?!9y6r*|_B6D;+oy(pbO~7SicWN|*xiI#G z{mzJ|I(k1l87~ApRb@`Z3v6YaS&{q#>Y}K_wu70kyy^$>fk?iYkc zAS0HBYVFv_$b_98Pn+_9nUK3WTS1fg&dpj?9lDU66gTA4&gRHxBHE4}3^Y(dj`ysF zsMNbQOm+PwE#$aJZF>`#&`<8sHd3K)sF%G{K6{S-XAO z2Ktd$@*}$B2Re}FD2<^MmlH!rI^6g#j+o?#m?kF@h(8H8<&*BFa5XR?vN>0@U_p=@ zE9HE53~rUS*Ci7QIndim != tensor2->ndim) { - fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for element-wise multiplication\n", tensor1->ndim, tensor2->ndim); + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for element-wise division\n", tensor1->ndim, tensor2->ndim); exit(1); } @@ -664,7 +664,7 @@ extern "C" { for (int i = 0; i < ndim; i++) { if (tensor1->shape[i] != tensor2->shape[i]) { - fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for subtraction\n", tensor1->shape[i], tensor2->shape[i], i); + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for division\n", tensor1->shape[i], tensor2->shape[i], i); exit(1); } shape[i] = tensor1->shape[i]; diff --git a/norch/libtensor.so b/norch/libtensor.so index de7ec33a5d384ea53ebe16737bffeeb968ac13c6..81254adae29b489efe3a4ff78e3b660f92ca5fc6 100755 GIT binary patch delta 4254 zcmaKwdstM}8piipgER~vA_K@pbWjlyQHz6=VPWAh1th#=-a!IIAyhIg4J=bc0gqwJ&DNqN^xRg5R?m1n(tj90zUDDJD`%0dBsF87x zCFbzMHGFWYeDCl9&dSYuk7RRGiag@zTs!uedGqm4`9q23X(wXYar4_JeqkNv!jrdn zp9S*9Q(iiDP2N@aBV%{vEuRf%?2P&2&u_8s<$b5Ub?m5IcWyZ2h12BD`hkpROqD$^ zfU)uNqhEl*&&snK{1|tNlV80!i?IXprArGKpAsX7U+&BJ*l0Pr5ounN<@B#S7-y5@ z4NZQGwV2=e`WCaw+pl1X3vxyCAjbB~ms>m;FB&i3ygG*Qxsh_vcY4OBJ}HM@gLUi^ za&#*=Xq=pMJ$IP?6Qd+e+~R(WZueoM_4_pSE{$|$qa7@o+G3=|u56UU8=72Wq;gd* z*W?l-HK{T|le3Iu>$^wp%hkw&{g zI%?9+NU4x|uF2o1+zlPybnEK>2WnDPo2FXGXoUJwQ}q<1s!ud^j5esMQd9fsn5sU| z)H`%XReLqHn?}217fLi`cB3Wk=w_WJUvZ<|s?65pYPYH@?rgIY9Wb%Uj!BDGE?%+t z#dL!tQQ1Hi;q<_Ov@yk>Ri*gz&cv)yhppm(og`({TE%6s{kT=MgN>hAg}#?0-2wx_ z#?w|Y5j=OsDi(oBXRRU!{P%gQ*aKc|u!=e`<+4?@frp#mx0fX6Z{Y`%uEGy)z6L*d zwGDoIDNAzt5d|<1_A?5A3vQqQSoJFkJS0h5Z=(Qs_bv*6l~xo0zZEC|I&eko0SDSC zq7K|=uZT9#&Y%drUXq48DH!c2VV!mmxU^#8kE8n@iiC?{tj9| zUnjg};NxJyql%aVWD7=~Q$K9-XF$Z1B=_MeGJ|&%joJuCo=<4i1~62xA|tWG;N* z(s_!Q39g@ym4c-U6!8{Vk)Vi^;92lHumiMnm88CjiWm&$EzD9xG%jivDq<-pzn};U zXrHWz3NSSVTMU+fzk|lbF!q(CXTYbyu}k0s^T9k&p9UWo0XBe#(%}Pt1%2@`P0N4} zd=E?l{j-+Ch>JO(dIz|)0!Gj=QxP5D>6c){-Ns`j-c#`Vmth1STcwEgVC8D~z#HIc zF!fdVz}=wH9SyC84-5zAfFHdEA9x+y4Q4fF!-$LEbvT4z!Rs)3;OPp6f`iw?2u=gn zfM?!>5tKGyYr&+A@PQW4$y1W<tHDyRR{;($I2ZidQ*x9(!i64( zsJ(!`*~EkRs!B5C@_Mq zRXnGAAWavmyI;nL@{Ytd^8u_cnSg|sFIdIYZrPJ;*_p`x=f$4m7R0aM*4fi+6^I*Y zmpYpd)vAc678P@d$vha;r9jiVt5Op{Kg>KuSyR(k1Q5Jo$BbmK=Rni{g^NPZ7WJlr0D?92&R=7<^Kw% zA~hOJmFjT&2)d*W2ZyMYy{S(j*YnjOU<{L9J@``Eg{hfaji|5HRxdClwLf zZBL_Yd%7N9NbW`|BB5J+W>;KotpVeY(N+su3#UpIVbrK1h^#6C$g~apdeJl6pqwd9 z1*5Iou)|LDp{gXh2=I<6bZ;BFYKtSkBJAwdI2xxCdY;PGIC`xJYYdN9ch#F}intw% zp;JXD?@2!wq2MHPCO(vJm`>?CkP$~%bkTI0L9n&WP;GwPI)lno{pn2lr3gvQYSO)# z)QUkCP0q!5;}T|3NHKCMW>Z-)ax#jMb9OeZ#2_=#PUNt$bfg$_<%(vvB+ z1UWg$lwN|IlPTJg4j_j&q|iYOVsEdMbai4?Exnd#3wm+~%7jzY4vb|kRc(HJ{Zh)` z0Y?p`?chFCj}%^?Mwiqv6IpklN+|U&g(D+Fn`l8PQnE8BxfHWkFQfcYlsUbO_N!XI zLQV3ZpGvu&^`p*G_#?@EC$?_&Dhk`FzKg4>X6@v?U8?l^c#XYx_AvGQ;OC@zdz7RP zpGusdj)VtSU6r}Qyle?wVeOYr>MJ#q>p||WbCcTD3s3KyT&=(ZX_8KIm-J;-;}7!Q z9DmqT{2~4eYo^GL`F#9=Use7wKg+pY87;5oA*{V>S2e#rhQH!Kxg9*jaDQU>j(6}d rhvq;2k5q5v0NwuqO`AhD delta 4203 zcmaKwdt8)N8pq#rhH$tF%m7|N(GXD(id8Ewpjl>0pmapCju6#GCw0aWbUlggg)>Jb#S>3ws>5e=rPyN~!>&%mL)$W_`8~E&O)x53N zjn~ds|EaEHY`Ep^z4^R4!*Y9XPyS(sYO1y|{xID#bMH{gt(q}>Ub>91=hdWr=NVr- z+cNmT5x!!U`slzO&bZ~{L%Dp}+iKY7Nlq-(vbydIJ~P>pa5R<$S^jqPCQGms*5Bjv zr>h?w>ug}7)XEdTFg8_P`&B>2!Yunw-ea$*yH0g6umJVMnSPAlouJ-7+k^45<5mA} zz`w_<&wUH7iBqRF1Tdcdn)=>%uQTSQHa5;+eD_#&@HrpG%U)H-o<~~N7&YU9ALGfd zsCgFy7>luNxpa>usU?>&#WQNvm0pZ_s^_lyGyWh(z4QGb#t%iSp+6cKSBI-n*I+Fh zrjET1elS!`{b^NSW3X9KMz8e^qz7Is(&(WpXByzmB3)Wct+5*>n&r(RTz=5yB@=Cu z@*7>AG|@#VYjt_RL}nkzJ-Xa(qH$8z=yH#V@}%6V%fFiFu#{_cnP;MhQfBLNk%=Nr zkaKjIYNFYY#uQynqfI7s9IvY}bWy6&x{4vQ8ETNOB50gc!MX~hJgIzj}FZj@ADX-}Ayid(#U$)x)BV#XCW9-FSQgY57 zu!}uTisDsg7w5o&qjqr{%sOco#-|j;0)~KDU)#lKaNcRVNCTbD*hM}#?Hjw;1}^{3 zE>3_Q&f7&3`1&RIofT!*zu^a+et;kBcO8Ck`Ook>D>=%}Ur+!8jW!67yaI>o>j7CK{>7j`b@cWLM z7!CfIaFr;!f^ z1tTBK2aT?Zau*B%r}TjzYyi_hr7!&8_E7l25BtNPg99F>iDobt^m0>_J75&}%K%Lz zfCHb=#1e2>lqSkS?>}kcAXqV26PLhd@G%(moF=-t<0AuyfDzAYA`vvdpowg7K3EQ3 z1&@H^hiD?lh|b@~K?t}5Y(k)3Y7Z0|iWPuw4nybQ2g5b76a33aO*DY3Mrq<9xZ@>E z?5QZn#%SV2@Y<_bG1y_8Cf0y`V>Ph{eDyW>z&Ua7f&1gN9MP$xqI8*{iNWBI1au6( zF;Nq_;NnS|*bEj;##Vu!Ow+_|@XJI^n7y!)B=|u0>6(}V_MU;2f z0DlCnpp&a3{Dp6Grf3P~O!1drM&@~%A@I}z%F`>4kMwXhTxRYajEYRI!n^MY zyLbRP;{1jTx5gVVEGT(35A2!wrCm%m{4UK9Yj{$|jPf5-CDPcZR0|~ge`^;L+GLM$ zWM?9KOk?|TE8uhhfD#WWOW({WhjFKhp(lUw1R0-rb|6~`T@@tM`!9DqtP7^Th zh1+d=wKEZax!Eo*wCVjHj^3?^Gu)Na+Qg?j;#FJ;QeMIB`e2&`)sfJE0@o;cE!w?B z%Ow6sl@hn97RX5wf6yFiTDd0rw5c87sBOh~*;AVE#~i;c-qW%8D#Q!%NN6wKU{F$& zl`ickw4gw)Tl;Y{9+vNVXkufV2G%$lh(f%XYS&@8g02F|Bc9bne{}0;_s!OGul3}! zB0KgE;#G*JnQ=+|-p}9p?c_zgo|4z|z?i^J?WdXXgnke4_uB00LdUK~A>K%}D8rti zo&~(?)0r4q(oGYy+f1D7nAnPVolFf(Hg<1+;s&IpBHkb8vt#{+UXF{i1@ZZK;BIdd zuW-c8_-25U2h*^6>zr2%ecmO*{-$Ak_x-Un)!pN_X2k_UzXduW=(ZoUw7lhMdIeaQyR_;Q33b~PY9zX{Q zc`!RbS7pwK6%K5ZMNvRV~eT~9~nU-EWEGZjR-~Q+7+P!EVKs< zz488*mRZp28QNmueOMiR4aHXtqPrHXFK;mUS$Qze8bZUYJjiX)P)!8E?9|bf)?2wR zqUkgkGbct9v2FGgKWu)zA+Er8kCG9ZNNy9K(i)eoHDLS# zMQ%iE=V+otBV|c^MK+1!RJ#%V?x&xn+C?5kKqW;MVTX58yi^sm2;kN@DlI}+`SJ95 z5q5TYJpD@&T9eF+rvSnl&yAP6+J$BicVai_EkgMkT1zPSAKC@QgC~(^IWle|gAbfU z52P)Bva|(o+hiJ63_Wuyts^Am79**2D&=92HPfzQd~j?Uot8NXi8QPPIbJ2m$w(x> z66Dm<&=Smbk&;R2r9!5Ro=#hl!b@k+=@OhPyXl6^8J$9YrI>nQ3JokpxO*zS zA$4pjy)Sj;TY8NnrC8BtbgUE}&zW>n=7h{7&obnsrs->nE5kgQX*3Cg*xaRMt*zL~ zVA(Q9U(pq$;KR5k$INr3Er9o)OX1}xGb2MUGrt^V=4a4i3}QvLa+GPHL*=k|Eoe`r#;lr0HtyNc z5;3fz#<+_gaqe;@M4m)=)yvx4LK+}B`I z;^cvUm!_3^f#0Pm2E|uNv(=2K=AInC(rNrYev?I0%zi!{zt(Fu?dPXCPqop)gFKwY z)l?qjKMmp+jI^qShr2$R7{0m|KF}rl*Z&Gc6T8m`xj6ngKrxW6Po{5LNz3o^fv#yb Ny&X(ncb^9v{trN;GAsZ9 diff --git a/norch/nn/functional.py b/norch/nn/functional.py index 9b5c3ec..75654cf 100644 --- a/norch/nn/functional.py +++ b/norch/nn/functional.py @@ -4,15 +4,15 @@ def sigmoid(x): return 1.0 / (1.0 + (math.e) ** (-x)) def softmax(x, dim=None): - e = math.e ** x - s = e.sum(axis=dim, keepdim=True) - #l = s.log() - print('x', x) - print('\n\n') - #print('log', l) + if dim is not None and dim < 0: + dim = x.ndim + dim + + x_max = x.max(axis=dim, keepdim=True) + exp_x = math.e ** (x - x_max) - print('\n\n') - #print('x-log', x - l) - - - #return math.e ** (x - sum.log()) \ No newline at end of file + if dim is not None: + sum_exp_x = exp_x.sum(axis=dim, keepdim=True) + exp_x.zeros_like() + return exp_x / sum_exp_x + else: + sum_exp_x = exp_x.sum() + return exp_x / sum_exp_x \ No newline at end of file diff --git a/norch/tensor.py b/norch/tensor.py index 52a5558..49df15d 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -613,6 +613,9 @@ class Tensor: result_data.grad_fn = DivisionBackward(self, other) elif isinstance(self, Tensor) and isinstance(other, Tensor): + if other.numel == 1: + return self.__truediv__(other.tensor.contents.data[0]) + Tensor._C.tensor_div_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] Tensor._C.tensor_div_tensor.restype = ctypes.POINTER(CTensor) @@ -749,8 +752,12 @@ class Tensor: return result_data def sum(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + if axis == None: axis = -1 + Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.sum_tensor.restype = ctypes.POINTER(CTensor) @@ -786,12 +793,15 @@ class Tensor: return result_data def max(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + if axis == None: axis = -1 + Tensor._C.max_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.max_tensor.restype = ctypes.POINTER(CTensor) - print(axis, keepdim) result_tensor_ptr = Tensor._C.max_tensor(self.tensor, axis, keepdim) result_data = Tensor() @@ -824,8 +834,12 @@ class Tensor: return result_data def min(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + if axis == None: axis = -1 + Tensor._C.min_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.min_tensor.restype = ctypes.POINTER(CTensor) diff --git a/test.py b/test.py index f6125ae..6bd8b2a 100644 --- a/test.py +++ b/test.py @@ -7,8 +7,9 @@ import random random.seed(1) torch_tensor = norch.Tensor([[[2, 2], [-1, -1]], [[1., 2], [3, 3]]], requires_grad=True)#.to(self.device) +torch_tensor = norch.Tensor([[[2, 2], [2, 2]], [[2, 2.], [2, 2]]]) b = norch.nn.functional.softmax(torch_tensor) - +print(b) """a = norch.Tensor([[[4.186502456665039]]]) b = norch.Tensor([[[2.0, 2.0,],[-1.0, -1.0,]],[[1.0, 2.0,],[3.0, 3.0,]]]) diff --git a/tests/test_nn.py b/tests/test_nn.py index daaac21..1010c82 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -95,13 +95,13 @@ class TestNNModuleActivationFn(unittest.TestCase): """ # Test different axes - axes = [None, 0, 1, -1] + axes = [0, 1, 2, -1] # Define the input tensors for different test cases test_cases = [ - (norch.Tensor([[1., 2, 3], [4, 5, 6]]), torch.tensor([[1., 2, 3], [4, 5, 6]])), - (norch.Tensor([[1., -1, 0], [2, -2, 0]]), torch.tensor([[1., -1, 0], [2, -2, 0]])), - (norch.Tensor([[0., 0, 0], [0, 0, 0]]), torch.tensor([[0., 0, 0], [0, 0, 0]])) + (norch.Tensor([[[1., 2, 3], [4, 5, 6]]]), torch.tensor([[[1., 2, 3], [4, 5, 6]]])), + (norch.Tensor([[[1., -1, 0], [2, -2, 0]]]), torch.tensor([[[1., -1, 0], [2, -2, 0]]])), + (norch.Tensor([[[0., 0, 0], [0, 0, 0]]]), torch.tensor([[[0., 0, 0], [0, 0, 0]]])) ] for dim in axes: @@ -121,7 +121,5 @@ class TestNNModuleActivationFn(unittest.TestCase): softmax_torch_expected = softmax_fn_torch.forward(torch_input) # Compare the results - print(softmax_torch_result) - print(softmax_torch_expected) self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected)) diff --git a/tests/test_operations.py b/tests/test_operations.py index dffe0a1..961cb17 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -260,6 +260,17 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum(axis=-2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor, dim=-2) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_sum_axis_keepdim(self): """ @@ -301,6 +312,17 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max(axis=-1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.max(torch_tensor, dim=-1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_max_axis_keepdim(self): """ @@ -341,6 +363,17 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min(axis=-1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.min(torch_tensor, dim=-1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + def test_min_axis_keepdim(self): """