From ffe10e48c50313a851a51892a2b102bf91e5c0ab Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Tue, 7 May 2024 00:10:25 -0300 Subject: [PATCH] Fix grad update step! loss decreasing --- norch/__pycache__/tensor.cpython-38.pyc | Bin 12577 -> 12501 bytes norch/nn/__pycache__/module.cpython-38.pyc | Bin 4043 -> 4054 bytes norch/nn/__pycache__/parameter.cpython-38.pyc | Bin 728 -> 728 bytes norch/nn/module.py | 2 +- .../optimizers/__pycache__/sgd.cpython-38.pyc | Bin 1150 -> 1195 bytes norch/optim/optimizers/sgd.py | 8 +++++--- norch/tensor.py | 4 +--- norch/utils/__pycache__/utils.cpython-38.pyc | Bin 636 -> 776 bytes norch/utils/utils.py | 4 +++- test.py | 14 ++++++-------- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 62b221093a148680694adcbb27692e701b87604b..6f19674784a797d892940f1477c5d115b29a660d 100644 GIT binary patch delta 1499 zcmY+CUu;u#6vywmy}iA??QOSpUB^Z{wsq^;u`z}ULt(H?!97UE_}eiqgVPl{*}7jl z2Iej^7GsFU2Yw0uc_3Rf8VttZjrxd2gNd4GG`{S~81=#MU`*767d@v{)F%CW&;6eB zJHLB=_qTTX+iB~tWtj^7CSMwwxp>R^6;?8!C^;nysAScgr=YERiTWCVlJl(^RmcBd zA7M>hgRF_e=tcc9WCQ3!-wSGTxnjodp!G9h`3k0bXq%QukRuZxv%?n0gTD88F&9Q z!T?z}Y<>(-aK8xz0aoO9!J(C}=mR)PK`EK2OehW={j4Dd$pM5s%&H<&RynG`GRzre z52_V-1{|uKQS-2@mbHqq>~U1_g`=)|*OU|LvbOAXw1Qskgbu~FJ{NK{O)2P(ce|o0 zwoZ8;<^5uOgu_+Cq4!?a%YJ?;n2ugiN0q6;TC;M%(aLH?71tci@yz;2)zu)y-WmG( zpg5jY%h5PmxVA7N|FFiakciV$eho36T!L zCQc}#nI6+15+pZ3B;~FCNTdI$oS3)sVjABdyQC5n_h$GAzzx?^-=lgDlJUMAh+OY@ zf+BH%5n+ro8VTz@+b)%bZNDdfi5%1qvA>I(%TqB;&lM^2rJHXY1{jv_w8!Nq(IR{; z3$e63+4Mu;2$vsaWdB~{ho9X-Yz(5u*geh|WK?IawfL!dFI3A--h1)1ZL1(>V?ET- zluH}d11EWb&G@IQ=Bw7VaNRcd?Uvsm@NfU9JdkRZcQziQS8R(P3QY3Kt&9PFDL?IU ze(_1$9+a=fpSE(8NUI{r`N{i(RtvInG_fx*%9$BP^&EKlTrO|hQ}W%!KB#(ElU}W4 z+o#;7Hou2<-PF-nnbmu&l{yU_JA=(SO)&vUgN@~u=5Zol?Hio4%UX5{xh+X4&!ZxRaH zwkV6m9G<&i+dsN{)B8ca2lAudhT(;+!-w92La1F%DW z+CNhp)zC3lo<4K-arxMmJ}WH(yiJf{$!E7rtndXEyBQ;l>is#)(Fw*B;}m0>!Dlb> zjB|_v<2+-YQDg{4iBV=OGAWw<1 V&WswS889#~5XX0(zn<+p=fgJnAhAgt2Pcq_kb?OrBu$EnN>va6+8V+(VO|0W#Af}% zNA#>flM~{$$4banQ6+k)gx;-H50!eN5>lm7N)A<3^+bg#_10562CG9Lge?kyqF^8te71ifPz@48H2`6o)o|St`L`d9 zno)y76vr`$p>-%Upo!siQLr$AQPLPTphen}NNw;DjZYS-;9QKoT z;Q;O+P2o-)Bu(QG?jqfW!#G0Pjk|G_v2SYG$5=_uvHDa)I}xeMEGqX>IT=0*TwYi&16o4>vv$yQ1@NG z>st(w8dZzzz#Lr&5U#IE>Djt^|LO_-qxEK@QFT$5DoNM~ICj~wRY>P!cM}K5bBCUC zjvyciej}!z#Ku+)dgW)5o^l7l6;)9a6>7B#H9=L#k_K;?YNK$!PmPjn1krY8E?iV8 z_8F>|q~LR|lvJFoO(b|+c*RylIj?5$0lk)k1OX=fSDN@xy(^8!0Y3G3!KkM@$yPmGNXoLKr2I^O5=MPk_0owkfN_1h zEu|~TB0Tk8BrS-(%URxIocnvi45F03>KDm=?^tU$Bnuoq%h=5bF*wnA;5l1@<>gr^ zLwcm6UuWBYj=s;>qFAsNXLK?+)SruI+tlVPdiV+wBFg3g}gr5b<3Kkc`vt4 zP4kMP+`^UR%Th|v+Y(9Lk{XE~;p`U~XX-0hvgf6o)eEUNVAD}5mZh}4$EmQ7ihSQa zSYK*fxA#QhqL=MyASwQ89;^0Jds@w$}v^p}~RC|t@7=*PXI(Gu55GwS{5 zbT$kRX-vj@E4vJTwX>Pb*{owlrF7J?jhC)T`PE0o?Od*un~s+Mm+P0D1<$?Tb4LOG z@FoWK05p3yc02;e=*hvaVL>+z72%Lx9hz%D!&Cozkte@&h@l$V!_0SK(7Tc*9+$h(A1l^Mum0b*w$E>-{%H4F06BPlYg;kF$zsqWiOI0(gX?>X@LknIKdBMX>Z=ZuEWl#J^3zQ LHKW$#Nd6)K#qk>x delta 109 zcmca6e_Eb5l$V!_0SJCqTBg0&$h(AXvJDHbkQ7j)hG7B2LIxLxSe{zO8pg@j*i_h< zig+1gHQ2 diff --git a/norch/nn/module.py b/norch/nn/module.py index d0ffaf1..eaabd9a 100644 --- a/norch/nn/module.py +++ b/norch/nn/module.py @@ -47,7 +47,7 @@ class Module(ABC): yield module._grads def zero_grad(self): - for parameter in self.parameters(): + for _, _, parameter in self.parameters(): parameter.zero_grad() def to(self, device): diff --git a/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc b/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc index 869d1b15573861bb07a62fa670699f8980188121..a46889f9cecd1f65a7c9dd5b84cfd4b82145cd9d 100644 GIT binary patch delta 274 zcmX|5u}TCn5S__vHoGe;h$o&`c;8^BAXsR1t=Jr=vXCeq-g?GtmuRt{kWveKD}QHe zJO4qPfH?4md2iku=BM~5>x;Usfa~D(=;rIQeiJVRGs!q*Jhj{$cvxbOXK;D75CIny zM4*s4;f1GO1j&2SxCfpQa%P#&M=2$}T=Fzds`U9T1m#ppdDRnwPu{=eF+5fp&LQqo zIVpzHwB5?s#u>ZKKMd;DxW+jf4{3T**qx-|-H@*yba!uttZmr6!RowRH`CrvQl(h8l(}#%4wrhS-o8rdq}r z=31s&=2{k@C{qffBts1l*Rle6%r&ecAhj$tELlL71W1&nhSi**mZgNPhN*_RhNYRA zk)Z~p7OK;Rp^z(uA(%mv$*+hVXqP6tz%9s0u>xg4DAVm!uYO16jp9K!Sl$f^qU$ MW-ms$$r3D+0bFJ_)Bpeg diff --git a/norch/optim/optimizers/sgd.py b/norch/optim/optimizers/sgd.py index 4604f41..3539114 100644 --- a/norch/optim/optimizers/sgd.py +++ b/norch/optim/optimizers/sgd.py @@ -9,13 +9,15 @@ class SGD(Optimizer): self._cache = {'velocity': [p.zeros_like() for (_, _, p) in self.parameters]} def step(self): - for i, (module, name, parameter) in enumerate(self.parameters): + for i, (module, name, _) in enumerate(self.parameters): + parameter = getattr(module, name) + velocity = self._cache['velocity'][i] velocity = self.momentum * velocity - self.lr * parameter.grad - parameter += velocity + updated_parameter = parameter + velocity - setattr(module, name, parameter) + setattr(module, name, updated_parameter) self._cache['velocity'][i] = velocity diff --git a/norch/tensor.py b/norch/tensor.py index ac0d996..845ceb4 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -164,9 +164,7 @@ class Tensor: visited.add(tensor) def zero_grad(self): - tmp = self.zeros_like() - self.detach() - self.grad = tmp + self.grad = None def __getitem__(self, indices): if len(indices) != self.ndim: diff --git a/norch/utils/__pycache__/utils.cpython-38.pyc b/norch/utils/__pycache__/utils.cpython-38.pyc index 1abde9b2fe37603503bf60ac3f5d4a5a603522fa..2285e44016ab030e2d805a098095fe51c03a36a5 100644 GIT binary patch delta 401 zcmZ8c%}T>S5T2QA(zH#aC$UhkLOh6gQ9%lVFCfKBv6RB5wUBJ;wmk?H4;J+r@&bCy z#fR`Q_6>XoL1$ACabSLSzWL^x{Sq%;col{XU~3eopb2+?n)uuH7sz&lWlm!3-%s7!x*K_RA9G0N##R8v-Ej$vs4p5w9 z`$1cc5>0ceRIG>do6#(`AZYtl-Q>_>p9>BikX3M!SQ(dPCUJzu{X+>YrQ!u#xt3=O x5CIAbFu-lnwh;+!_Ze56EYsPYAvLS|8{ta*_XOjgrI}nNaTl3V-QlC}`~b3IRtW$A delta 298 zcmYk0y-LJD5XWa`H!*6`Xc`L(8(|eh&_YD65?>$)av&kFd+{L2$tHFO7I&Ps;a)(? zdl?^KUquUN?<^dc-+cXNnD_8UCih9Q18k??SEEIr9HuE38Z=fg1Wj7ZVTk?RF+TM5 z$;EYtANUC+PFZmPDP-^lBh*p{^QfaucZRnx!wj^@K!yF+<8q&lE?1z?rKM?%Ejp9i zqS2E&uf}bs$kExk!+eEqcigZbZ=Z^1W5sfci&rI_s7&L+&58>+mWGc7e=2_1Zh35b rNQPit+Ao*Ijf|PxMP=2FO)Q{#Hc41`|VA diff --git a/norch/utils/utils.py b/norch/utils/utils.py index 2614a09..e977d6d 100644 --- a/norch/utils/utils.py +++ b/norch/utils/utils.py @@ -1,4 +1,6 @@ import random +import numpy as np + def generate_random_list(shape): """ @@ -9,7 +11,7 @@ def generate_random_list(shape): else: inner_shape = shape[1:] if len(inner_shape) == 0: - return [random.random()] * shape[0] + return [random.uniform(-1, 1) for _ in range(shape[0])] else: return [generate_random_list(inner_shape) for _ in range(shape[0])] \ No newline at end of file diff --git a/test.py b/test.py index 0b30999..b2af4cf 100644 --- a/test.py +++ b/test.py @@ -93,24 +93,22 @@ if __name__ == "__main__": return out modelo = MeuModulo() - input_list = [[0.05 for _ in range(5)]] + input_list = [[0.5 for _ in range(5)]] input = norch.Tensor(input_list).T criterion = nn.MSELoss() - optimizer = norch.optim.SGD(modelo.parameters(), lr=0.1) + optimizer = norch.optim.SGD(modelo.parameters(), lr=1) - target_list = [[0.1 for _ in range(2)]] + target_list = [[random.random() for _ in range(2)]] target = norch.Tensor(target_list).T - - for epoch in range(50): - cpu_percent = psutil.cpu_percent(interval=1) - memory_usage = psutil.virtual_memory() + + for epoch in range(10): output = modelo(input) loss = criterion(output, target) optimizer.zero_grad() loss.backward() + #print('fora grad', modelo.layer1.weight.grad, "\n\n") optimizer.step() - print(loss)