aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixes -Ofast optimizations breaking model by skipping them for gelu_backward

Browse source
This commit is contained in:
Coenraad 2024-04-20 21:33:00 +02:00
parent 4e3b7ea0f3
commit f534e4bdfe
2 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -1,5 +1,5 @@
CC ?= clang
CFLAGS = -Ofast -fno-finite-math-only -Wno-unused-result -march=native
CFLAGS = -Ofast -Wno-unused-result -march=native
LDFLAGS =
LDLIBS = -lm
INCLUDES =

3
train_gpt2.c
View file

@ -363,6 +363,8 @@ void gelu_forward(float* out, float* inp, int N) {
}
}
#pragma float_control(precise, on, push) // On msvc /fp:fast is a lot faster, but the expf inside coshf breaks the model
__attribute__((optimize("no-finite-math-only"))) // same for gcc -Ofast
void gelu_backward(float* dinp, float* inp, float* dout, int N) {
for (int i = 0; i < N; i++) {
float x = inp[i];
@ -375,6 +377,7 @@ void gelu_backward(float* dinp, float* inp, float* dout, int N) {
dinp[i] += local_grad * dout[i];
}
}
#pragma float_control(pop)
void residual_forward(float* out, float* inp1, float* inp2, int N) {
for (int i = 0; i < N; i++) {