From f534e4bdfec2384bd627649eb7e3c3cbe97d6f8b Mon Sep 17 00:00:00 2001
From: Coenraad <coenraad@wish.org.za>
Date: Sat, 20 Apr 2024 21:33:00 +0200
Subject: [PATCH] Fixes -Ofast optimizations breaking model by skipping them
 for gelu_backward

---
 Makefile     | 2 +-
 train_gpt2.c | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 632b850..4b1d828 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 CC ?= clang
-CFLAGS = -Ofast -fno-finite-math-only -Wno-unused-result -march=native
+CFLAGS = -Ofast -Wno-unused-result -march=native
 LDFLAGS =
 LDLIBS = -lm
 INCLUDES =
diff --git a/train_gpt2.c b/train_gpt2.c
index 7af8408..3b3ab70 100644
--- a/train_gpt2.c
+++ b/train_gpt2.c
@@ -363,6 +363,8 @@ void gelu_forward(float* out, float* inp, int N) {
     }
 }
 
+#pragma float_control(precise, on, push) // On msvc /fp:fast is a lot faster, but the expf inside coshf breaks the model
+__attribute__((optimize("no-finite-math-only"))) // same for gcc -Ofast 
 void gelu_backward(float* dinp, float* inp, float* dout, int N) {
     for (int i = 0; i < N; i++) {
         float x = inp[i];
@@ -375,6 +377,7 @@ void gelu_backward(float* dinp, float* inp, float* dout, int N) {
         dinp[i] += local_grad * dout[i];
     }
 }
+#pragma float_control(pop) 
 
 void residual_forward(float* out, float* inp1, float* inp2, int N) {
     for (int i = 0; i < N; i++) {