From c6efd90a861e038b0bfb5c9cd4c19199a6f5f9eb Mon Sep 17 00:00:00 2001
From: Andrej Karpathy <andrej.karpathy@gmail.com>
Date: Tue, 2 Jul 2024 16:49:09 +0000
Subject: [PATCH] upload good 1558M sh script

---
 scripts/run_gpt2_1558M.sh | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/scripts/run_gpt2_1558M.sh b/scripts/run_gpt2_1558M.sh
index 28095a6..9290412 100644
--- a/scripts/run_gpt2_1558M.sh
+++ b/scripts/run_gpt2_1558M.sh
@@ -1,13 +1,13 @@
-# GPT-2 (1558M) repro on FineWeb
-# 1558M parameter model on 100B tokens
-# => 6 * 1558Me6 * 100e9 = 6.966e20 ~= 1e21 capability model
-# => 100,000 steps on 1M tokens/step (1,048,576 to be precise)
-# on 8X A100 80GB SXM ($14/hr) steps in ~7s/iter
-# => training time 100,000 steps * 7s = 194.4 hours ~= 8.1 days ~= $2721.6
+# GPT-2 (1558M) repro on FineWeb-EDU
+# 1558M parameter model on 32B tokens
+# => 6 * 1558e6 * 32e9 = 6.966e20 ~= 3e20 capability model
+# 32,000 steps on ~1M tokens/step (1,048,576 to be precise)
+# on 8X H100 80GB SXM ($28/hr) steps in 2.80s/iter
+# => training time 32,000 steps * 2.7s => 24 hours ~= 1 day ~= $672
 
 make train_gpt2cu USE_CUDNN=1
 out_dir="log_gpt2_1558M"
-done_file="$out_dir/DONE_00100000"
+done_file="$out_dir/DONE_00032000"
 
 # in case the training stalls or crashes, loop to resume (-y 1)
 while true; do
@@ -18,24 +18,24 @@ while true; do
         break
     fi
 
-    # run python dev/data/fineweb.py --version 100B to prepro data
-    # run python dev/data/hellaswag.py to prepro hellaswag eval
     mpirun -np 8 ./train_gpt2cu \
-                -i "dev/data/fineweb100B/fineweb_train_*.bin" \
-                -j "dev/data/fineweb100B/fineweb_val_*.bin" \
+                -i "dev/data/edu_fineweb100B/edu_fineweb_train_*.bin" \
+                -j "dev/data/edu_fineweb100B/edu_fineweb_val_*.bin" \
                 -o $out_dir \
-                -v 250 -s 300000 -g 144 \
+                -v 250 -s 300000 -g 384 \
                 -h 1 \
                 -b 16 -t 1024 \
                 -d 1048576 \
                 -r 0 \
                 -z 1 \
                 -c 0.1 \
-                -l 0.0002 \
+                -k "cosine" \
+                -l 0.0006 \
                 -q 0.1 \
                 -u 700 \
                 -n 2000 \
-                -x 100000 \
+                -x 32000 \
+                -ge 1 \
                 -y 1 \
                 -e "d48"