From 22d1607f6d8a16c5f6aa302ef764f855642cd199 Mon Sep 17 00:00:00 2001
From: Andrej Karpathy <andrej.karpathy@gmail.com>
Date: Thu, 30 May 2024 20:11:59 +0000
Subject: [PATCH] delete these files, they are now in scripts/ folder and
 better

---
 run124M.sh | 46 ----------------------------------------------
 run350M.sh | 47 -----------------------------------------------
 2 files changed, 93 deletions(-)
 delete mode 100644 run124M.sh
 delete mode 100644 run350M.sh

diff --git a/run124M.sh b/run124M.sh
deleted file mode 100644
index 4c9735f..0000000
--- a/run124M.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-
-# GPT-2 (124M) repro on FineWeb
-# 124M parameter model on 10B tokens
-# => 6 * 124e6 * 10e9 = 7.44e18 ~= 7e18 capability model
-# 18,865 steps of 524,288 tokens/step
-# on 8X A100 80GB SXM ($14/hr) steps in ~300ms/iter
-# => training time 18,865 * 300ms = 94.3 min ~= $20
-
-# if you are running out of memory on your GPU,
-# - try -r 1 (recompute GeLU, trading off speed and memory)
-# - start dividing -b 64 by 2 (i.e. 32, 16, ...) until it works
-
-make train_gpt2cu USE_CUDNN=1
-done_file="log124M/DONE_00018865"
-
-# in case the training stalls or crashes, loop to resume (-y 1)
-while true; do
-
-    # exit condition is that optimization has finished
-    if [ -f "$done_file" ]; then
-        echo "File $done_file exists. Exiting the loop."
-        break
-    fi
-
-    # run python dev/data/fineweb.py --version 10B to prepro data
-    # run python dev/data/hellaswag.py to prepro hellaswag eval
-    mpirun -np 8 ./train_gpt2cu \
-                -i "dev/data/fineweb10B/fineweb_train_*.bin" \
-                -j "dev/data/fineweb10B/fineweb_val_*.bin" \
-                -o log124M \
-                -v 250 -s 20000 -g 144 \
-                -h 1 \
-                -b 64 -t 1024 \
-                -d 524288 \
-                -r 0 \
-                -z 1 \
-                -c 0.1 \
-                -l 0.0006 \
-                -q 0.0 \
-                -u 700 \
-                -n 5000 \
-                -y 1 \
-                -e "d12"
-
-    sleep 1
-done
diff --git a/run350M.sh b/run350M.sh
deleted file mode 100644
index d1c3af8..0000000
--- a/run350M.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-
-# GPT-2 (350M) repro on FineWeb
-# 350M parameter model on ~30B tokens
-# => 6 * 350e6 * 31.5e9 = 6.615e19 ~= 7e19 capability model (10X 124M)
-# 60K steps on 524,288 tokens/step
-# on 8X A100 80GB SXM ($14/hr) steps in ~820ms/iter
-# => training time 60,000 steps * 820ms = 13.7 hours ~= $200 (10X 124M)
-
-# if you are running out of memory on your GPU,
-# - try -r 1 (recompute GeLU, trading off speed and memory)
-# - start dividing -b 64 by 2 (i.e. 32, 16, ...) until it works
-
-make train_gpt2cu USE_CUDNN=1
-done_file="log350M/DONE_00060000"
-
-# in case the training stalls or crashes, loop to resume (-y 1)
-while true; do
-
-    # exit condition is that optimization has finished
-    if [ -f "$done_file" ]; then
-        echo "File $done_file exists. Exiting the loop."
-        break
-    fi
-
-    # run python dev/data/fineweb.py --version 100B to prepro data
-    # run python dev/data/hellaswag.py to prepro hellaswag eval
-    mpirun -np 8 ./train_gpt2cu \
-                -i "dev/data/fineweb100B/fineweb_train_*.bin" \
-                -j "dev/data/fineweb100B/fineweb_val_*.bin" \
-                -o log350M \
-                -v 250 -s 100000 -g 144 \
-                -h 1 \
-                -b 64 -t 1024 \
-                -d 524288 \
-                -r 0 \
-                -z 1 \
-                -c 0.1 \
-                -l 0.0003 \
-                -q 0.0 \
-                -u 700 \
-                -n 2000 \
-                -x 60000 \
-                -y 1 \
-                -e "d24"
-
-    sleep 1
-done